From 98830cc38f379c6b0e6f9c6f4dcee5765b5b45f4 Mon Sep 17 00:00:00 2001 From: cvachha Date: Tue, 3 May 2022 21:57:36 -0700 Subject: [PATCH] Progress on denoising and separated functions I put the temporal and spatial denoising into 2 separate functions and am implementing finding the absolute difference. I have also edited the merge.h file to add the function signatures. --- include/hdrplus/merge.h | 7 +- src/merge.cpp | 170 +++++++++++++++++++++++++--------------- 2 files changed, 112 insertions(+), 65 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 23c5d55..981937c 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -107,7 +107,12 @@ class merge float lambda_shot, \ float lambda_read); - std::vector getChannels(cv::Mat input_image); + std::vector getChannels(cv::Mat input_image); //helper function + + //temporal denoise + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + }; diff --git a/src/merge.cpp b/src/merge.cpp index 8e0fdc5..619d937 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -203,7 +203,76 @@ namespace hdrplus } // TODO: 4.2 Temporal Denoising + std::vector temporal_denoised_tiles = temporal_denoise(reference_tiles, reference_tiles_DFT, noise_varaince) + + + // TODO: 4.3 Spatial Denoising + + ////adding after here + + std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + //// + + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + + + + + + // 4.4 Cosine Window Merging + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); + } + + + //Helper function to get the channels from the input image + std::vector getChannels(cv::Mat input_image){ + std::vector channels[4]; + + for (int y = 0; y < input_image.rows; ++y) { + for (int x = 0; x < input_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(input_image.at(y, x)); + } + else { + channels[1].push_back(input_image.at(y, x)); + } + } + else { + if (x % 2 == 0) { + channels[2].push_back(input_image.at(y, x)); + } + else { + channels[3].push_back(input_image.at(y, x)); + } + } + } + } + return channels; + + } + + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { //goal: temporially denoise using the weiner filter //input: //1. array of 2D dft tiles of the reference image @@ -215,7 +284,7 @@ namespace hdrplus //tile_size = TILE_SIZE; - /* + double temporal_factor = 8.0 //8 by default double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; @@ -244,36 +313,53 @@ namespace hdrplus //get the dft of the alternate image //std::vector alternate_tiles_DFT; - - std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - //find the squared absolute difference across all the tiles + //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + //find reference_tiles_DFT - alternate_tiles_DFT_list + std::vector> tile_difference_list; //list of tile differences + for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { + std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; + tile_difference_list.push_back(single_tile_difference); + } - std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; - //use merged_image_tiles_fft into part 4.3 + // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + //get the real and imaginary components + /* + std::vector> absolute_difference_list; + for (auto individual_difference : tile_difference_list) { + for (int i =0; i < individual_difference.rows; i++ ) { + std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); + for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { + row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag + } + } + //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag + absolute_difference_list.push_back(single_tile_difference); + } + */ - // TODO: 4.3 Spatial Denoising + //find the squared absolute difference across all the tiles - // Apply IFFT on reference tiles (frequency to spatial) - std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { - cv::Mat denoised_tile; - cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); - denoised_tile.convertTo(denoised_tile, CV_16U); - denoised_tiles.push_back(denoised_tile); - } - reference_tiles = denoised_tiles; - //adding after here + std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + + std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; + + return merged_image_tiles_fft + + } + + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + double spatial_factor = 1; //to be added double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; @@ -282,52 +368,8 @@ namespace hdrplus std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) - - //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - - */ - - // 4.4 Cosine Window Merging - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); - } - - // Merge tiles - return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - } - - - //Helper function to get the channels from the input image - std::vector getChannels(cv::Mat input_image){ - std::vector channels[4]; - - for (int y = 0; y < input_image.rows; ++y) { - for (int x = 0; x < input_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(input_image.at(y, x)); - } - else { - channels[1].push_back(input_image.at(y, x)); - } - } - else { - if (x % 2 == 0) { - channels[2].push_back(input_image.at(y, x)); - } - else { - channels[3].push_back(input_image.at(y, x)); - } - } - } - } - return channels; - } - - //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + } // namespace hdrplus \ No newline at end of file