diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index c7c5642..8a829f5 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -44,6 +44,20 @@ class merge */ private: + float tileRMS(cv::Mat tile) { + cv::Mat squared; + cv::multiply(tile, tile, squared); + return sqrt(cv::mean(squared)[0]); + } + + std::vector getNoiseVariance(std::vector tiles, float lambda_shot, float lambda_read) { + std::vector noise_variance; + for (auto tile : tiles) { + noise_variance.push_back(lambda_shot * tileRMS(tile) + lambda_read); + } + return noise_variance; + } + cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); for (int i = 0; i < input.cols; ++i) { @@ -88,7 +102,9 @@ class merge cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image); + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 4ed50e9..e49cec4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -10,9 +10,12 @@ namespace hdrplus void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + // 4.1 Noise Parameters and RMS + // Noise parameters calculated from baseline ISO noise parameters double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + // 4.2-4.4 Denoising and Merging // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; // cv::imwrite("ref.jpg", reference_image); @@ -45,7 +48,7 @@ void merge::process( hdrplus::burst& burst_images, \ // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); // Apply merging on the channel - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels @@ -150,14 +153,39 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image) { - + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read) { + // Get tiles of the reference image std::vector reference_tiles = getReferenceTiles(channel_image); - // TODO: Temporal Denoising + // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) + std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); + + // Apply FFT on reference tiles (spatial to frequency) + std::vector reference_tiles_DFT; + for (auto ref_tile : reference_tiles) { + cv::Mat ref_tile_DFT; + ref_tile.convertTo(ref_tile_DFT, CV_32F); + cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE|cv::DFT_COMPLEX_OUTPUT); + reference_tiles_DFT.push_back(ref_tile_DFT); + } + + // TODO: 4.2 Temporal Denoising + + // TODO: 4.3 Spatial Denoising - // TODO: Spatial Denoising + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE|cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; for (auto tile : reference_tiles) {