From 959d572aae6814cd74a6c5eacec74a52cb8432ee Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 4 May 2022 21:06:23 -0700 Subject: [PATCH] Spatial denoising; too much blur --- include/hdrplus/merge.h | 67 ++++++++++++++++++++++++++++++++++++++++- src/merge.cpp | 47 ++++++++++++++++++++++------- 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 827fa2f..2d10be7 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -96,6 +96,71 @@ class merge return img; } + void circshift(cv::Mat &out, const cv::Point &delta) + { + cv::Size sz = out.size(); + + // error checking + assert(sz.height > 0 && sz.width > 0); + + // no need to shift + if ((sz.height == 1 && sz.width == 1) || (delta.x == 0 && delta.y == 0)) + return; + + // delta transform + int x = delta.x; + int y = delta.y; + if (x > 0) x = x % sz.width; + if (y > 0) y = y % sz.height; + if (x < 0) x = x % sz.width + sz.width; + if (y < 0) y = y % sz.height + sz.height; + + + // in case of multiple dimensions + std::vector planes; + split(out, planes); + + for (size_t i = 0; i < planes.size(); i++) + { + // vertical + cv::Mat tmp0, tmp1, tmp2, tmp3; + cv::Mat q0(planes[i], cv::Rect(0, 0, sz.width, sz.height - y)); + cv::Mat q1(planes[i], cv::Rect(0, sz.height - y, sz.width, y)); + q0.copyTo(tmp0); + q1.copyTo(tmp1); + tmp0.copyTo(planes[i](cv::Rect(0, y, sz.width, sz.height - y))); + tmp1.copyTo(planes[i](cv::Rect(0, 0, sz.width, y))); + + // horizontal + cv::Mat q2(planes[i], cv::Rect(0, 0, sz.width - x, sz.height)); + cv::Mat q3(planes[i], cv::Rect(sz.width - x, 0, x, sz.height)); + q2.copyTo(tmp2); + q3.copyTo(tmp3); + tmp2.copyTo(planes[i](cv::Rect(x, 0, sz.width - x, sz.height))); + tmp3.copyTo(planes[i](cv::Rect(0, 0, x, sz.height))); + } + + cv::merge(planes, out); + } + + void fftshift(cv::Mat &out) + { + cv::Size sz = out.size(); + cv::Point pt(0, 0); + pt.x = (int) floor(sz.width / 2.0); + pt.y = (int) floor(sz.height / 2.0); + circshift(out, pt); + } + + void ifftshift(cv::Mat &out) + { + cv::Size sz = out.size(); + cv::Point pt(0, 0); + pt.x = (int) ceil(sz.width / 2.0); + pt.y = (int) ceil(sz.height / 2.0); + circshift(out, pt); + } + std::vector getReferenceTiles(cv::Mat reference_image); cv::Mat mergeTiles(std::vector tiles, int rows, int cols); @@ -109,7 +174,7 @@ class merge //temporal denoise std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); - std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector spatial_denoise(std::vector tiles, int num_alts, std::vector noise_variance, float spatial_factor); }; diff --git a/src/merge.cpp b/src/merge.cpp index 8c2972d..6636d9a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -97,6 +97,7 @@ namespace hdrplus cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); burst_images.merged_bayer_image = merged(vertical, horizontal); + cv::imwrite("merged.jpg", burst_images.merged_bayer_image); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -192,14 +193,14 @@ namespace hdrplus ////adding after here - //std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + std::vector spatial_denoised_tiles = spatial_denoise(reference_tiles_DFT, alternate_channel_i_list.size(), noise_variance, 0.1); //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised //// reference_tiles_DFT = spatial_denoised_tiles; // Apply IFFT on reference tiles (frequency to spatial) std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { + for (auto dft_tile : reference_tiles) { cv::Mat denoised_tile; cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); denoised_tile.convertTo(denoised_tile, CV_16U); @@ -351,18 +352,42 @@ namespace hdrplus } - // std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + std::vector merge::spatial_denoise(std::vector tiles, int num_alts, std::vector noise_variance, float spatial_factor) { - // double spatial_factor = 1; //to be added - // double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; + double spatial_noise_scaling = (TILE_SIZE * TILE_SIZE * (1.0 / 16)) * spatial_factor; - // //calculate the spatial denoising - // spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; - // std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; + // Calculate |w| using ifftshift + cv::Mat row_distances = cv::Mat::zeros(1, TILE_SIZE, CV_32F); + for(int i = 0; i < TILE_SIZE; ++i) { + row_distances.at(i) = i - offset; + } + row_distances = cv::repeat(row_distances.t(), 1, TILE_SIZE); + cv::Mat col_distances = row_distances.t(); + cv::Mat distances; + cv::sqrt(row_distances.mul(row_distances) + col_distances.mul(col_distances), distances); + ifftshift(distances); + + std::vector denoised; + // Loop through all tiles + for (int i = 0; i < tiles.size(); ++i) { + cv::Mat tile = tiles[i]; + float coeff = noise_variance[i] / num_alts * spatial_noise_scaling; - // merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) + // Calculate absolute difference + cv::Mat complexMats[2]; + cv::split(tile, complexMats); // planes[0] = Re(DFT(I)), planes[1] = Im(DFT(I)) + cv::magnitude(complexMats[0], complexMats[1], complexMats[0]); // planes[0] = magnitude + cv::Mat absolute_diff = complexMats[0].mul(complexMats[0]); + + // Division + cv::Mat scale; + cv::divide(absolute_diff, absolute_diff + distances * coeff, scale); + cv::merge(std::vector{scale, scale}, scale); + denoised.push_back(tile.mul(scale)); + } - // } + return denoised; + } std::pair merge::getNoiseParams( int ISO, \