From 83250c3b707c039f9fdfb06e4ef47d5f9f4bcd73 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Mon, 18 Apr 2022 01:04:18 -0700 Subject: [PATCH 01/59] Create merge branch --- src/merge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/merge.cpp b/src/merge.cpp index aaa46db..b8d220c 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,7 +7,7 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ - std::vector>>>& aligements ) + std::vector>>>& alignments ) { } From 5421402c689ce2932c536a7400e566daa909ed0b Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 02/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 8 ++++++++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index d782cf6..95062ba 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; @@ -26,6 +29,11 @@ class merge int ISO, \ int white_level, \ double black_level ); + + private: + std::pair getNoiseParams( int ISO, \ + int white_level, \ + double black_level ); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 7df0a5d..28d67e3 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -12,7 +13,29 @@ void merge::process( const hdrplus::burst& burst_images, \ int white_level, \ double black_level ) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 9c937dad93cf26fafdc2f46b0ccadeef9222439d Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 03/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 8 ++++++++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index d782cf6..95062ba 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; @@ -26,6 +29,11 @@ class merge int ISO, \ int white_level, \ double black_level ); + + private: + std::pair getNoiseParams( int ISO, \ + int white_level, \ + double black_level ); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 7df0a5d..28d67e3 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -12,7 +13,29 @@ void merge::process( const hdrplus::burst& burst_images, \ int white_level, \ double black_level ) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 6947c94f2770f45da19f5cd263d8f6027777d911 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 04/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 3 +++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f980153..68956ea 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; diff --git a/src/merge.cpp b/src/merge.cpp index bb7b30b..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -9,7 +10,29 @@ namespace hdrplus void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 9f3ca4c452abb1ba3b4382e1a31a0177e53fdac5 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 04:09:44 -0700 Subject: [PATCH 05/59] Modify get noise param accordingly --- src/merge.cpp | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..bd2242b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,28 +11,7 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); - - -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); } } // namespace hdrplus From dd0dfbc03932d31e8b8efcdb29b42211ef221f12 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 06/59] Obtain reference tiles --- src/merge.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index bd2242b..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -12,6 +12,26 @@ void merge::process( const hdrplus::burst& burst_images, \ { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // Obtain tiles + std::vector reference_tiles; + cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + std::cout << reference_image.rows << " " << reference_image.cols << std::endl; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + + + + // cv::Mat outputImg = reference_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", outputImg1); } } // namespace hdrplus From f9ebe29b3589e1b1090146584536893dde0788d7 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 07/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 3 +++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f980153..68956ea 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; diff --git a/src/merge.cpp b/src/merge.cpp index bb7b30b..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -9,7 +10,29 @@ namespace hdrplus void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From b1d0832ac527f057bfee80b147493eab42b213ce Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Mon, 18 Apr 2022 01:04:18 -0700 Subject: [PATCH 08/59] Create merge branch --- src/merge.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..8cd71c4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,7 +8,11 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ +<<<<<<< HEAD std::vector>>>& alignments) +======= + std::vector>>>& alignments ) +>>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From 4d6d6824981a44015d97096e992a5be64043a54c Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 09/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 8cd71c4..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,11 +8,7 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ -<<<<<<< HEAD std::vector>>>& alignments) -======= - std::vector>>>& alignments ) ->>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From cdce3dc925ec262613cfbb9b5bac3be02509d2dc Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 04:09:44 -0700 Subject: [PATCH 10/59] Modify get noise param accordingly --- src/merge.cpp | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..bd2242b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,28 +11,7 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); - - -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); } } // namespace hdrplus From f363db4af9438ea1a757887102d13d7a046cca11 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 11/59] Obtain reference tiles --- src/merge.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index bd2242b..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -12,6 +12,26 @@ void merge::process( const hdrplus::burst& burst_images, \ { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // Obtain tiles + std::vector reference_tiles; + cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + std::cout << reference_image.rows << " " << reference_image.cols << std::endl; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + + + + // cv::Mat outputImg = reference_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", outputImg1); } } // namespace hdrplus From 3dc19fec6ea5e6bfb81866dd57965b118c3e4268 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 12/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 3 +++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f980153..68956ea 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; diff --git a/src/merge.cpp b/src/merge.cpp index bb7b30b..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -9,7 +10,29 @@ namespace hdrplus void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 8092d6db0995ca75568741661dd53b25f752d390 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Mon, 18 Apr 2022 01:04:18 -0700 Subject: [PATCH 13/59] Create merge branch --- src/merge.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..8cd71c4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,7 +8,11 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ +<<<<<<< HEAD std::vector>>>& alignments) +======= + std::vector>>>& alignments ) +>>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From 864ad269316dac6664f1a5a514fa75bcecbe7896 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 14/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 8cd71c4..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,11 +8,7 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ -<<<<<<< HEAD std::vector>>>& alignments) -======= - std::vector>>>& alignments ) ->>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From 3c01dd07fa2dbac5dbb972543c488583424c1ac1 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 04:09:44 -0700 Subject: [PATCH 15/59] Modify get noise param accordingly --- src/merge.cpp | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..bd2242b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,28 +11,7 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); - - -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); } } // namespace hdrplus From b4825748af725252e2028726994750231057ee3b Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 16/59] Obtain reference tiles --- src/merge.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index bd2242b..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -12,6 +12,26 @@ void merge::process( const hdrplus::burst& burst_images, \ { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // Obtain tiles + std::vector reference_tiles; + cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + std::cout << reference_image.rows << " " << reference_image.cols << std::endl; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + + + + // cv::Mat outputImg = reference_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", outputImg1); } } // namespace hdrplus From 576139d07afdd255b023a898b28771fe02c38813 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 17/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 21b8204..404c27b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -34,4 +34,23 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 274c98d164bac50aac553d5c6a428ad3fa2af9e1 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 18/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 404c27b..e99b4fb 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -53,4 +53,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 62946e72f6b253c288c0e2abf4f65f324c838976 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 19/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index e99b4fb..c937d8a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -72,4 +72,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 9a8b783f2c1b60954ae6855772e6d1db9a582fe8 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 20/59] Obtain reference tiles --- src/merge.cpp | 57 --------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index c937d8a..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -34,61 +34,4 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - } // namespace hdrplus From 9cadda29d8d33e6d238a69718faeca5d658ba002 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Thu, 28 Apr 2022 03:53:22 -0700 Subject: [PATCH 21/59] Implemented cosine window 1d and 2d --- include/hdrplus/merge.h | 30 ++++++++++++++++++++++++++++++ src/merge.cpp | 24 +++++++++++++----------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 68956ea..d3aa550 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -2,6 +2,7 @@ #include #include // all opencv header +#include #include "hdrplus/burst.h" namespace hdrplus @@ -26,6 +27,35 @@ class merge */ void process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments); + + private: + cv::Mat cosineWindow1D(cv::Mat input, int window_size = 16) { + cv::Mat output = input.clone(); + for (int i = 0; i < input.cols; ++i) { + output.at(0, i) = 1. / 2. - 1. / 2. * cos(2 * M_PI * (input.at(0, i) + 1 / 2.) / window_size); + } + return output; + } + + cv::Mat cosineWindow2D(cv::Mat tile) { + int window_size = tile.rows; // Assuming square tile + cv::Mat output_tile = tile.clone(); + + cv::Mat window = cv::Mat::zeros(1, window_size, CV_32F); + for(int i = 0; i < window_size; ++i) { + window.at(i) = i; + } + + cv::Mat window_x = cosineWindow1D(window, window_size); + window_x = cv::repeat(window_x, window_size, 1); + cv::Mat window_2d = window_x.mul(window_x.t()); + + cv::Mat window_applied; + cv::multiply(tile, window_2d, window_applied, 1, CV_32F); + return window_applied; + } + + std::vector getReferenceTiles(cv::Mat reference_image); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 21b8204..ed7602b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -14,24 +14,26 @@ void merge::process( const hdrplus::burst& burst_images, \ std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); // Obtain tiles - std::vector reference_tiles; cv::Mat reference_image = burst_images.grayscale_images_pad[0]; - std::cout << reference_image.rows << " " << reference_image.cols << std::endl; - for (int y = 0; y < reference_image.rows - 8; y += 8) { - for (int x = 0; x < reference_image.cols - 8; x += 8) { - cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); - reference_tiles.push_back(tile); - } - } - - + std::vector reference_tiles = getReferenceTiles(reference_image); // cv::Mat outputImg = reference_image.clone(); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); // cv::imwrite("tile0.jpg", outputImg1); } +std::vector merge::getReferenceTiles(cv::Mat reference_image) { + std::vector reference_tiles; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + return reference_tiles; +} + } // namespace hdrplus From 0ab9ee0fd726b974ad821a20568e406a27957238 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Fri, 29 Apr 2022 03:32:18 -0700 Subject: [PATCH 22/59] Implemented actual merging (4.4) --- include/hdrplus/merge.h | 19 +++++++++- src/merge.cpp | 78 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index d3aa550..bc098dc 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -5,12 +5,15 @@ #include #include "hdrplus/burst.h" +#define TILE_SIZE 16 + namespace hdrplus { class merge { public: + int offset = TILE_SIZE / 2; float baseline_lambda_shot = 3.24 * pow( 10, -4 ); float baseline_lambda_read = 4.3 * pow( 10, -6 ); @@ -29,7 +32,7 @@ class merge std::vector>>>& alignments); private: - cv::Mat cosineWindow1D(cv::Mat input, int window_size = 16) { + cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); for (int i = 0; i < input.cols; ++i) { output.at(0, i) = 1. / 2. - 1. / 2. * cos(2 * M_PI * (input.at(0, i) + 1 / 2.) / window_size); @@ -55,7 +58,21 @@ class merge return window_applied; } + cv::Mat cat2Dtiles(std::vector> tiles) { + std::vector rows; + for (auto row_tiles : tiles) { + cv::Mat row; + cv::hconcat(row_tiles, row); + rows.push_back(row); + } + cv::Mat img; + cv::vconcat(rows, img); + return img; + } + std::vector getReferenceTiles(cv::Mat reference_image); + + cv::Mat mergeTiles(std::vector tiles, int rows, int cols); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index ed7602b..0d20a53 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -13,27 +13,95 @@ void merge::process( const hdrplus::burst& burst_images, \ double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Obtain tiles - cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + // Obtain Reference tiles + cv::Mat reference_image = burst_images.grayscale_images_pad[burst_images.reference_image_idx]; std::vector reference_tiles = getReferenceTiles(reference_image); + // Temporal Denoising + + // Spatial Denoising + + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + cv::Mat merged = mergeTiles(windowed_tiles, reference_image.rows, reference_image.cols); + // cv::Mat outputImg = reference_image.clone(); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); + // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); // cv::imwrite("tile0.jpg", outputImg1); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { std::vector reference_tiles; - for (int y = 0; y < reference_image.rows - 8; y += 8) { - for (int x = 0; x < reference_image.cols - 8; x += 8) { - cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + for (int y = 0; y < reference_image.rows - offset; y += offset) { + for (int x = 0; x < reference_image.cols - offset; x += offset) { + cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); reference_tiles.push_back(tile); } } return reference_tiles; } +cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols){ + // 1. get all four subsets: original (evenly split), horizontal overlapped, + // vertical overlapped, 2D overlapped + std::vector> tiles_original; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_original.push_back(row); + } + + std::vector> tiles_horizontal; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_horizontal.push_back(row); + } + + std::vector> tiles_vertical; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_vertical.push_back(row); + } + + std::vector> tiles_2d; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_2d.push_back(row); + } + + // 2. Concatenate the four subsets + cv::Mat img_original = cat2Dtiles(tiles_original); + cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); + cv::Mat img_vertical = cat2Dtiles(tiles_vertical); + cv::Mat img_2d = cat2Dtiles(tiles_2d); + + // 3. Add the four subsets together + img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; + img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; + img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; + + return img_original; +} + } // namespace hdrplus From 9823b41183ec08a46c0303a41a00921a2a27897f Mon Sep 17 00:00:00 2001 From: cvachha Date: Fri, 29 Apr 2022 22:29:40 -0700 Subject: [PATCH 23/59] Added a few helper function signatures Added a few helper function signatures which are commented out in merge.h --- include/hdrplus/merge.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index bc098dc..671337a 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -30,7 +30,19 @@ class merge */ void process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments); - + + + /* + std::vector get_other_tiles(); //return the other tile list T_1 to T_n + + std::vector vector_math(string operation, reference_tile, other_tile_list); //for loop allowing operations across single element and list + + std::vector scalar_vector_math(string operation, scalar num, std::vector tile_list); //for loop allowing operations across single element and list + + std::vector average_vector(std::vector tile_list); //take average of vector elements + + */ + private: cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); From 8a098405f017468c3816d8bd19b31b3e94316069 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 01:08:39 -0700 Subject: [PATCH 24/59] Accessing channel data signatures --- include/hdrplus/merge.h | 4 +++ src/merge.cpp | 63 +++++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index bc098dc..ffba5d3 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -73,6 +73,10 @@ class merge std::vector getReferenceTiles(cv::Mat reference_image); cv::Mat mergeTiles(std::vector tiles, int rows, int cols); + + cv::Mat processChannel( const hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 0d20a53..6ea1707 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -13,31 +13,16 @@ void merge::process( const hdrplus::burst& burst_images, \ double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Obtain Reference tiles - cv::Mat reference_image = burst_images.grayscale_images_pad[burst_images.reference_image_idx]; - std::vector reference_tiles = getReferenceTiles(reference_image); - - // Temporal Denoising - - // Spatial Denoising - - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); - } - - // Merge tiles - cv::Mat merged = mergeTiles(windowed_tiles, reference_image.rows, reference_image.cols); - - // cv::Mat outputImg = reference_image.clone(); + // Call merge on each channel + cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; + reference_image.convertTo(reference_image, CV_32F); + + // Get Channels + // cv::Mat channel_0(reference_image.rows, reference_image.cols, CV_32F, (uchar*)reference_image.data, 2 * sizeof(float)); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); - // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); - // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); - // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); - // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); - // cv::imwrite("tile0.jpg", outputImg1); + + //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_0); + } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -104,4 +89,34 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols return img_original; } +cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image) { + + std::vector reference_tiles = getReferenceTiles(channel_image); + + // Temporal Denoising + + // Spatial Denoising + + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + cv::Mat merged = mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); + + // cv::Mat outputImg = channel_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); + // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); + // cv::cvtColor(merged, merged, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", merged); + return merged; +} + } // namespace hdrplus From 39466f893ae0356ddd30225c4de4b4d9a42e48c1 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 02:41:56 -0700 Subject: [PATCH 25/59] Finished merging channels (4.4) --- include/hdrplus/burst.h | 3 ++ include/hdrplus/merge.h | 4 +- src/merge.cpp | 91 +++++++++++++++++++++++++++++++---------- 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/include/hdrplus/burst.h b/include/hdrplus/burst.h index def6b78..c4c9dda 100644 --- a/include/hdrplus/burst.h +++ b/include/hdrplus/burst.h @@ -33,6 +33,9 @@ class burst // number of image (including reference) in burst int num_images; + + // Bayer image after merging, stored as cv::Mat + cv::Mat merged_bayer_image; }; } // namespace hdrplus diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f208c70..c7c5642 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -28,7 +28,7 @@ class merge * Outer most vector is per alternative image. * Inner most two vector is for horizontal & vertical tiles */ - void process( const hdrplus::burst& burst_images, \ + void process( hdrplus::burst& burst_images, \ std::vector>>>& alignments); @@ -86,7 +86,7 @@ class merge cv::Mat mergeTiles(std::vector tiles, int rows, int cols); - cv::Mat processChannel( const hdrplus::burst& burst_images, \ + cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image); }; diff --git a/src/merge.cpp b/src/merge.cpp index 6ea1707..4ed50e9 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,22 +7,81 @@ namespace hdrplus { -void merge::process( const hdrplus::burst& burst_images, \ +void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Call merge on each channel + // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - reference_image.convertTo(reference_image, CV_32F); + // cv::imwrite("ref.jpg", reference_image); - // Get Channels - // cv::Mat channel_0(reference_image.rows, reference_image.cols, CV_32F, (uchar*)reference_image.data, 2 * sizeof(float)); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // Get raw channels + std::vector channels[4]; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(reference_image.at(y, x)); + } else { + channels[1].push_back(reference_image.at(y, x)); + } + } else { + if (x % 2 == 0) { + channels[2].push_back(reference_image.at(y, x)); + } else { + channels[3].push_back(reference_image.at(y, x)); + } + } + } + } + + // For each channel, perform denoising and merge + for (int i = 0; i < 4; ++i) { + // Get channel mat + cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + + // Apply merging on the channel + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); - //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_0); + // Put channel raw data back to channels + channels[i] = merged_channel.reshape(1, merged_channel.total()); + } + + // Write all channels back to a bayer mat + std::vector merged_raw; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + merged_raw.push_back(channels[0][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[1][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } else { + if (x % 2 == 0) { + merged_raw.push_back(channels[2][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[3][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } + } + } + // Create merged mat + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); + // cv::imwrite("merged.jpg", merged); + + // Remove padding + std::vector padding = burst_images.padding_info_bayer; + cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); + cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); + burst_images.merged_bayer_image = merged(vertical, horizontal); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -89,15 +148,15 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols return img_original; } -cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ +cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image) { std::vector reference_tiles = getReferenceTiles(channel_image); - // Temporal Denoising + // TODO: Temporal Denoising - // Spatial Denoising + // TODO: Spatial Denoising // Process tiles through 2D cosine window std::vector windowed_tiles; @@ -106,17 +165,7 @@ cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ } // Merge tiles - cv::Mat merged = mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - - // cv::Mat outputImg = channel_image.clone(); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); - // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); - // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); - // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); - // cv::cvtColor(merged, merged, cv::COLOR_GRAY2RGB); - // cv::imwrite("tile0.jpg", merged); - return merged; + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } } // namespace hdrplus From 94d539bdd5665186bed5e51f1e69b81054e827f7 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 03:51:40 -0700 Subject: [PATCH 26/59] Finished noise variance (4.1) and added FFT/IFFT --- include/hdrplus/merge.h | 18 +++++++++++++++++- src/merge.cpp | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index c7c5642..8a829f5 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -44,6 +44,20 @@ class merge */ private: + float tileRMS(cv::Mat tile) { + cv::Mat squared; + cv::multiply(tile, tile, squared); + return sqrt(cv::mean(squared)[0]); + } + + std::vector getNoiseVariance(std::vector tiles, float lambda_shot, float lambda_read) { + std::vector noise_variance; + for (auto tile : tiles) { + noise_variance.push_back(lambda_shot * tileRMS(tile) + lambda_read); + } + return noise_variance; + } + cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); for (int i = 0; i < input.cols; ++i) { @@ -88,7 +102,9 @@ class merge cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image); + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 4ed50e9..e49cec4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -10,9 +10,12 @@ namespace hdrplus void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + // 4.1 Noise Parameters and RMS + // Noise parameters calculated from baseline ISO noise parameters double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + // 4.2-4.4 Denoising and Merging // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; // cv::imwrite("ref.jpg", reference_image); @@ -45,7 +48,7 @@ void merge::process( hdrplus::burst& burst_images, \ // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); // Apply merging on the channel - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels @@ -150,14 +153,39 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image) { - + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read) { + // Get tiles of the reference image std::vector reference_tiles = getReferenceTiles(channel_image); - // TODO: Temporal Denoising + // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) + std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); + + // Apply FFT on reference tiles (spatial to frequency) + std::vector reference_tiles_DFT; + for (auto ref_tile : reference_tiles) { + cv::Mat ref_tile_DFT; + ref_tile.convertTo(ref_tile_DFT, CV_32F); + cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE|cv::DFT_COMPLEX_OUTPUT); + reference_tiles_DFT.push_back(ref_tile_DFT); + } + + // TODO: 4.2 Temporal Denoising + + // TODO: 4.3 Spatial Denoising - // TODO: Spatial Denoising + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE|cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; for (auto tile : reference_tiles) { From fc4c380eb9c826b06ba7d481930bacdd492c8d40 Mon Sep 17 00:00:00 2001 From: cvachha Date: Sun, 1 May 2022 06:18:36 -0700 Subject: [PATCH 27/59] Added code in progress for temporal and spatial denoising I added some of my code for temporal and spatial denoising, but I commented it out since it still is incomplete since I need to find a way to allow for multiple image channels. I created a helper function which we could use to return the channels. I also edited the merge.h file to include the signature of the helper function. --- include/hdrplus/merge.h | 3 + src/merge.cpp | 421 +++++++++++++++++++++++++--------------- 2 files changed, 267 insertions(+), 157 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 8a829f5..261846b 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -105,6 +105,9 @@ class merge cv::Mat channel_image, \ float lambda_shot, \ float lambda_read); + + std::vector getChannels(cv::Mat input_image); + }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index e49cec4..09b154a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,193 +7,300 @@ namespace hdrplus { -void merge::process( hdrplus::burst& burst_images, \ - std::vector>>>& alignments) -{ - // 4.1 Noise Parameters and RMS - // Noise parameters calculated from baseline ISO noise parameters - double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - - // 4.2-4.4 Denoising and Merging - // Get padded bayer image - cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - // cv::imwrite("ref.jpg", reference_image); - - // Get raw channels - std::vector channels[4]; - - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(reference_image.at(y, x)); - } else { - channels[1].push_back(reference_image.at(y, x)); + void merge::process(hdrplus::burst& burst_images, \ + std::vector>>>& alignments) + { + // 4.1 Noise Parameters and RMS + // Noise parameters calculated from baseline ISO noise parameters + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // 4.2-4.4 Denoising and Merging + + // Get padded bayer image + cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; + // cv::imwrite("ref.jpg", reference_image); + + // Get raw channels + std::vector channels[4]; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(reference_image.at(y, x)); + } + else { + channels[1].push_back(reference_image.at(y, x)); + } } - } else { - if (x % 2 == 0) { - channels[2].push_back(reference_image.at(y, x)); - } else { - channels[3].push_back(reference_image.at(y, x)); + else { + if (x % 2 == 0) { + channels[2].push_back(reference_image.at(y, x)); + } + else { + channels[3].push_back(reference_image.at(y, x)); + } } } } - } - // For each channel, perform denoising and merge - for (int i = 0; i < 4; ++i) { - // Get channel mat - cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); - // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + // For each channel, perform denoising and merge + for (int i = 0; i < 4; ++i) { + // Get channel mat + cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); - // Apply merging on the channel - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); - // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); + // Apply merging on the channel + + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images - // Put channel raw data back to channels - channels[i] = merged_channel.reshape(1, merged_channel.total()); - } + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); - // Write all channels back to a bayer mat - std::vector merged_raw; + // Put channel raw data back to channels + channels[i] = merged_channel.reshape(1, merged_channel.total()); + } - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - merged_raw.push_back(channels[0][(y/2)*(reference_image.cols/2) + (x/2)]); - } else { - merged_raw.push_back(channels[1][(y/2)*(reference_image.cols/2) + (x/2)]); + // Write all channels back to a bayer mat + std::vector merged_raw; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + merged_raw.push_back(channels[0][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } + else { + merged_raw.push_back(channels[1][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } } - } else { - if (x % 2 == 0) { - merged_raw.push_back(channels[2][(y/2)*(reference_image.cols/2) + (x/2)]); - } else { - merged_raw.push_back(channels[3][(y/2)*(reference_image.cols/2) + (x/2)]); + else { + if (x % 2 == 0) { + merged_raw.push_back(channels[2][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } + else { + merged_raw.push_back(channels[3][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } } } } + + // Create merged mat + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); + // cv::imwrite("merged.jpg", merged); + + // Remove padding + std::vector padding = burst_images.padding_info_bayer; + cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); + cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); + burst_images.merged_bayer_image = merged(vertical, horizontal); } - - // Create merged mat - cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); - // cv::imwrite("merged.jpg", merged); - - // Remove padding - std::vector padding = burst_images.padding_info_bayer; - cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); - cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); - burst_images.merged_bayer_image = merged(vertical, horizontal); -} - -std::vector merge::getReferenceTiles(cv::Mat reference_image) { - std::vector reference_tiles; - for (int y = 0; y < reference_image.rows - offset; y += offset) { - for (int x = 0; x < reference_image.cols - offset; x += offset) { - cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); - reference_tiles.push_back(tile); + + std::vector merge::getReferenceTiles(cv::Mat reference_image) { + std::vector reference_tiles; + for (int y = 0; y < reference_image.rows - offset; y += offset) { + for (int x = 0; x < reference_image.cols - offset; x += offset) { + cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); + reference_tiles.push_back(tile); + } } + return reference_tiles; } - return reference_tiles; -} - -cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols){ - // 1. get all four subsets: original (evenly split), horizontal overlapped, - // vertical overlapped, 2D overlapped - std::vector> tiles_original; - for (int y = 0; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 0; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + + cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols) { + // 1. get all four subsets: original (evenly split), horizontal overlapped, + // vertical overlapped, 2D overlapped + std::vector> tiles_original; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_original.push_back(row); } - tiles_original.push_back(row); - } - std::vector> tiles_horizontal; - for (int y = 0; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 1; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_horizontal; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_horizontal.push_back(row); } - tiles_horizontal.push_back(row); - } - std::vector> tiles_vertical; - for (int y = 1; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 0; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_vertical; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_vertical.push_back(row); } - tiles_vertical.push_back(row); - } - std::vector> tiles_2d; - for (int y = 1; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 1; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_2d; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_2d.push_back(row); } - tiles_2d.push_back(row); - } - // 2. Concatenate the four subsets - cv::Mat img_original = cat2Dtiles(tiles_original); - cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); - cv::Mat img_vertical = cat2Dtiles(tiles_vertical); - cv::Mat img_2d = cat2Dtiles(tiles_2d); - - // 3. Add the four subsets together - img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; - img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; - img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; - - return img_original; -} - -cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ - std::vector>>>& alignments, \ - cv::Mat channel_image, \ - float lambda_shot, \ - float lambda_read) { - // Get tiles of the reference image - std::vector reference_tiles = getReferenceTiles(channel_image); - - // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) - std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); - - // Apply FFT on reference tiles (spatial to frequency) - std::vector reference_tiles_DFT; - for (auto ref_tile : reference_tiles) { - cv::Mat ref_tile_DFT; - ref_tile.convertTo(ref_tile_DFT, CV_32F); - cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE|cv::DFT_COMPLEX_OUTPUT); - reference_tiles_DFT.push_back(ref_tile_DFT); + // 2. Concatenate the four subsets + cv::Mat img_original = cat2Dtiles(tiles_original); + cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); + cv::Mat img_vertical = cat2Dtiles(tiles_vertical); + cv::Mat img_2d = cat2Dtiles(tiles_2d); + + // 3. Add the four subsets together + img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; + img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; + img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; + + return img_original; } - // TODO: 4.2 Temporal Denoising + cv::Mat merge::processChannel(hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read) { + // Get tiles of the reference image + std::vector reference_tiles = getReferenceTiles(channel_image); + + // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) + std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); + + // Apply FFT on reference tiles (spatial to frequency) + std::vector reference_tiles_DFT; + for (auto ref_tile : reference_tiles) { + cv::Mat ref_tile_DFT; + ref_tile.convertTo(ref_tile_DFT, CV_32F); + cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + reference_tiles_DFT.push_back(ref_tile_DFT); + } + + // TODO: 4.2 Temporal Denoising + + //goal: temporially denoise using the weiner filter + //input: + //1. array of 2D dft tiles of the reference image + //2. array of 2D dft tiles ocf the aligned alternate image + //3. estimated noise varaince + //4. temporal factor + //return: merged image patches dft + + /* + + //tile_size = TILE_SIZE; + double temporal_factor = 8 //8 by default + + double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; - // TODO: 4.3 Spatial Denoising + //start calculating the merged image tiles fft + + for (int i = 0;i < burst_images.num_images; i++) { + if (i != burst_images.reference_image_idx) { - // Apply IFFT on reference tiles (frequency to spatial) - std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { - cv::Mat denoised_tile; - cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE|cv::DFT_REAL_OUTPUT); - denoised_tile.convertTo(denoised_tile, CV_16U); - denoised_tiles.push_back(denoised_tile); + } + } + //sample of 0th image + altername_image = burst_images.bayer_images_pad[0] + + //get the tiles of the alternate image + std::vector alternate_image_tiles = getReferenceTiles(channel_image); + + //get the dft of the alternate image + std::vector alternate_tiles_DFT; + for (auto alt_tile : alternate_tiles_DFT) { + cv::Mat alt_tile_DFT; + alt_tile.convertTo(alt_tile_DFT, CV_32F); + cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + alternate_tiles_DFT.push_back(alt_tile_DFT); + } + + + + std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT; + std::vector tile_sq_asolute_diff = tile_differences; //tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + + std::vector merged_image_tiles_fft = alternate_tiles_DFT + A * tile_differences; + + //use merged_image_tiles_fft into part 4.3 + + + + // TODO: 4.3 Spatial Denoising + + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + + //adding after here + double spatial_factor = 1; //to be added + double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; + + //calculate the spatial denoising + spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; + std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; + + merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) + + //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + + */ + + + // 4.4 Cosine Window Merging + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } - reference_tiles = denoised_tiles; - // 4.4 Cosine Window Merging - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); + + //Helper function to get the channels from the input image + std::vector getChannels(cv::Mat input_image){ + std::vector channels[4]; + + for (int y = 0; y < input_image.rows; ++y) { + for (int x = 0; x < input_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(input_image.at(y, x)); + } + else { + channels[1].push_back(input_image.at(y, x)); + } + } + else { + if (x % 2 == 0) { + channels[2].push_back(input_image.at(y, x)); + } + else { + channels[3].push_back(input_image.at(y, x)); + } + } + } + } + return channels; + } - // Merge tiles - return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); -} + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments -} // namespace hdrplus +} // namespace hdrplus \ No newline at end of file From e44a831525d50d5dea829073bf1c3eb0bacb5c8b Mon Sep 17 00:00:00 2001 From: cvachha Date: Mon, 2 May 2022 03:01:31 -0700 Subject: [PATCH 28/59] Added a loop to get the channels and tiles for each alternate image Added some loops to get the channels for the alternate images. Then this list of channels from each alternate image gets sent to the processChannel function. Over there, there is a loop to create a list of the DFTs of each of the alternate images. I also modified the function signature of processChannel and commented out the code additions in processChannel since some of it is not complete yet. --- include/hdrplus/merge.h | 1 + src/merge.cpp | 69 ++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 261846b..23c5d55 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -103,6 +103,7 @@ class merge cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image, \ + std::vector alternate_channel_i_list,\ float lambda_shot, \ float lambda_read); diff --git a/src/merge.cpp b/src/merge.cpp index 09b154a..8e0fdc5 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -45,6 +45,7 @@ namespace hdrplus } } + ///// // For each channel, perform denoising and merge for (int i = 0; i < 4; ++i) { // Get channel mat @@ -56,7 +57,25 @@ namespace hdrplus //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + //create list of channel_i of alternate images: + std::vector alternate_channel_i_list; + for (int j = 0; j < burst_images.num_images; j++) { + if (j != burst_images.reference_image_idx) { + + //get alternate image + cv::Mat alt_image = burst_images.bayer_images_pad[j]; + std::vector alt_img_channel = getChannels(alt_image); //get channel array from alternate image + cv::Mat alt_channel_i(alt_image.rows / 2, alt_image.cols / 2, CV_16U, alt_img_channel[i].data()); + + alternate_channel_i_list.push_back(alt_channel_i) + } + } + + ///// + + //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, alternate_channel_i_list, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels @@ -165,6 +184,7 @@ namespace hdrplus cv::Mat merge::processChannel(hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image, \ + std::vector alternate_channel_i_list,\ float lambda_shot, \ float lambda_read) { // Get tiles of the reference image @@ -192,43 +212,50 @@ namespace hdrplus //4. temporal factor //return: merged image patches dft - /* + //tile_size = TILE_SIZE; - double temporal_factor = 8 //8 by default + /* + double temporal_factor = 8.0 //8 by default double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; //start calculating the merged image tiles fft - for (int i = 0;i < burst_images.num_images; i++) { - if (i != burst_images.reference_image_idx) { + + //get the tiles of the alternate image as a list + + std::vector> alternate_channel_i_tile_list; //list of alt channel tiles + std::vector> alternate_tiles_DFT_list; //list of alt channel tiles + for (auto alt_img_channel : alternate_channel_i_list) { + std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image + alternate_channel_i_tile_list.push_back(alt_img_channel_tile) + + std::vector alternate_tiles_DFT_list; + for (auto alt_tile : alt_img_channel_tile) { + cv::Mat alt_tile_DFT; + alt_tile.convertTo(alt_tile_DFT, CV_32F); + cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + alternate_tiles_DFT_list.push_back(alt_tile_DFT); } + alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); } - //sample of 0th image - altername_image = burst_images.bayer_images_pad[0] - - //get the tiles of the alternate image - std::vector alternate_image_tiles = getReferenceTiles(channel_image); //get the dft of the alternate image - std::vector alternate_tiles_DFT; - for (auto alt_tile : alternate_tiles_DFT) { - cv::Mat alt_tile_DFT; - alt_tile.convertTo(alt_tile_DFT, CV_32F); - cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); - alternate_tiles_DFT.push_back(alt_tile_DFT); - } + //std::vector alternate_tiles_DFT; + - std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT; - std::vector tile_sq_asolute_diff = tile_differences; //tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; + std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + //find the squared absolute difference across all the tiles std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) - std::vector merged_image_tiles_fft = alternate_tiles_DFT + A * tile_differences; + std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; //use merged_image_tiles_fft into part 4.3 @@ -258,9 +285,9 @@ namespace hdrplus //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + */ - // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; From f20002fcd3e2cc721e8c9f1ce3b8d65f554b4e6a Mon Sep 17 00:00:00 2001 From: cvachha Date: Tue, 3 May 2022 21:57:36 -0700 Subject: [PATCH 29/59] Progress on denoising and separated functions I put the temporal and spatial denoising into 2 separate functions and am implementing finding the absolute difference. I have also edited the merge.h file to add the function signatures. --- include/hdrplus/merge.h | 7 +- src/merge.cpp | 170 +++++++++++++++++++++++++--------------- 2 files changed, 112 insertions(+), 65 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 23c5d55..981937c 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -107,7 +107,12 @@ class merge float lambda_shot, \ float lambda_read); - std::vector getChannels(cv::Mat input_image); + std::vector getChannels(cv::Mat input_image); //helper function + + //temporal denoise + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + }; diff --git a/src/merge.cpp b/src/merge.cpp index 8e0fdc5..619d937 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -203,7 +203,76 @@ namespace hdrplus } // TODO: 4.2 Temporal Denoising + std::vector temporal_denoised_tiles = temporal_denoise(reference_tiles, reference_tiles_DFT, noise_varaince) + + + // TODO: 4.3 Spatial Denoising + + ////adding after here + + std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + //// + + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + + + + + + // 4.4 Cosine Window Merging + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); + } + + + //Helper function to get the channels from the input image + std::vector getChannels(cv::Mat input_image){ + std::vector channels[4]; + + for (int y = 0; y < input_image.rows; ++y) { + for (int x = 0; x < input_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(input_image.at(y, x)); + } + else { + channels[1].push_back(input_image.at(y, x)); + } + } + else { + if (x % 2 == 0) { + channels[2].push_back(input_image.at(y, x)); + } + else { + channels[3].push_back(input_image.at(y, x)); + } + } + } + } + return channels; + + } + + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { //goal: temporially denoise using the weiner filter //input: //1. array of 2D dft tiles of the reference image @@ -215,7 +284,7 @@ namespace hdrplus //tile_size = TILE_SIZE; - /* + double temporal_factor = 8.0 //8 by default double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; @@ -244,36 +313,53 @@ namespace hdrplus //get the dft of the alternate image //std::vector alternate_tiles_DFT; - - std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - //find the squared absolute difference across all the tiles + //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + //find reference_tiles_DFT - alternate_tiles_DFT_list + std::vector> tile_difference_list; //list of tile differences + for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { + std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; + tile_difference_list.push_back(single_tile_difference); + } - std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; - //use merged_image_tiles_fft into part 4.3 + // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + //get the real and imaginary components + /* + std::vector> absolute_difference_list; + for (auto individual_difference : tile_difference_list) { + for (int i =0; i < individual_difference.rows; i++ ) { + std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); + for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { + row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag + } + } + //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag + absolute_difference_list.push_back(single_tile_difference); + } + */ - // TODO: 4.3 Spatial Denoising + //find the squared absolute difference across all the tiles - // Apply IFFT on reference tiles (frequency to spatial) - std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { - cv::Mat denoised_tile; - cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); - denoised_tile.convertTo(denoised_tile, CV_16U); - denoised_tiles.push_back(denoised_tile); - } - reference_tiles = denoised_tiles; - //adding after here + std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + + std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; + + return merged_image_tiles_fft + + } + + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + double spatial_factor = 1; //to be added double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; @@ -282,52 +368,8 @@ namespace hdrplus std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) - - //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - - */ - - // 4.4 Cosine Window Merging - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); - } - - // Merge tiles - return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - } - - - //Helper function to get the channels from the input image - std::vector getChannels(cv::Mat input_image){ - std::vector channels[4]; - - for (int y = 0; y < input_image.rows; ++y) { - for (int x = 0; x < input_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(input_image.at(y, x)); - } - else { - channels[1].push_back(input_image.at(y, x)); - } - } - else { - if (x % 2 == 0) { - channels[2].push_back(input_image.at(y, x)); - } - else { - channels[3].push_back(input_image.at(y, x)); - } - } - } - } - return channels; - } - - //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + } // namespace hdrplus \ No newline at end of file From 3870fed854ffed1f8a1860c589c6cd945c3fa117 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 30/59] Compute lambda_shot and lambda_read --- include/hdrplus/merge.h | 3 +++ src/merge.cpp | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f980153..68956ea 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -10,6 +10,9 @@ namespace hdrplus class merge { public: + float baseline_lambda_shot = 3.24 * pow( 10, -4 ); + float baseline_lambda_read = 4.3 * pow( 10, -6 ); + merge() = default; ~merge() = default; diff --git a/src/merge.cpp b/src/merge.cpp index bb7b30b..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -1,5 +1,6 @@ #include // all opencv header #include +#include #include "hdrplus/merge.h" #include "hdrplus/burst.h" @@ -9,7 +10,29 @@ namespace hdrplus void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); + } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 8548e98dda7cb532b079adaddbd87495ff5c9346 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Mon, 18 Apr 2022 01:04:18 -0700 Subject: [PATCH 31/59] Create merge branch --- src/merge.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..8cd71c4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,7 +8,11 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ +<<<<<<< HEAD std::vector>>>& alignments) +======= + std::vector>>>& alignments ) +>>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From e201e7960e5d805ebb97df1da76487eb5311c92f Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 32/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 8cd71c4..81f3942 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,11 +8,7 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ -<<<<<<< HEAD std::vector>>>& alignments) -======= - std::vector>>>& alignments ) ->>>>>>> Create merge branch { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); From a47fba9c7f8b4a938610f373afee32d30487c3c4 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 04:09:44 -0700 Subject: [PATCH 33/59] Modify get noise param accordingly --- src/merge.cpp | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 81f3942..bd2242b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,28 +11,7 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); - - -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); } } // namespace hdrplus From fce0da52fcfb60bcf62c45567d681dc9c078ce1d Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 34/59] Obtain reference tiles --- src/merge.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index bd2242b..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -12,6 +12,26 @@ void merge::process( const hdrplus::burst& burst_images, \ { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // Obtain tiles + std::vector reference_tiles; + cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + std::cout << reference_image.rows << " " << reference_image.cols << std::endl; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + + + + // cv::Mat outputImg = reference_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", outputImg1); } } // namespace hdrplus From 03eb5e2af46cc62c6fdf27ae9d6d1366987854c9 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 35/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 21b8204..404c27b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -34,4 +34,23 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From b12c3383c424f0175ce1bfbfe081639a24ab6942 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 36/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 404c27b..e99b4fb 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -53,4 +53,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From ecd5ba5902e6f2df15e738d5e86d589bda894fb2 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 37/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index e99b4fb..c937d8a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -72,4 +72,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 4512999f3a8f3b7f97c0987ca76cccf47442f977 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 38/59] Obtain reference tiles --- src/merge.cpp | 57 --------------------------------------------------- 1 file changed, 57 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index c937d8a..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -34,61 +34,4 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - } // namespace hdrplus From 187bea5522625a44f7eb86ee04f4530dab994fce Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 39/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 21b8204..5fd40c0 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,6 +11,7 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; +<<<<<<< HEAD std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); // Obtain tiles @@ -23,6 +24,9 @@ void merge::process( const hdrplus::burst& burst_images, \ reference_tiles.push_back(tile); } } +======= + std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); +>>>>>>> Compute lambda_shot and lambda_read @@ -34,4 +38,23 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From f9ff858a31c68f870acb4819e5a96f0a2afe09cc Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Mon, 18 Apr 2022 01:04:18 -0700 Subject: [PATCH 40/59] Create merge branch --- src/merge.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 5fd40c0..7fa7d39 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,7 +8,11 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ +<<<<<<< HEAD std::vector>>>& alignments) +======= + std::vector>>>& alignments ) +>>>>>>> Create merge branch { double lambda_shot, lambda_read; <<<<<<< HEAD From 7325ae29d84677d83a2fd93c28f3a71bceb32f82 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 41/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 7fa7d39..5fd40c0 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -8,11 +8,7 @@ namespace hdrplus { void merge::process( const hdrplus::burst& burst_images, \ -<<<<<<< HEAD std::vector>>>& alignments) -======= - std::vector>>>& alignments ) ->>>>>>> Create merge branch { double lambda_shot, lambda_read; <<<<<<< HEAD From aae684e001dc6cf5b67539c624e364185b87b54d Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 04:09:44 -0700 Subject: [PATCH 42/59] Modify get noise param accordingly --- src/merge.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index 5fd40c0..404c27b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -11,7 +11,6 @@ void merge::process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; -<<<<<<< HEAD std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); // Obtain tiles @@ -24,9 +23,6 @@ void merge::process( const hdrplus::burst& burst_images, \ reference_tiles.push_back(tile); } } -======= - std::tie(lambda_shot, lambda_read) = merge::getNoiseParams(ISO, white_level, black_level); ->>>>>>> Compute lambda_shot and lambda_read From fea53acda8164dee03fdd848989d1e097f3e7ae5 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 43/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index 404c27b..e99b4fb 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -53,4 +53,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From e2eff697d219eaaead800b24d9ed0d88df3be0f2 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 44/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index e99b4fb..c937d8a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -72,4 +72,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 32c7b943bf53d2795fca01967b9bec4dc9470f8a Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 20 Apr 2022 01:36:33 -0700 Subject: [PATCH 45/59] Compute lambda_shot and lambda_read --- src/merge.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/merge.cpp b/src/merge.cpp index c937d8a..b2fe07a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -91,4 +91,23 @@ std::pair merge::getNoiseParams( int ISO, \ return std::make_pair(lambda_shot, lambda_read); } +std::pair merge::getNoiseParams( int ISO, \ + int white_level, \ + double black_level ) +{ + // Set ISO to 100 if not positive + ISO = ISO <= 0 ? 100 : ISO; + + // Calculate shot noise and read noise parameters w.r.t ISO 100 + double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; + double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; + + // Rescale shot and read noise to normal range + double lambda_shot = lambda_shot_p * (white_level - black_level); + double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); + + // return pair + return std::make_pair(lambda_shot, lambda_read); +} + } // namespace hdrplus From 9783e57eba58d3458c2229e1b0ec73b7db91b043 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 27 Apr 2022 15:45:12 -0700 Subject: [PATCH 46/59] Obtain reference tiles --- src/merge.cpp | 76 --------------------------------------------------- 1 file changed, 76 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index b2fe07a..21b8204 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -34,80 +34,4 @@ void merge::process( const hdrplus::burst& burst_images, \ // cv::imwrite("tile0.jpg", outputImg1); } -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - -std::pair merge::getNoiseParams( int ISO, \ - int white_level, \ - double black_level ) -{ - // Set ISO to 100 if not positive - ISO = ISO <= 0 ? 100 : ISO; - - // Calculate shot noise and read noise parameters w.r.t ISO 100 - double lambda_shot_p = ISO / 100.0f * baseline_lambda_shot; - double lambda_read_p = (ISO / 100.0f) * (ISO / 100.0f) * baseline_lambda_read; - - // Rescale shot and read noise to normal range - double lambda_shot = lambda_shot_p * (white_level - black_level); - double lambda_read = lambda_read_p * (white_level - black_level) * (white_level - black_level); - - // return pair - return std::make_pair(lambda_shot, lambda_read); -} - } // namespace hdrplus From be20500402a49ae1f7ee387ff2e18cd29850d094 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Thu, 28 Apr 2022 03:53:22 -0700 Subject: [PATCH 47/59] Implemented cosine window 1d and 2d --- include/hdrplus/merge.h | 30 ++++++++++++++++++++++++++++++ src/merge.cpp | 24 +++++++++++++----------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 68956ea..d3aa550 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -2,6 +2,7 @@ #include #include // all opencv header +#include #include "hdrplus/burst.h" namespace hdrplus @@ -26,6 +27,35 @@ class merge */ void process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments); + + private: + cv::Mat cosineWindow1D(cv::Mat input, int window_size = 16) { + cv::Mat output = input.clone(); + for (int i = 0; i < input.cols; ++i) { + output.at(0, i) = 1. / 2. - 1. / 2. * cos(2 * M_PI * (input.at(0, i) + 1 / 2.) / window_size); + } + return output; + } + + cv::Mat cosineWindow2D(cv::Mat tile) { + int window_size = tile.rows; // Assuming square tile + cv::Mat output_tile = tile.clone(); + + cv::Mat window = cv::Mat::zeros(1, window_size, CV_32F); + for(int i = 0; i < window_size; ++i) { + window.at(i) = i; + } + + cv::Mat window_x = cosineWindow1D(window, window_size); + window_x = cv::repeat(window_x, window_size, 1); + cv::Mat window_2d = window_x.mul(window_x.t()); + + cv::Mat window_applied; + cv::multiply(tile, window_2d, window_applied, 1, CV_32F); + return window_applied; + } + + std::vector getReferenceTiles(cv::Mat reference_image); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 21b8204..ed7602b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -14,24 +14,26 @@ void merge::process( const hdrplus::burst& burst_images, \ std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); // Obtain tiles - std::vector reference_tiles; cv::Mat reference_image = burst_images.grayscale_images_pad[0]; - std::cout << reference_image.rows << " " << reference_image.cols << std::endl; - for (int y = 0; y < reference_image.rows - 8; y += 8) { - for (int x = 0; x < reference_image.cols - 8; x += 8) { - cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); - reference_tiles.push_back(tile); - } - } - - + std::vector reference_tiles = getReferenceTiles(reference_image); // cv::Mat outputImg = reference_image.clone(); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); // cv::imwrite("tile0.jpg", outputImg1); } +std::vector merge::getReferenceTiles(cv::Mat reference_image) { + std::vector reference_tiles; + for (int y = 0; y < reference_image.rows - 8; y += 8) { + for (int x = 0; x < reference_image.cols - 8; x += 8) { + cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + reference_tiles.push_back(tile); + } + } + return reference_tiles; +} + } // namespace hdrplus From 257b2f6520995c1ae6309a88cd1705e33fb2e16c Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Fri, 29 Apr 2022 03:32:18 -0700 Subject: [PATCH 48/59] Implemented actual merging (4.4) --- include/hdrplus/merge.h | 19 +++++++++- src/merge.cpp | 78 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index d3aa550..bc098dc 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -5,12 +5,15 @@ #include #include "hdrplus/burst.h" +#define TILE_SIZE 16 + namespace hdrplus { class merge { public: + int offset = TILE_SIZE / 2; float baseline_lambda_shot = 3.24 * pow( 10, -4 ); float baseline_lambda_read = 4.3 * pow( 10, -6 ); @@ -29,7 +32,7 @@ class merge std::vector>>>& alignments); private: - cv::Mat cosineWindow1D(cv::Mat input, int window_size = 16) { + cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); for (int i = 0; i < input.cols; ++i) { output.at(0, i) = 1. / 2. - 1. / 2. * cos(2 * M_PI * (input.at(0, i) + 1 / 2.) / window_size); @@ -55,7 +58,21 @@ class merge return window_applied; } + cv::Mat cat2Dtiles(std::vector> tiles) { + std::vector rows; + for (auto row_tiles : tiles) { + cv::Mat row; + cv::hconcat(row_tiles, row); + rows.push_back(row); + } + cv::Mat img; + cv::vconcat(rows, img); + return img; + } + std::vector getReferenceTiles(cv::Mat reference_image); + + cv::Mat mergeTiles(std::vector tiles, int rows, int cols); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index ed7602b..0d20a53 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -13,27 +13,95 @@ void merge::process( const hdrplus::burst& burst_images, \ double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Obtain tiles - cv::Mat reference_image = burst_images.grayscale_images_pad[0]; + // Obtain Reference tiles + cv::Mat reference_image = burst_images.grayscale_images_pad[burst_images.reference_image_idx]; std::vector reference_tiles = getReferenceTiles(reference_image); + // Temporal Denoising + + // Spatial Denoising + + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + cv::Mat merged = mergeTiles(windowed_tiles, reference_image.rows, reference_image.cols); + // cv::Mat outputImg = reference_image.clone(); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); + // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); // cv::imwrite("tile0.jpg", outputImg1); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { std::vector reference_tiles; - for (int y = 0; y < reference_image.rows - 8; y += 8) { - for (int x = 0; x < reference_image.cols - 8; x += 8) { - cv::Mat tile = reference_image(cv::Rect(x, y, 16, 16)); + for (int y = 0; y < reference_image.rows - offset; y += offset) { + for (int x = 0; x < reference_image.cols - offset; x += offset) { + cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); reference_tiles.push_back(tile); } } return reference_tiles; } +cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols){ + // 1. get all four subsets: original (evenly split), horizontal overlapped, + // vertical overlapped, 2D overlapped + std::vector> tiles_original; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_original.push_back(row); + } + + std::vector> tiles_horizontal; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_horizontal.push_back(row); + } + + std::vector> tiles_vertical; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_vertical.push_back(row); + } + + std::vector> tiles_2d; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_2d.push_back(row); + } + + // 2. Concatenate the four subsets + cv::Mat img_original = cat2Dtiles(tiles_original); + cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); + cv::Mat img_vertical = cat2Dtiles(tiles_vertical); + cv::Mat img_2d = cat2Dtiles(tiles_2d); + + // 3. Add the four subsets together + img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; + img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; + img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; + + return img_original; +} + } // namespace hdrplus From 7477ea13cb442d1b8827b9131e2f75eb05b1f1ce Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 01:08:39 -0700 Subject: [PATCH 49/59] Accessing channel data signatures --- include/hdrplus/merge.h | 4 +++ src/merge.cpp | 63 +++++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index bc098dc..ffba5d3 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -73,6 +73,10 @@ class merge std::vector getReferenceTiles(cv::Mat reference_image); cv::Mat mergeTiles(std::vector tiles, int rows, int cols); + + cv::Mat processChannel( const hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 0d20a53..6ea1707 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -13,31 +13,16 @@ void merge::process( const hdrplus::burst& burst_images, \ double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Obtain Reference tiles - cv::Mat reference_image = burst_images.grayscale_images_pad[burst_images.reference_image_idx]; - std::vector reference_tiles = getReferenceTiles(reference_image); - - // Temporal Denoising - - // Spatial Denoising - - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); - } - - // Merge tiles - cv::Mat merged = mergeTiles(windowed_tiles, reference_image.rows, reference_image.cols); - - // cv::Mat outputImg = reference_image.clone(); + // Call merge on each channel + cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; + reference_image.convertTo(reference_image, CV_32F); + + // Get Channels + // cv::Mat channel_0(reference_image.rows, reference_image.cols, CV_32F, (uchar*)reference_image.data, 2 * sizeof(float)); // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); - // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); - // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); - // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); - // cv::cvtColor(outputImg1, outputImg1, cv::COLOR_GRAY2RGB); - // cv::imwrite("tile0.jpg", outputImg1); + + //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_0); + } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -104,4 +89,34 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols return img_original; } +cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image) { + + std::vector reference_tiles = getReferenceTiles(channel_image); + + // Temporal Denoising + + // Spatial Denoising + + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + cv::Mat merged = mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); + + // cv::Mat outputImg = channel_image.clone(); + // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // cv::imwrite("ref.jpg", outputImg); + // cv::Mat outputImg1 = reference_tiles[0].clone(); + // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); + // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); + // cv::cvtColor(merged, merged, cv::COLOR_GRAY2RGB); + // cv::imwrite("tile0.jpg", merged); + return merged; +} + } // namespace hdrplus From 0ff5656c439e12114e6aea7bf432492f7978007d Mon Sep 17 00:00:00 2001 From: cvachha Date: Fri, 29 Apr 2022 22:29:40 -0700 Subject: [PATCH 50/59] Added a few helper function signatures Added a few helper function signatures which are commented out in merge.h --- include/hdrplus/merge.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index ffba5d3..f208c70 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -30,7 +30,19 @@ class merge */ void process( const hdrplus::burst& burst_images, \ std::vector>>>& alignments); - + + + /* + std::vector get_other_tiles(); //return the other tile list T_1 to T_n + + std::vector vector_math(string operation, reference_tile, other_tile_list); //for loop allowing operations across single element and list + + std::vector scalar_vector_math(string operation, scalar num, std::vector tile_list); //for loop allowing operations across single element and list + + std::vector average_vector(std::vector tile_list); //take average of vector elements + + */ + private: cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); From 838a74765ec0ca603111b14599a703de2a0ce62e Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 02:41:56 -0700 Subject: [PATCH 51/59] Finished merging channels (4.4) --- include/hdrplus/burst.h | 3 ++ include/hdrplus/merge.h | 4 +- src/merge.cpp | 91 +++++++++++++++++++++++++++++++---------- 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/include/hdrplus/burst.h b/include/hdrplus/burst.h index def6b78..c4c9dda 100644 --- a/include/hdrplus/burst.h +++ b/include/hdrplus/burst.h @@ -33,6 +33,9 @@ class burst // number of image (including reference) in burst int num_images; + + // Bayer image after merging, stored as cv::Mat + cv::Mat merged_bayer_image; }; } // namespace hdrplus diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f208c70..c7c5642 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -28,7 +28,7 @@ class merge * Outer most vector is per alternative image. * Inner most two vector is for horizontal & vertical tiles */ - void process( const hdrplus::burst& burst_images, \ + void process( hdrplus::burst& burst_images, \ std::vector>>>& alignments); @@ -86,7 +86,7 @@ class merge cv::Mat mergeTiles(std::vector tiles, int rows, int cols); - cv::Mat processChannel( const hdrplus::burst& burst_images, \ + cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image); }; diff --git a/src/merge.cpp b/src/merge.cpp index 6ea1707..4ed50e9 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,22 +7,81 @@ namespace hdrplus { -void merge::process( const hdrplus::burst& burst_images, \ +void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Call merge on each channel + // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - reference_image.convertTo(reference_image, CV_32F); + // cv::imwrite("ref.jpg", reference_image); - // Get Channels - // cv::Mat channel_0(reference_image.rows, reference_image.cols, CV_32F, (uchar*)reference_image.data, 2 * sizeof(float)); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // Get raw channels + std::vector channels[4]; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(reference_image.at(y, x)); + } else { + channels[1].push_back(reference_image.at(y, x)); + } + } else { + if (x % 2 == 0) { + channels[2].push_back(reference_image.at(y, x)); + } else { + channels[3].push_back(reference_image.at(y, x)); + } + } + } + } + + // For each channel, perform denoising and merge + for (int i = 0; i < 4; ++i) { + // Get channel mat + cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + + // Apply merging on the channel + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); - //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_0); + // Put channel raw data back to channels + channels[i] = merged_channel.reshape(1, merged_channel.total()); + } + + // Write all channels back to a bayer mat + std::vector merged_raw; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + merged_raw.push_back(channels[0][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[1][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } else { + if (x % 2 == 0) { + merged_raw.push_back(channels[2][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[3][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } + } + } + // Create merged mat + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); + // cv::imwrite("merged.jpg", merged); + + // Remove padding + std::vector padding = burst_images.padding_info_bayer; + cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); + cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); + burst_images.merged_bayer_image = merged(vertical, horizontal); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -89,15 +148,15 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols return img_original; } -cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ +cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image) { std::vector reference_tiles = getReferenceTiles(channel_image); - // Temporal Denoising + // TODO: Temporal Denoising - // Spatial Denoising + // TODO: Spatial Denoising // Process tiles through 2D cosine window std::vector windowed_tiles; @@ -106,17 +165,7 @@ cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ } // Merge tiles - cv::Mat merged = mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - - // cv::Mat outputImg = channel_image.clone(); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); - // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); - // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); - // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); - // cv::cvtColor(merged, merged, cv::COLOR_GRAY2RGB); - // cv::imwrite("tile0.jpg", merged); - return merged; + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } } // namespace hdrplus From 135449d772f89fc1ea16f17f46b273e746d2bd7d Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 03:51:40 -0700 Subject: [PATCH 52/59] Finished noise variance (4.1) and added FFT/IFFT --- include/hdrplus/merge.h | 18 +++++++++++++++++- src/merge.cpp | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index c7c5642..8a829f5 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -44,6 +44,20 @@ class merge */ private: + float tileRMS(cv::Mat tile) { + cv::Mat squared; + cv::multiply(tile, tile, squared); + return sqrt(cv::mean(squared)[0]); + } + + std::vector getNoiseVariance(std::vector tiles, float lambda_shot, float lambda_read) { + std::vector noise_variance; + for (auto tile : tiles) { + noise_variance.push_back(lambda_shot * tileRMS(tile) + lambda_read); + } + return noise_variance; + } + cv::Mat cosineWindow1D(cv::Mat input, int window_size = TILE_SIZE) { cv::Mat output = input.clone(); for (int i = 0; i < input.cols; ++i) { @@ -88,7 +102,9 @@ class merge cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image); + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read); }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 4ed50e9..e49cec4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -10,9 +10,12 @@ namespace hdrplus void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { + // 4.1 Noise Parameters and RMS + // Noise parameters calculated from baseline ISO noise parameters double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + // 4.2-4.4 Denoising and Merging // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; // cv::imwrite("ref.jpg", reference_image); @@ -45,7 +48,7 @@ void merge::process( hdrplus::burst& burst_images, \ // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); // Apply merging on the channel - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels @@ -150,14 +153,39 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ - cv::Mat channel_image) { - + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read) { + // Get tiles of the reference image std::vector reference_tiles = getReferenceTiles(channel_image); - // TODO: Temporal Denoising + // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) + std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); + + // Apply FFT on reference tiles (spatial to frequency) + std::vector reference_tiles_DFT; + for (auto ref_tile : reference_tiles) { + cv::Mat ref_tile_DFT; + ref_tile.convertTo(ref_tile_DFT, CV_32F); + cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE|cv::DFT_COMPLEX_OUTPUT); + reference_tiles_DFT.push_back(ref_tile_DFT); + } + + // TODO: 4.2 Temporal Denoising + + // TODO: 4.3 Spatial Denoising - // TODO: Spatial Denoising + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE|cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; for (auto tile : reference_tiles) { From f8ea17b1d20b7dd2f4a7882e785fa240373624b6 Mon Sep 17 00:00:00 2001 From: cvachha Date: Sun, 1 May 2022 06:18:36 -0700 Subject: [PATCH 53/59] Added code in progress for temporal and spatial denoising I added some of my code for temporal and spatial denoising, but I commented it out since it still is incomplete since I need to find a way to allow for multiple image channels. I created a helper function which we could use to return the channels. I also edited the merge.h file to include the signature of the helper function. --- include/hdrplus/merge.h | 3 + src/merge.cpp | 421 +++++++++++++++++++++++++--------------- 2 files changed, 267 insertions(+), 157 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 8a829f5..261846b 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -105,6 +105,9 @@ class merge cv::Mat channel_image, \ float lambda_shot, \ float lambda_read); + + std::vector getChannels(cv::Mat input_image); + }; } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index e49cec4..09b154a 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,193 +7,300 @@ namespace hdrplus { -void merge::process( hdrplus::burst& burst_images, \ - std::vector>>>& alignments) -{ - // 4.1 Noise Parameters and RMS - // Noise parameters calculated from baseline ISO noise parameters - double lambda_shot, lambda_read; - std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - - // 4.2-4.4 Denoising and Merging - // Get padded bayer image - cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - // cv::imwrite("ref.jpg", reference_image); - - // Get raw channels - std::vector channels[4]; - - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(reference_image.at(y, x)); - } else { - channels[1].push_back(reference_image.at(y, x)); + void merge::process(hdrplus::burst& burst_images, \ + std::vector>>>& alignments) + { + // 4.1 Noise Parameters and RMS + // Noise parameters calculated from baseline ISO noise parameters + double lambda_shot, lambda_read; + std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); + + // 4.2-4.4 Denoising and Merging + + // Get padded bayer image + cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; + // cv::imwrite("ref.jpg", reference_image); + + // Get raw channels + std::vector channels[4]; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(reference_image.at(y, x)); + } + else { + channels[1].push_back(reference_image.at(y, x)); + } } - } else { - if (x % 2 == 0) { - channels[2].push_back(reference_image.at(y, x)); - } else { - channels[3].push_back(reference_image.at(y, x)); + else { + if (x % 2 == 0) { + channels[2].push_back(reference_image.at(y, x)); + } + else { + channels[3].push_back(reference_image.at(y, x)); + } } } } - } - // For each channel, perform denoising and merge - for (int i = 0; i < 4; ++i) { - // Get channel mat - cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); - // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + // For each channel, perform denoising and merge + for (int i = 0; i < 4; ++i) { + // Get channel mat + cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); - // Apply merging on the channel - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); - // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); + // Apply merging on the channel + + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images - // Put channel raw data back to channels - channels[i] = merged_channel.reshape(1, merged_channel.total()); - } + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); - // Write all channels back to a bayer mat - std::vector merged_raw; + // Put channel raw data back to channels + channels[i] = merged_channel.reshape(1, merged_channel.total()); + } - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - merged_raw.push_back(channels[0][(y/2)*(reference_image.cols/2) + (x/2)]); - } else { - merged_raw.push_back(channels[1][(y/2)*(reference_image.cols/2) + (x/2)]); + // Write all channels back to a bayer mat + std::vector merged_raw; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + merged_raw.push_back(channels[0][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } + else { + merged_raw.push_back(channels[1][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } } - } else { - if (x % 2 == 0) { - merged_raw.push_back(channels[2][(y/2)*(reference_image.cols/2) + (x/2)]); - } else { - merged_raw.push_back(channels[3][(y/2)*(reference_image.cols/2) + (x/2)]); + else { + if (x % 2 == 0) { + merged_raw.push_back(channels[2][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } + else { + merged_raw.push_back(channels[3][(y / 2) * (reference_image.cols / 2) + (x / 2)]); + } } } } + + // Create merged mat + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); + // cv::imwrite("merged.jpg", merged); + + // Remove padding + std::vector padding = burst_images.padding_info_bayer; + cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); + cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); + burst_images.merged_bayer_image = merged(vertical, horizontal); } - - // Create merged mat - cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); - // cv::imwrite("merged.jpg", merged); - - // Remove padding - std::vector padding = burst_images.padding_info_bayer; - cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); - cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); - burst_images.merged_bayer_image = merged(vertical, horizontal); -} - -std::vector merge::getReferenceTiles(cv::Mat reference_image) { - std::vector reference_tiles; - for (int y = 0; y < reference_image.rows - offset; y += offset) { - for (int x = 0; x < reference_image.cols - offset; x += offset) { - cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); - reference_tiles.push_back(tile); + + std::vector merge::getReferenceTiles(cv::Mat reference_image) { + std::vector reference_tiles; + for (int y = 0; y < reference_image.rows - offset; y += offset) { + for (int x = 0; x < reference_image.cols - offset; x += offset) { + cv::Mat tile = reference_image(cv::Rect(x, y, TILE_SIZE, TILE_SIZE)); + reference_tiles.push_back(tile); + } } + return reference_tiles; } - return reference_tiles; -} - -cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols){ - // 1. get all four subsets: original (evenly split), horizontal overlapped, - // vertical overlapped, 2D overlapped - std::vector> tiles_original; - for (int y = 0; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 0; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + + cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols) { + // 1. get all four subsets: original (evenly split), horizontal overlapped, + // vertical overlapped, 2D overlapped + std::vector> tiles_original; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_original.push_back(row); } - tiles_original.push_back(row); - } - std::vector> tiles_horizontal; - for (int y = 0; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 1; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_horizontal; + for (int y = 0; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_horizontal.push_back(row); } - tiles_horizontal.push_back(row); - } - std::vector> tiles_vertical; - for (int y = 1; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 0; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_vertical; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 0; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_vertical.push_back(row); } - tiles_vertical.push_back(row); - } - std::vector> tiles_2d; - for (int y = 1; y < num_rows / offset - 1; y += 2) { - std::vector row; - for (int x = 1; x < num_cols / offset - 1; x += 2) { - row.push_back(tiles[y * (num_cols / offset - 1) + x]); + std::vector> tiles_2d; + for (int y = 1; y < num_rows / offset - 1; y += 2) { + std::vector row; + for (int x = 1; x < num_cols / offset - 1; x += 2) { + row.push_back(tiles[y * (num_cols / offset - 1) + x]); + } + tiles_2d.push_back(row); } - tiles_2d.push_back(row); - } - // 2. Concatenate the four subsets - cv::Mat img_original = cat2Dtiles(tiles_original); - cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); - cv::Mat img_vertical = cat2Dtiles(tiles_vertical); - cv::Mat img_2d = cat2Dtiles(tiles_2d); - - // 3. Add the four subsets together - img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; - img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; - img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; - - return img_original; -} - -cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ - std::vector>>>& alignments, \ - cv::Mat channel_image, \ - float lambda_shot, \ - float lambda_read) { - // Get tiles of the reference image - std::vector reference_tiles = getReferenceTiles(channel_image); - - // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) - std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); - - // Apply FFT on reference tiles (spatial to frequency) - std::vector reference_tiles_DFT; - for (auto ref_tile : reference_tiles) { - cv::Mat ref_tile_DFT; - ref_tile.convertTo(ref_tile_DFT, CV_32F); - cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE|cv::DFT_COMPLEX_OUTPUT); - reference_tiles_DFT.push_back(ref_tile_DFT); + // 2. Concatenate the four subsets + cv::Mat img_original = cat2Dtiles(tiles_original); + cv::Mat img_horizontal = cat2Dtiles(tiles_horizontal); + cv::Mat img_vertical = cat2Dtiles(tiles_vertical); + cv::Mat img_2d = cat2Dtiles(tiles_2d); + + // 3. Add the four subsets together + img_original(cv::Rect(offset, 0, num_cols - TILE_SIZE, num_rows)) += img_horizontal; + img_original(cv::Rect(0, offset, num_cols, num_rows - TILE_SIZE)) += img_vertical; + img_original(cv::Rect(offset, offset, num_cols - TILE_SIZE, num_rows - TILE_SIZE)) += img_2d; + + return img_original; } - // TODO: 4.2 Temporal Denoising + cv::Mat merge::processChannel(hdrplus::burst& burst_images, \ + std::vector>>>& alignments, \ + cv::Mat channel_image, \ + float lambda_shot, \ + float lambda_read) { + // Get tiles of the reference image + std::vector reference_tiles = getReferenceTiles(channel_image); + + // Get noise variance (sigma**2 = lambda_shot * tileRMS + lambda_read) + std::vector noise_variance = getNoiseVariance(reference_tiles, lambda_shot, lambda_read); + + // Apply FFT on reference tiles (spatial to frequency) + std::vector reference_tiles_DFT; + for (auto ref_tile : reference_tiles) { + cv::Mat ref_tile_DFT; + ref_tile.convertTo(ref_tile_DFT, CV_32F); + cv::dft(ref_tile_DFT, ref_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + reference_tiles_DFT.push_back(ref_tile_DFT); + } + + // TODO: 4.2 Temporal Denoising + + //goal: temporially denoise using the weiner filter + //input: + //1. array of 2D dft tiles of the reference image + //2. array of 2D dft tiles ocf the aligned alternate image + //3. estimated noise varaince + //4. temporal factor + //return: merged image patches dft + + /* + + //tile_size = TILE_SIZE; + double temporal_factor = 8 //8 by default + + double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; - // TODO: 4.3 Spatial Denoising + //start calculating the merged image tiles fft + + for (int i = 0;i < burst_images.num_images; i++) { + if (i != burst_images.reference_image_idx) { - // Apply IFFT on reference tiles (frequency to spatial) - std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { - cv::Mat denoised_tile; - cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE|cv::DFT_REAL_OUTPUT); - denoised_tile.convertTo(denoised_tile, CV_16U); - denoised_tiles.push_back(denoised_tile); + } + } + //sample of 0th image + altername_image = burst_images.bayer_images_pad[0] + + //get the tiles of the alternate image + std::vector alternate_image_tiles = getReferenceTiles(channel_image); + + //get the dft of the alternate image + std::vector alternate_tiles_DFT; + for (auto alt_tile : alternate_tiles_DFT) { + cv::Mat alt_tile_DFT; + alt_tile.convertTo(alt_tile_DFT, CV_32F); + cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + alternate_tiles_DFT.push_back(alt_tile_DFT); + } + + + + std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT; + std::vector tile_sq_asolute_diff = tile_differences; //tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + + std::vector merged_image_tiles_fft = alternate_tiles_DFT + A * tile_differences; + + //use merged_image_tiles_fft into part 4.3 + + + + // TODO: 4.3 Spatial Denoising + + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + + //adding after here + double spatial_factor = 1; //to be added + double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; + + //calculate the spatial denoising + spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; + std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; + + merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) + + //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + + */ + + + // 4.4 Cosine Window Merging + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } - reference_tiles = denoised_tiles; - // 4.4 Cosine Window Merging - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); + + //Helper function to get the channels from the input image + std::vector getChannels(cv::Mat input_image){ + std::vector channels[4]; + + for (int y = 0; y < input_image.rows; ++y) { + for (int x = 0; x < input_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(input_image.at(y, x)); + } + else { + channels[1].push_back(input_image.at(y, x)); + } + } + else { + if (x % 2 == 0) { + channels[2].push_back(input_image.at(y, x)); + } + else { + channels[3].push_back(input_image.at(y, x)); + } + } + } + } + return channels; + } - // Merge tiles - return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); -} + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments -} // namespace hdrplus +} // namespace hdrplus \ No newline at end of file From f964ed18094991743e2318ff9538becea3bdf172 Mon Sep 17 00:00:00 2001 From: cvachha Date: Mon, 2 May 2022 03:01:31 -0700 Subject: [PATCH 54/59] Added a loop to get the channels and tiles for each alternate image Added some loops to get the channels for the alternate images. Then this list of channels from each alternate image gets sent to the processChannel function. Over there, there is a loop to create a list of the DFTs of each of the alternate images. I also modified the function signature of processChannel and commented out the code additions in processChannel since some of it is not complete yet. --- include/hdrplus/merge.h | 1 + src/merge.cpp | 69 ++++++++++++++++++++++++++++------------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 261846b..23c5d55 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -103,6 +103,7 @@ class merge cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image, \ + std::vector alternate_channel_i_list,\ float lambda_shot, \ float lambda_read); diff --git a/src/merge.cpp b/src/merge.cpp index 09b154a..8e0fdc5 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -45,6 +45,7 @@ namespace hdrplus } } + ///// // For each channel, perform denoising and merge for (int i = 0; i < 4; ++i) { // Get channel mat @@ -56,7 +57,25 @@ namespace hdrplus //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images - cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + //create list of channel_i of alternate images: + std::vector alternate_channel_i_list; + for (int j = 0; j < burst_images.num_images; j++) { + if (j != burst_images.reference_image_idx) { + + //get alternate image + cv::Mat alt_image = burst_images.bayer_images_pad[j]; + std::vector alt_img_channel = getChannels(alt_image); //get channel array from alternate image + cv::Mat alt_channel_i(alt_image.rows / 2, alt_image.cols / 2, CV_16U, alt_img_channel[i].data()); + + alternate_channel_i_list.push_back(alt_channel_i) + } + } + + ///// + + //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, alternate_channel_i_list, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels @@ -165,6 +184,7 @@ namespace hdrplus cv::Mat merge::processChannel(hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image, \ + std::vector alternate_channel_i_list,\ float lambda_shot, \ float lambda_read) { // Get tiles of the reference image @@ -192,43 +212,50 @@ namespace hdrplus //4. temporal factor //return: merged image patches dft - /* + //tile_size = TILE_SIZE; - double temporal_factor = 8 //8 by default + /* + double temporal_factor = 8.0 //8 by default double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; //start calculating the merged image tiles fft - for (int i = 0;i < burst_images.num_images; i++) { - if (i != burst_images.reference_image_idx) { + + //get the tiles of the alternate image as a list + + std::vector> alternate_channel_i_tile_list; //list of alt channel tiles + std::vector> alternate_tiles_DFT_list; //list of alt channel tiles + for (auto alt_img_channel : alternate_channel_i_list) { + std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image + alternate_channel_i_tile_list.push_back(alt_img_channel_tile) + + std::vector alternate_tiles_DFT_list; + for (auto alt_tile : alt_img_channel_tile) { + cv::Mat alt_tile_DFT; + alt_tile.convertTo(alt_tile_DFT, CV_32F); + cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + alternate_tiles_DFT_list.push_back(alt_tile_DFT); } + alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); } - //sample of 0th image - altername_image = burst_images.bayer_images_pad[0] - - //get the tiles of the alternate image - std::vector alternate_image_tiles = getReferenceTiles(channel_image); //get the dft of the alternate image - std::vector alternate_tiles_DFT; - for (auto alt_tile : alternate_tiles_DFT) { - cv::Mat alt_tile_DFT; - alt_tile.convertTo(alt_tile_DFT, CV_32F); - cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); - alternate_tiles_DFT.push_back(alt_tile_DFT); - } + //std::vector alternate_tiles_DFT; + - std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT; - std::vector tile_sq_asolute_diff = tile_differences; //tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; + std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + //find the squared absolute difference across all the tiles std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) - std::vector merged_image_tiles_fft = alternate_tiles_DFT + A * tile_differences; + std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; //use merged_image_tiles_fft into part 4.3 @@ -258,9 +285,9 @@ namespace hdrplus //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + */ - // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; From b4b96199b24f92aeeb08dc887def4263190fcaf5 Mon Sep 17 00:00:00 2001 From: cvachha Date: Tue, 3 May 2022 21:57:36 -0700 Subject: [PATCH 55/59] Progress on denoising and separated functions I put the temporal and spatial denoising into 2 separate functions and am implementing finding the absolute difference. I have also edited the merge.h file to add the function signatures. --- include/hdrplus/merge.h | 7 +- src/merge.cpp | 170 +++++++++++++++++++++++++--------------- 2 files changed, 112 insertions(+), 65 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 23c5d55..981937c 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -107,7 +107,12 @@ class merge float lambda_shot, \ float lambda_read); - std::vector getChannels(cv::Mat input_image); + std::vector getChannels(cv::Mat input_image); //helper function + + //temporal denoise + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + }; diff --git a/src/merge.cpp b/src/merge.cpp index 8e0fdc5..619d937 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -203,7 +203,76 @@ namespace hdrplus } // TODO: 4.2 Temporal Denoising + std::vector temporal_denoised_tiles = temporal_denoise(reference_tiles, reference_tiles_DFT, noise_varaince) + + + // TODO: 4.3 Spatial Denoising + + ////adding after here + + std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image + //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + //// + + // Apply IFFT on reference tiles (frequency to spatial) + std::vector denoised_tiles; + for (auto dft_tile : reference_tiles_DFT) { + cv::Mat denoised_tile; + cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); + denoised_tile.convertTo(denoised_tile, CV_16U); + denoised_tiles.push_back(denoised_tile); + } + reference_tiles = denoised_tiles; + + + + + + // 4.4 Cosine Window Merging + // Process tiles through 2D cosine window + std::vector windowed_tiles; + for (auto tile : reference_tiles) { + windowed_tiles.push_back(cosineWindow2D(tile)); + } + + // Merge tiles + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); + } + + + //Helper function to get the channels from the input image + std::vector getChannels(cv::Mat input_image){ + std::vector channels[4]; + + for (int y = 0; y < input_image.rows; ++y) { + for (int x = 0; x < input_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(input_image.at(y, x)); + } + else { + channels[1].push_back(input_image.at(y, x)); + } + } + else { + if (x % 2 == 0) { + channels[2].push_back(input_image.at(y, x)); + } + else { + channels[3].push_back(input_image.at(y, x)); + } + } + } + } + return channels; + + } + + //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { //goal: temporially denoise using the weiner filter //input: //1. array of 2D dft tiles of the reference image @@ -215,7 +284,7 @@ namespace hdrplus //tile_size = TILE_SIZE; - /* + double temporal_factor = 8.0 //8 by default double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; @@ -244,36 +313,53 @@ namespace hdrplus //get the dft of the alternate image //std::vector alternate_tiles_DFT; - - std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - //find the squared absolute difference across all the tiles + //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + //find reference_tiles_DFT - alternate_tiles_DFT_list + std::vector> tile_difference_list; //list of tile differences + for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { + std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; + tile_difference_list.push_back(single_tile_difference); + } - std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; - //use merged_image_tiles_fft into part 4.3 + // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + + std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + //get the real and imaginary components + /* + std::vector> absolute_difference_list; + for (auto individual_difference : tile_difference_list) { + for (int i =0; i < individual_difference.rows; i++ ) { + std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); + for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { + row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag + } + } + //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag + absolute_difference_list.push_back(single_tile_difference); + } + */ - // TODO: 4.3 Spatial Denoising + //find the squared absolute difference across all the tiles - // Apply IFFT on reference tiles (frequency to spatial) - std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { - cv::Mat denoised_tile; - cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); - denoised_tile.convertTo(denoised_tile, CV_16U); - denoised_tiles.push_back(denoised_tile); - } - reference_tiles = denoised_tiles; - //adding after here + std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + + std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; + + return merged_image_tiles_fft + + } + + std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + double spatial_factor = 1; //to be added double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; @@ -282,52 +368,8 @@ namespace hdrplus std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) - - //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - - */ - - // 4.4 Cosine Window Merging - // Process tiles through 2D cosine window - std::vector windowed_tiles; - for (auto tile : reference_tiles) { - windowed_tiles.push_back(cosineWindow2D(tile)); - } - - // Merge tiles - return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - } - - - //Helper function to get the channels from the input image - std::vector getChannels(cv::Mat input_image){ - std::vector channels[4]; - - for (int y = 0; y < input_image.rows; ++y) { - for (int x = 0; x < input_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(input_image.at(y, x)); - } - else { - channels[1].push_back(input_image.at(y, x)); - } - } - else { - if (x % 2 == 0) { - channels[2].push_back(input_image.at(y, x)); - } - else { - channels[3].push_back(input_image.at(y, x)); - } - } - } - } - return channels; - } - - //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments + } // namespace hdrplus \ No newline at end of file From f1da015285f6a5bbb951c8d6c1688000ab0dd6ff Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 4 May 2022 18:27:02 -0700 Subject: [PATCH 56/59] Change splitting/merging channels --- include/hdrplus/merge.h | 2 - include/hdrplus/utility.h | 55 ++++++++ src/merge.cpp | 268 +++++++++++++++----------------------- 3 files changed, 162 insertions(+), 163 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 981937c..827fa2f 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -106,8 +106,6 @@ class merge std::vector alternate_channel_i_list,\ float lambda_shot, \ float lambda_read); - - std::vector getChannels(cv::Mat input_image); //helper function //temporal denoise std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); diff --git a/include/hdrplus/utility.h b/include/hdrplus/utility.h index ee87631..9034608 100644 --- a/include/hdrplus/utility.h +++ b/include/hdrplus/utility.h @@ -127,4 +127,59 @@ cv::Mat downsample_nearest_neighbour( cv::Mat src_image ) return dst_image; } +/** + * @brief Extract RGB channel seprately from bayer image + * + * @tparam T data tyoe of bayer image. + * @return vector of RGB image. OpenCV internally maintain reference count. + * Thus this step won't create deep copy overhead. + * + * @example extract_rgb_fmom_bayer( bayer_img, rgb_vector_container ); + */ +template +void extract_rgb_fmom_bayer( const cv::Mat& bayer_img, \ + cv::Mat& red_img, cv::Mat& green_img1, cv::Mat& green_img2, cv::Mat& blue_img ) +{ + const T* bayer_img_ptr = (const T*)bayer_img.data; + int bayer_width = bayer_img.size().width; + int bayer_height = bayer_img.size().height; + int bayer_step = bayer_img.step1(); + + if ( bayer_width % 2 != 0 || bayer_height % 2 != 0 ) + { + throw std::runtime_error("Bayer image data size incorrect, must be multiplier of 2\n"); + } + + // RGB image is half the size of bayer image + int rgb_width = bayer_width / 2; + int rgb_height = bayer_height / 2; + red_img.create( rgb_height, rgb_width, bayer_img.type() ); + green_img1.create( rgb_height, rgb_width, bayer_img.type() ); + green_img2.create( rgb_height, rgb_width, bayer_img.type() ); + blue_img.create( rgb_height, rgb_width, bayer_img.type() ); + int rgb_step = red_img.step1(); + + T* r_img_ptr = (T*)red_img.data; + T* g1_img_ptr = (T*)green_img1.data; + T* g2_img_ptr = (T*)green_img2.data; + T* b_img_ptr = (T*)blue_img.data; + + for ( int rgb_row_i = 0; rgb_row_i < rgb_height; rgb_row_i++ ) + { + int rgb_row_i_offset = rgb_row_i * rgb_step; + + // Every RGB row corresbonding to two Bayer image row + int bayer_row_i_offset1 = ( rgb_row_i * 2 + 0 ) * bayer_step; // For RG + int bayer_row_i_offset2 = ( rgb_row_i * 2 + 1 ) * bayer_step; // For GB + + for ( int rgb_col_j = 0; rgb_col_j < rgb_width; rgb_col_j++ ) + { + r_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + ( rgb_col_j * 2 + 0 ) ]; + g1_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + ( rgb_col_j * 2 + 1 ) ]; + g2_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset2 + ( rgb_col_j * 2 + 0 ) ]; + b_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset2 + ( rgb_col_j * 2 + 1 ) ]; + } + } +} + } // namespace hdrplus diff --git a/src/merge.cpp b/src/merge.cpp index 619d937..d855f4b 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -3,6 +3,7 @@ #include #include "hdrplus/merge.h" #include "hdrplus/burst.h" +#include "hdrplus/utility.h" namespace hdrplus { @@ -19,40 +20,17 @@ namespace hdrplus // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - // cv::imwrite("ref.jpg", reference_image); + cv::imwrite("ref.jpg", reference_image); // Get raw channels - std::vector channels[4]; - - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(reference_image.at(y, x)); - } - else { - channels[1].push_back(reference_image.at(y, x)); - } - } - else { - if (x % 2 == 0) { - channels[2].push_back(reference_image.at(y, x)); - } - else { - channels[3].push_back(reference_image.at(y, x)); - } - } - } - } + std::vector channels(4, cv::Mat::zeros(reference_image.rows / 2, reference_image.cols / 2, CV_16U)); + hdrplus::extract_rgb_fmom_bayer(reference_image, channels[0], channels[1], channels[2], channels[3]); - ///// // For each channel, perform denoising and merge for (int i = 0; i < 4; ++i) { // Get channel mat - cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + cv::Mat channel_i = channels[i]; // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); - - // Apply merging on the channel //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images @@ -64,52 +42,56 @@ namespace hdrplus //get alternate image cv::Mat alt_image = burst_images.bayer_images_pad[j]; - std::vector alt_img_channel = getChannels(alt_image); //get channel array from alternate image - cv::Mat alt_channel_i(alt_image.rows / 2, alt_image.cols / 2, CV_16U, alt_img_channel[i].data()); + std::vector alt_channels(4, cv::Mat::zeros(reference_image.rows / 2, reference_image.cols / 2, CV_16U)); + hdrplus::extract_rgb_fmom_bayer(alt_image, alt_channels[0], alt_channels[1], alt_channels[2], alt_channels[3]); - alternate_channel_i_list.push_back(alt_channel_i) + alternate_channel_i_list.push_back(alt_channels[i]); } } - - ///// - - //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, lambda_shot, lambda_read); + // Apply merging on the channel cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, alternate_channel_i_list, lambda_shot, lambda_read); // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels - channels[i] = merged_channel.reshape(1, merged_channel.total()); + merged_channel.convertTo(channels[i], CV_16U); } // Write all channels back to a bayer mat - std::vector merged_raw; - - for (int y = 0; y < reference_image.rows; ++y) { - for (int x = 0; x < reference_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - merged_raw.push_back(channels[0][(y / 2) * (reference_image.cols / 2) + (x / 2)]); - } - else { - merged_raw.push_back(channels[1][(y / 2) * (reference_image.cols / 2) + (x / 2)]); - } + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U); + int x, y; + for (y = 0; y < reference_image.rows; ++y){ + uint16_t* row = merged.ptr(y); + if (y % 2 == 0){ + uint16_t* i0 = channels[0].ptr(y / 2); + uint16_t* i1 = channels[1].ptr(y / 2); + + for (x = 0; x < reference_image.cols;){ + //R + row[x] = i0[x / 2]; + x++; + + //G1 + row[x] = i1[x / 2]; + x++; } - else { - if (x % 2 == 0) { - merged_raw.push_back(channels[2][(y / 2) * (reference_image.cols / 2) + (x / 2)]); - } - else { - merged_raw.push_back(channels[3][(y / 2) * (reference_image.cols / 2) + (x / 2)]); - } + } + else { + uint16_t* i2 = channels[2].ptr(y / 2); + uint16_t* i3 = channels[3].ptr(y / 2); + + for(x = 0; x < reference_image.cols;){ + //G2 + row[x] = i2[x / 2]; + x++; + + //B + row[x] = i3[x / 2]; + x++; } } } - // Create merged mat - cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); - // cv::imwrite("merged.jpg", merged); - // Remove padding std::vector padding = burst_images.padding_info_bayer; cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); @@ -203,14 +185,14 @@ namespace hdrplus } // TODO: 4.2 Temporal Denoising - std::vector temporal_denoised_tiles = temporal_denoise(reference_tiles, reference_tiles_DFT, noise_varaince) + //std::vector temporal_denoised_tiles = temporal_denoise(reference_tiles, reference_tiles_DFT, noise_varaince) // TODO: 4.3 Spatial Denoising ////adding after here - std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + //std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised //// @@ -225,10 +207,6 @@ namespace hdrplus } reference_tiles = denoised_tiles; - - - - // 4.4 Cosine Window Merging // Process tiles through 2D cosine window std::vector windowed_tiles; @@ -239,137 +217,105 @@ namespace hdrplus // Merge tiles return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } - - - //Helper function to get the channels from the input image - std::vector getChannels(cv::Mat input_image){ - std::vector channels[4]; - - for (int y = 0; y < input_image.rows; ++y) { - for (int x = 0; x < input_image.cols; ++x) { - if (y % 2 == 0) { - if (x % 2 == 0) { - channels[0].push_back(input_image.at(y, x)); - } - else { - channels[1].push_back(input_image.at(y, x)); - } - } - else { - if (x % 2 == 0) { - channels[2].push_back(input_image.at(y, x)); - } - else { - channels[3].push_back(input_image.at(y, x)); - } - } - } - } - return channels; - - } - - //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments - - std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { - //goal: temporially denoise using the weiner filter - //input: - //1. array of 2D dft tiles of the reference image - //2. array of 2D dft tiles ocf the aligned alternate image - //3. estimated noise varaince - //4. temporal factor - //return: merged image patches dft + // std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + // //goal: temporially denoise using the weiner filter + // //input: + // //1. array of 2D dft tiles of the reference image + // //2. array of 2D dft tiles ocf the aligned alternate image + // //3. estimated noise varaince + // //4. temporal factor + // //return: merged image patches dft - //tile_size = TILE_SIZE; + // //tile_size = TILE_SIZE; - double temporal_factor = 8.0 //8 by default + // double temporal_factor = 8.0 //8 by default - double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; + // double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; - //start calculating the merged image tiles fft + // //start calculating the merged image tiles fft - //get the tiles of the alternate image as a list + // //get the tiles of the alternate image as a list - std::vector> alternate_channel_i_tile_list; //list of alt channel tiles - std::vector> alternate_tiles_DFT_list; //list of alt channel tiles + // std::vector> alternate_channel_i_tile_list; //list of alt channel tiles + // std::vector> alternate_tiles_DFT_list; //list of alt channel tiles - for (auto alt_img_channel : alternate_channel_i_list) { - std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image - alternate_channel_i_tile_list.push_back(alt_img_channel_tile) + // for (auto alt_img_channel : alternate_channel_i_list) { + // std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image + // alternate_channel_i_tile_list.push_back(alt_img_channel_tile) - std::vector alternate_tiles_DFT_list; - for (auto alt_tile : alt_img_channel_tile) { - cv::Mat alt_tile_DFT; - alt_tile.convertTo(alt_tile_DFT, CV_32F); - cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); - alternate_tiles_DFT_list.push_back(alt_tile_DFT); - } - alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); - } + // std::vector alternate_tiles_DFT_list; + // for (auto alt_tile : alt_img_channel_tile) { + // cv::Mat alt_tile_DFT; + // alt_tile.convertTo(alt_tile_DFT, CV_32F); + // cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + // alternate_tiles_DFT_list.push_back(alt_tile_DFT); + // } + // alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); + // } - //get the dft of the alternate image - //std::vector alternate_tiles_DFT; + // //get the dft of the alternate image + // //std::vector alternate_tiles_DFT; - //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; + // //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - //find reference_tiles_DFT - alternate_tiles_DFT_list - std::vector> tile_difference_list; //list of tile differences - for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { - std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; - tile_difference_list.push_back(single_tile_difference); - } + // //find reference_tiles_DFT - alternate_tiles_DFT_list + // std::vector> tile_difference_list; //list of tile differences + // for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { + // std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; + // tile_difference_list.push_back(single_tile_difference); + // } - // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + // // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - - //get the real and imaginary components - /* - std::vector> absolute_difference_list; - for (auto individual_difference : tile_difference_list) { - for (int i =0; i < individual_difference.rows; i++ ) { - std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); - for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { - row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag - } - } + // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag - absolute_difference_list.push_back(single_tile_difference); - } - */ + // //get the real and imaginary components + // /* + // std::vector> absolute_difference_list; + // for (auto individual_difference : tile_difference_list) { + // for (int i =0; i < individual_difference.rows; i++ ) { + // std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); + // for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { + // row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag + // } + // } - //find the squared absolute difference across all the tiles + // //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag + // absolute_difference_list.push_back(single_tile_difference); + // } + // */ + // //find the squared absolute difference across all the tiles - std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) - std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; + // std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) - return merged_image_tiles_fft + // std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; - } + // return merged_image_tiles_fft - std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + // } + + // std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { - double spatial_factor = 1; //to be added - double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; + // double spatial_factor = 1; //to be added + // double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; - //calculate the spatial denoising - spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; - std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; + // //calculate the spatial denoising + // spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; + // std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; - merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) + // merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) - } + // } } // namespace hdrplus \ No newline at end of file From 201399beb55a7f1c5ef40fea3fcc1d74c0e0e563 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Wed, 4 May 2022 21:06:23 -0700 Subject: [PATCH 57/59] Spatial denoising; too much blur --- include/hdrplus/merge.h | 67 ++++++++++++++++++++++++++++++++++++++++- src/merge.cpp | 49 ++++++++++++++++++++++-------- 2 files changed, 103 insertions(+), 13 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 827fa2f..2d10be7 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -96,6 +96,71 @@ class merge return img; } + void circshift(cv::Mat &out, const cv::Point &delta) + { + cv::Size sz = out.size(); + + // error checking + assert(sz.height > 0 && sz.width > 0); + + // no need to shift + if ((sz.height == 1 && sz.width == 1) || (delta.x == 0 && delta.y == 0)) + return; + + // delta transform + int x = delta.x; + int y = delta.y; + if (x > 0) x = x % sz.width; + if (y > 0) y = y % sz.height; + if (x < 0) x = x % sz.width + sz.width; + if (y < 0) y = y % sz.height + sz.height; + + + // in case of multiple dimensions + std::vector planes; + split(out, planes); + + for (size_t i = 0; i < planes.size(); i++) + { + // vertical + cv::Mat tmp0, tmp1, tmp2, tmp3; + cv::Mat q0(planes[i], cv::Rect(0, 0, sz.width, sz.height - y)); + cv::Mat q1(planes[i], cv::Rect(0, sz.height - y, sz.width, y)); + q0.copyTo(tmp0); + q1.copyTo(tmp1); + tmp0.copyTo(planes[i](cv::Rect(0, y, sz.width, sz.height - y))); + tmp1.copyTo(planes[i](cv::Rect(0, 0, sz.width, y))); + + // horizontal + cv::Mat q2(planes[i], cv::Rect(0, 0, sz.width - x, sz.height)); + cv::Mat q3(planes[i], cv::Rect(sz.width - x, 0, x, sz.height)); + q2.copyTo(tmp2); + q3.copyTo(tmp3); + tmp2.copyTo(planes[i](cv::Rect(x, 0, sz.width - x, sz.height))); + tmp3.copyTo(planes[i](cv::Rect(0, 0, x, sz.height))); + } + + cv::merge(planes, out); + } + + void fftshift(cv::Mat &out) + { + cv::Size sz = out.size(); + cv::Point pt(0, 0); + pt.x = (int) floor(sz.width / 2.0); + pt.y = (int) floor(sz.height / 2.0); + circshift(out, pt); + } + + void ifftshift(cv::Mat &out) + { + cv::Size sz = out.size(); + cv::Point pt(0, 0); + pt.x = (int) ceil(sz.width / 2.0); + pt.y = (int) ceil(sz.height / 2.0); + circshift(out, pt); + } + std::vector getReferenceTiles(cv::Mat reference_image); cv::Mat mergeTiles(std::vector tiles, int rows, int cols); @@ -109,7 +174,7 @@ class merge //temporal denoise std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); - std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector spatial_denoise(std::vector tiles, int num_alts, std::vector noise_variance, float spatial_factor); }; diff --git a/src/merge.cpp b/src/merge.cpp index d855f4b..49521aa 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -97,6 +97,7 @@ namespace hdrplus cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); burst_images.merged_bayer_image = merged(vertical, horizontal); + cv::imwrite("merged.jpg", burst_images.merged_bayer_image); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -192,14 +193,14 @@ namespace hdrplus ////adding after here - //std::vector spatial_denoised_tiles = spatial_denoise( reference_tiles, reference_tiles_DFT, noise_varaince) + std::vector spatial_denoised_tiles = spatial_denoise(reference_tiles_DFT, alternate_channel_i_list.size(), noise_variance, 0.1); //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised //// // Apply IFFT on reference tiles (frequency to spatial) std::vector denoised_tiles; - for (auto dft_tile : reference_tiles_DFT) { + for (auto dft_tile : reference_tiles) { cv::Mat denoised_tile; cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); denoised_tile.convertTo(denoised_tile, CV_16U); @@ -304,18 +305,42 @@ namespace hdrplus // } - // std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + std::vector merge::spatial_denoise(std::vector tiles, int num_alts, std::vector noise_variance, float spatial_factor) { - // double spatial_factor = 1; //to be added - // double spatial_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*spatial_factor; + double spatial_noise_scaling = (TILE_SIZE * TILE_SIZE * (1.0 / 16)) * spatial_factor; - // //calculate the spatial denoising - // spatial_tile_dist = reference_tiles.real**2 + reference_tiles.imag**2; - // std::vector WienerCoeff = denoised_tiles*spatial_noise_scaling*noise_variance; - - // merged_channel_tiles_spatial = reference_tiles*spatial_tile_dist/(spatial_tile_dist+WienerCoeff) + // Calculate |w| using ifftshift + cv::Mat row_distances = cv::Mat::zeros(1, TILE_SIZE, CV_32F); + for(int i = 0; i < TILE_SIZE; ++i) { + row_distances.at(i) = i - offset; + } + row_distances = cv::repeat(row_distances.t(), 1, TILE_SIZE); + cv::Mat col_distances = row_distances.t(); + cv::Mat distances; + cv::sqrt(row_distances.mul(row_distances) + col_distances.mul(col_distances), distances); + ifftshift(distances); + + std::vector denoised; + // Loop through all tiles + for (int i = 0; i < tiles.size(); ++i) { + cv::Mat tile = tiles[i]; + float coeff = noise_variance[i] / num_alts * spatial_noise_scaling; + + // Calculate absolute difference + cv::Mat complexMats[2]; + cv::split(tile, complexMats); // planes[0] = Re(DFT(I)), planes[1] = Im(DFT(I)) + cv::magnitude(complexMats[0], complexMats[1], complexMats[0]); // planes[0] = magnitude + cv::Mat absolute_diff = complexMats[0].mul(complexMats[0]); + + // Division + cv::Mat scale; + cv::divide(absolute_diff, absolute_diff + distances * coeff, scale); + cv::merge(std::vector{scale, scale}, scale); + denoised.push_back(tile.mul(scale)); + } - // } + return denoised; + } } // namespace hdrplus \ No newline at end of file From 07308f0d533048089cc28fe44c9468464a797f4f Mon Sep 17 00:00:00 2001 From: cvachha Date: Wed, 4 May 2022 21:49:06 -0700 Subject: [PATCH 58/59] Updating temporal denoising Updating the temporal denoising before committing and pulling in change for spatial denoising. --- src/merge.cpp | 165 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 106 insertions(+), 59 deletions(-) diff --git a/src/merge.cpp b/src/merge.cpp index d855f4b..26160e4 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -196,7 +196,7 @@ namespace hdrplus //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image //reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised //// - + reference_tiles_DFT = spatial_denoised_tiles; // Apply IFFT on reference tiles (frequency to spatial) std::vector denoised_tiles; for (auto dft_tile : reference_tiles_DFT) { @@ -218,91 +218,138 @@ namespace hdrplus return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } - // std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { - // //goal: temporially denoise using the weiner filter - // //input: - // //1. array of 2D dft tiles of the reference image - // //2. array of 2D dft tiles ocf the aligned alternate image - // //3. estimated noise varaince - // //4. temporal factor - // //return: merged image patches dft + std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + //goal: temporially denoise using the weiner filter + //input: + //1. array of 2D dft tiles of the reference image + //2. array of 2D dft tiles ocf the aligned alternate image + //3. estimated noise varaince + //4. temporal factor + //return: merged image patches dft - // //tile_size = TILE_SIZE; + //tile_size = TILE_SIZE; - // double temporal_factor = 8.0 //8 by default - - // double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; + - // //start calculating the merged image tiles fft + //start calculating the merged image tiles fft - // //get the tiles of the alternate image as a list + //get the tiles of the alternate image as a list - // std::vector> alternate_channel_i_tile_list; //list of alt channel tiles - // std::vector> alternate_tiles_DFT_list; //list of alt channel tiles + std::vector> alternate_channel_i_tile_list; //list of alt channel tiles + std::vector> alternate_tiles_DFT_list; //list of alt channel tiles - // for (auto alt_img_channel : alternate_channel_i_list) { - // std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image - // alternate_channel_i_tile_list.push_back(alt_img_channel_tile) + for (auto alt_img_channel : alternate_channel_i_list) { + std::vector alt_img_channel_tile = getReferenceTiles(alt_img_channel); //get tiles from alt image + alternate_channel_i_tile_list.push_back(alt_img_channel_tile) + + std::vector alternate_tiles_DFT; + for (auto alt_tile : alt_img_channel_tile) { + cv::Mat alt_tile_DFT; + alt_tile.convertTo(alt_tile_DFT, CV_32F); + cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); + alternate_tiles_DFT.push_back(alt_tile_DFT); + } + alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); + } - // std::vector alternate_tiles_DFT_list; - // for (auto alt_tile : alt_img_channel_tile) { - // cv::Mat alt_tile_DFT; - // alt_tile.convertTo(alt_tile_DFT, CV_32F); - // cv::dft(alt_tile_DFT, alt_tile_DFT, cv::DFT_SCALE | cv::DFT_COMPLEX_OUTPUT); - // alternate_tiles_DFT_list.push_back(alt_tile_DFT); - // } - // alternate_tiles_DFT_list.push_back(alternate_tiles_DFT); - // } + //get the dft of the alternate image + //std::vector alternate_tiles_DFT; - // //get the dft of the alternate image - // //std::vector alternate_tiles_DFT; + //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; - // //std::vector tile_differences = reference_tiles_DFT - alternate_tiles_DFT_list; + //find reference_tiles_DFT - alternate_tiles_DFT_list + // std::vector> tile_difference_list; //list of tile differences + // for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { + // std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; + // tile_difference_list.push_back(single_tile_difference); + // } - // //find reference_tiles_DFT - alternate_tiles_DFT_list - // std::vector> tile_difference_list; //list of tile differences - // for (auto individual_alternate_tile_DFT : alternate_tiles_DFT_list) { - // std::vector single_tile_difference = reference_tiles_DFT - individual_alternate_tile_DFT; - // tile_difference_list.push_back(single_tile_difference); - // } + // //find the squared absolute difference across all the tiles + // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + // std::vector copy_diff = tile_differences.clone(); //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist + //get the real and imaginary components (real**2 + imag**2) + + // std::vector absolute_distance_list; + // for (auto individual_difference : tile_difference_list) { + // cv::Mat copy_diff = individual_difference.clone(); + // for (int i = 0 ; i < individual_difference.rows; i++ ) { + // //std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); + // for (int j = 0; j < individual_difference.cols*individual_difference.channels(); j++) { + // std::complex single_complex_num = individual_difference.at>(i,j); + // copy_diff.at>(i,j) = math.pow(single_complex_num.real(),2)+math.pow(single_complex_num.imag(),2); //.real and .imag + // } + // } + + // //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag + // absolute_distance_list.push_back(copy_diff); + // } - // // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - // std::vector tile_sq_asolute_diff = tile_differences; //squared absolute difference is tile_differences.real**2 + tile_differnce.imag**2; //also tile_dist - // //get the real and imaginary components - // /* - // std::vector> absolute_difference_list; - // for (auto individual_difference : tile_difference_list) { - // for (int i =0; i < individual_difference.rows; i++ ) { - // std::complex* row_ptr = tile_sq_asolute_diff.ptr>(i); - // for (int j = 0; j< individual_difference.cols*individual_difference.channels(); j++) { - // row_ptr = math.pow(individual_difference.at>(i,j).real(),2)+math.pow(individual_difference.at>(i,j).imag(),2); //.real and .imag - // } - // } + //get shrinkage operator + //std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + //std::vector A; + + //get tile_sq_asolute_diff+noise_variance + // std::vector A_denominator; + // for (int i = 0; i < absolute_distance_list.size();i++){ + // cv::Mat noise_var_mat( noise_variance[i],absolute_distance_list[i].rows,absolute_distance_list[i].cols,CV_16U); + // cv::Mat single_denominator = absolute_distance_list[i] + noise_var_mat; + // A_denominator.push_back(single_denominator); + // } + + // for (auto individual_distance : absolute_distance_list) { + // cv::Mat single_A = + // } + + //std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; - // //std::vector single_tile_difference = individual_difference.at>(0,0).real(); //.real and .imag - // absolute_difference_list.push_back(single_tile_difference); - // } - // */ - // //find the squared absolute difference across all the tiles + //calculate noise scaling + double temporal_factor = 8.0 //8 by default - // std::vector A = tile_sq_asolute_diff/(tile_sq_asolute_diff+noise_variance) + double temporal_noise_scaling = (pow(TILE_SIZE,2) * (1.0/16*2))*temporal_factor; - // std::vector merged_image_tiles_fft = alternate_tiles_DFT_list + A * tile_differences; + //loop across tiles - // return merged_image_tiles_fft + // Calculate absolute difference + + for (int i = 0; i < tiles.size(); ++i) { + cv::Mat tile = tiles[i]; + //float coeff = noise_variance[i] / num_alts * spatial_noise_scaling; - // } + // Calculate absolute difference + cv::Mat complexMats[2]; + cv::split(tile, complexMats); // planes[0] = Re(DFT(I)), planes[1] = Im(DFT(I)) + cv::magnitude(complexMats[0], complexMats[1], complexMats[0]); // planes[0] = magnitude + cv::Mat absolute_diff = complexMats[0].mul(complexMats[0]); + + //find shrinkage operator A + + //create a mat of only the noise variance + cv::Mat noise_var_mat(noise_variance[i]*temporal_noise_scaling,absolute_diff.rows,absolute_diff.cols,CV_16U); + cv::Mat A_denom = absolute_diff+noise_var_mat; + cv::Mat A = cv::divide(absolute_diff,A_denom); + + //update reference DFT + reference_tiles_DFT += alternate_tiles_DFT_list + cv::mul(A,absolute_diff); + + } + + //get average + reference_tiles_DFT = cv::divide(reference_tiles_DFT,tiles.size()) + + return reference_tiles_DFT + + } // std::vector spatial_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { From e44cc5eea046e5867238bfd6e076e25b10a65a13 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Thu, 5 May 2022 00:21:43 -0700 Subject: [PATCH 59/59] Fix identical channels --- include/hdrplus/merge.h | 2 +- src/merge.cpp | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index 2d10be7..d779e75 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -173,7 +173,7 @@ class merge float lambda_read); //temporal denoise - std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince); + std::vector temporal_denoise(std::vector tiles, std::vector alt_imgs, std::vector noise_variance, float temporal_factor); std::vector spatial_denoise(std::vector tiles, int num_alts, std::vector noise_variance, float spatial_factor); diff --git a/src/merge.cpp b/src/merge.cpp index 10b0fba..27495db 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -23,14 +23,15 @@ namespace hdrplus cv::imwrite("ref.jpg", reference_image); // Get raw channels - std::vector channels(4, cv::Mat::zeros(reference_image.rows / 2, reference_image.cols / 2, CV_16U)); + std::vector channels(4); hdrplus::extract_rgb_fmom_bayer(reference_image, channels[0], channels[1], channels[2], channels[3]); + std::vector processed_channels(4); // For each channel, perform denoising and merge for (int i = 0; i < 4; ++i) { // Get channel mat cv::Mat channel_i = channels[i]; - // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); //we should be getting the individual channel in the same place where we call the processChannel function with the reference channel in its arguments //possibly we could add another argument in the processChannel function which is the channel_i for the alternate image. maybe using a loop to cover all the other images @@ -42,7 +43,7 @@ namespace hdrplus //get alternate image cv::Mat alt_image = burst_images.bayer_images_pad[j]; - std::vector alt_channels(4, cv::Mat::zeros(reference_image.rows / 2, reference_image.cols / 2, CV_16U)); + std::vector alt_channels(4); hdrplus::extract_rgb_fmom_bayer(alt_image, alt_channels[0], alt_channels[1], alt_channels[2], alt_channels[3]); alternate_channel_i_list.push_back(alt_channels[i]); @@ -51,10 +52,10 @@ namespace hdrplus // Apply merging on the channel cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i, alternate_channel_i_list, lambda_shot, lambda_read); - // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); + cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); // Put channel raw data back to channels - merged_channel.convertTo(channels[i], CV_16U); + merged_channel.convertTo(processed_channels[i], CV_16U); } // Write all channels back to a bayer mat @@ -63,8 +64,8 @@ namespace hdrplus for (y = 0; y < reference_image.rows; ++y){ uint16_t* row = merged.ptr(y); if (y % 2 == 0){ - uint16_t* i0 = channels[0].ptr(y / 2); - uint16_t* i1 = channels[1].ptr(y / 2); + uint16_t* i0 = processed_channels[0].ptr(y / 2); + uint16_t* i1 = processed_channels[1].ptr(y / 2); for (x = 0; x < reference_image.cols;){ //R @@ -77,8 +78,8 @@ namespace hdrplus } } else { - uint16_t* i2 = channels[2].ptr(y / 2); - uint16_t* i3 = channels[3].ptr(y / 2); + uint16_t* i2 = processed_channels[2].ptr(y / 2); + uint16_t* i3 = processed_channels[3].ptr(y / 2); for(x = 0; x < reference_image.cols;){ //G2 @@ -195,12 +196,12 @@ namespace hdrplus std::vector spatial_denoised_tiles = spatial_denoise(reference_tiles_DFT, alternate_channel_i_list.size(), noise_variance, 0.1); //apply the cosineWindow2D over the merged_channel_tiles_spatial and reconstruct the image - reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised + // reference_tiles = spatial_denoised_tiles; //now reference tiles are temporally and spatially denoised //// reference_tiles_DFT = spatial_denoised_tiles; // Apply IFFT on reference tiles (frequency to spatial) std::vector denoised_tiles; - for (auto dft_tile : reference_tiles) { + for (auto dft_tile : reference_tiles_DFT) { cv::Mat denoised_tile; cv::dft(dft_tile, denoised_tile, cv::DFT_INVERSE | cv::DFT_REAL_OUTPUT); denoised_tile.convertTo(denoised_tile, CV_16U); @@ -219,7 +220,7 @@ namespace hdrplus return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } - std::vector temporal_denoise(std::vector reference_tiles, std::vector reference_tiles_DFT, std::vector noise_varaince) { + std::vector temporal_denoise(std::vector tiles, std::vector alt_imgs, std::vector noise_variance, float temporal_factor) { //goal: temporially denoise using the weiner filter //input: //1. array of 2D dft tiles of the reference image