From 5307cbfe3287b6d6057decff3c06c26d2e9e4132 Mon Sep 17 00:00:00 2001 From: Haohua-Lyu Date: Sat, 30 Apr 2022 02:41:56 -0700 Subject: [PATCH] Finished merging channels (4.4) --- include/hdrplus/burst.h | 3 ++ include/hdrplus/merge.h | 4 +- src/merge.cpp | 91 +++++++++++++++++++++++++++++++---------- 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/include/hdrplus/burst.h b/include/hdrplus/burst.h index def6b78..c4c9dda 100644 --- a/include/hdrplus/burst.h +++ b/include/hdrplus/burst.h @@ -33,6 +33,9 @@ class burst // number of image (including reference) in burst int num_images; + + // Bayer image after merging, stored as cv::Mat + cv::Mat merged_bayer_image; }; } // namespace hdrplus diff --git a/include/hdrplus/merge.h b/include/hdrplus/merge.h index f208c70..c7c5642 100644 --- a/include/hdrplus/merge.h +++ b/include/hdrplus/merge.h @@ -28,7 +28,7 @@ class merge * Outer most vector is per alternative image. * Inner most two vector is for horizontal & vertical tiles */ - void process( const hdrplus::burst& burst_images, \ + void process( hdrplus::burst& burst_images, \ std::vector>>>& alignments); @@ -86,7 +86,7 @@ class merge cv::Mat mergeTiles(std::vector tiles, int rows, int cols); - cv::Mat processChannel( const hdrplus::burst& burst_images, \ + cv::Mat processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image); }; diff --git a/src/merge.cpp b/src/merge.cpp index 6ea1707..4ed50e9 100644 --- a/src/merge.cpp +++ b/src/merge.cpp @@ -7,22 +7,81 @@ namespace hdrplus { -void merge::process( const hdrplus::burst& burst_images, \ +void merge::process( hdrplus::burst& burst_images, \ std::vector>>>& alignments) { double lambda_shot, lambda_read; std::tie(lambda_shot, lambda_read) = burst_images.bayer_images[burst_images.reference_image_idx].get_noise_params(); - // Call merge on each channel + // Get padded bayer image cv::Mat reference_image = burst_images.bayer_images_pad[burst_images.reference_image_idx]; - reference_image.convertTo(reference_image, CV_32F); + // cv::imwrite("ref.jpg", reference_image); - // Get Channels - // cv::Mat channel_0(reference_image.rows, reference_image.cols, CV_32F, (uchar*)reference_image.data, 2 * sizeof(float)); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); + // Get raw channels + std::vector channels[4]; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + channels[0].push_back(reference_image.at(y, x)); + } else { + channels[1].push_back(reference_image.at(y, x)); + } + } else { + if (x % 2 == 0) { + channels[2].push_back(reference_image.at(y, x)); + } else { + channels[3].push_back(reference_image.at(y, x)); + } + } + } + } + + // For each channel, perform denoising and merge + for (int i = 0; i < 4; ++i) { + // Get channel mat + cv::Mat channel_i(reference_image.rows / 2, reference_image.cols / 2, CV_16U, channels[i].data()); + // cv::imwrite("ref" + std::to_string(i) + ".jpg", channel_i); + + // Apply merging on the channel + cv::Mat merged_channel = processChannel(burst_images, alignments, channel_i); + // cv::imwrite("merged" + std::to_string(i) + ".jpg", merged_channel); - //cv::Mat merged_channel = processChannel(burst_images, alignments, channel_0); + // Put channel raw data back to channels + channels[i] = merged_channel.reshape(1, merged_channel.total()); + } + + // Write all channels back to a bayer mat + std::vector merged_raw; + + for (int y = 0; y < reference_image.rows; ++y) { + for (int x = 0; x < reference_image.cols; ++x) { + if (y % 2 == 0) { + if (x % 2 == 0) { + merged_raw.push_back(channels[0][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[1][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } else { + if (x % 2 == 0) { + merged_raw.push_back(channels[2][(y/2)*(reference_image.cols/2) + (x/2)]); + } else { + merged_raw.push_back(channels[3][(y/2)*(reference_image.cols/2) + (x/2)]); + } + } + } + } + // Create merged mat + cv::Mat merged(reference_image.rows, reference_image.cols, CV_16U, merged_raw.data()); + // cv::imwrite("merged.jpg", merged); + + // Remove padding + std::vector padding = burst_images.padding_info_bayer; + cv::Range horizontal = cv::Range(padding[2], reference_image.cols - padding[3]); + cv::Range vertical = cv::Range(padding[0], reference_image.rows - padding[1]); + burst_images.merged_bayer_image = merged(vertical, horizontal); } std::vector merge::getReferenceTiles(cv::Mat reference_image) { @@ -89,15 +148,15 @@ cv::Mat merge::mergeTiles(std::vector tiles, int num_rows, int num_cols return img_original; } -cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ +cv::Mat merge::processChannel( hdrplus::burst& burst_images, \ std::vector>>>& alignments, \ cv::Mat channel_image) { std::vector reference_tiles = getReferenceTiles(channel_image); - // Temporal Denoising + // TODO: Temporal Denoising - // Spatial Denoising + // TODO: Spatial Denoising // Process tiles through 2D cosine window std::vector windowed_tiles; @@ -106,17 +165,7 @@ cv::Mat merge::processChannel( const hdrplus::burst& burst_images, \ } // Merge tiles - cv::Mat merged = mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); - - // cv::Mat outputImg = channel_image.clone(); - // cv::cvtColor(outputImg, outputImg, cv::COLOR_GRAY2RGB); - // cv::imwrite("ref.jpg", outputImg); - // cv::Mat outputImg1 = reference_tiles[0].clone(); - // cv::Mat outputImg1 = cosineWindow2D(reference_tiles[0].clone()); - // cv::Mat outputImg1 = cat2Dtiles(tiles_2D); - // cv::cvtColor(merged, merged, cv::COLOR_GRAY2RGB); - // cv::imwrite("tile0.jpg", merged); - return merged; + return mergeTiles(windowed_tiles, channel_image.rows, channel_image.cols); } } // namespace hdrplus