diff --git a/.gitignore b/.gitignore index 2b7172e..574da8c 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,5 @@ CMakeSettings.json # Xiao local files -debug/ \ No newline at end of file +debug/ +*.txt \ No newline at end of file diff --git a/include/hdrplus/utility.h b/include/hdrplus/utility.h index 16b7717..102f2ff 100644 --- a/include/hdrplus/utility.h +++ b/include/hdrplus/utility.h @@ -130,7 +130,7 @@ void print_cvmat( cv::Mat image ) */ template void extract_rgb_fmom_bayer( const cv::Mat& bayer_img, \ - cv::Mat& red_img, cv::Mat& green_img1, cv::Mat& green_img2, cv::Mat& blue_img ) + cv::Mat& img_ch1, cv::Mat& img_ch2, cv::Mat& img_ch3, cv::Mat& img_ch4 ) { const T* bayer_img_ptr = (const T*)bayer_img.data; int bayer_width = bayer_img.size().width; @@ -145,31 +145,35 @@ void extract_rgb_fmom_bayer( const cv::Mat& bayer_img, \ // RGB image is half the size of bayer image int rgb_width = bayer_width / 2; int rgb_height = bayer_height / 2; - red_img.create( rgb_height, rgb_width, bayer_img.type() ); - green_img1.create( rgb_height, rgb_width, bayer_img.type() ); - green_img2.create( rgb_height, rgb_width, bayer_img.type() ); - blue_img.create( rgb_height, rgb_width, bayer_img.type() ); - int rgb_step = red_img.step1(); + img_ch1.create( rgb_height, rgb_width, bayer_img.type() ); + img_ch2.create( rgb_height, rgb_width, bayer_img.type() ); + img_ch3.create( rgb_height, rgb_width, bayer_img.type() ); + img_ch4.create( rgb_height, rgb_width, bayer_img.type() ); + int rgb_step = img_ch1.step1(); - T* r_img_ptr = (T*)red_img.data; - T* g1_img_ptr = (T*)green_img1.data; - T* g2_img_ptr = (T*)green_img2.data; - T* b_img_ptr = (T*)blue_img.data; + T* img_ch1_ptr = (T*)img_ch1.data; + T* img_ch2_ptr = (T*)img_ch2.data; + T* img_ch3_ptr = (T*)img_ch3.data; + T* img_ch4_ptr = (T*)img_ch4.data; for ( int rgb_row_i = 0; rgb_row_i < rgb_height; rgb_row_i++ ) { int rgb_row_i_offset = rgb_row_i * rgb_step; // Every RGB row corresbonding to two Bayer image row - int bayer_row_i_offset1 = ( rgb_row_i * 2 + 0 ) * bayer_step; // For RG - int bayer_row_i_offset2 = ( rgb_row_i * 2 + 1 ) * bayer_step; // For GB + int bayer_row_i_offset0 = ( rgb_row_i * 2 + 0 ) * bayer_step; // For RG + int bayer_row_i_offset1 = ( rgb_row_i * 2 + 1 ) * bayer_step; // For GB for ( int rgb_col_j = 0; rgb_col_j < rgb_width; rgb_col_j++ ) { - r_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + ( rgb_col_j * 2 + 0 ) ]; - g1_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + ( rgb_col_j * 2 + 1 ) ]; - g2_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset2 + ( rgb_col_j * 2 + 0 ) ]; - b_img_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset2 + ( rgb_col_j * 2 + 1 ) ]; + // img_ch1/2/3/4 : (0,0), (1,0), (0,1), (1,1) + int bayer_col_i_offset0 = rgb_col_j * 2 + 0; + int bayer_col_i_offset1 = rgb_col_j * 2 + 1; + + img_ch1_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset0 + bayer_col_i_offset0 ]; + img_ch3_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset0 + bayer_col_i_offset1 ]; + img_ch2_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + bayer_col_i_offset0 ]; + img_ch4_ptr[ rgb_row_i_offset + rgb_col_j ] = bayer_img_ptr[ bayer_row_i_offset1 + bayer_col_i_offset1 ]; } } } diff --git a/src/align.cpp b/src/align.cpp index 2db276e..0a100b0 100644 --- a/src/align.cpp +++ b/src/align.cpp @@ -19,14 +19,14 @@ static void build_per_grayimg_pyramid( \ const cv::Mat& src_image, \ const std::vector& inv_scale_factors ) { - #ifndef NDEBUG - printf("%s::%s build_per_grayimg_pyramid start with scale factor : ", __FILE__, __func__ ); - for ( int i = 0; i < inv_scale_factors.size(); ++i ) - { - printf("%d ", inv_scale_factors.at( i )); - } - printf("\n"); - #endif + // #ifndef NDEBUG + // printf("%s::%s build_per_grayimg_pyramid start with scale factor : ", __FILE__, __func__ ); + // for ( int i = 0; i < inv_scale_factors.size(); ++i ) + // { + // printf("%d ", inv_scale_factors.at( i )); + // } + // printf("\n"); + // #endif images_pyramid.resize( inv_scale_factors.size() ); @@ -43,13 +43,13 @@ static void build_per_grayimg_pyramid( \ downsample_image = src_image; break; case 2: - printf("downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); + //printf("downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); // // Gaussian blur - // cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); // // Downsample - // downsample_image = downsample_nearest_neighbour( blur_image ); - downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + downsample_image = downsample_nearest_neighbour( blur_image ); + //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); // Add images_pyramid.at( i ) = downsample_image.clone(); @@ -57,9 +57,9 @@ static void build_per_grayimg_pyramid( \ break; case 4: printf("downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); - // cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); - // downsample_image = downsample_nearest_neighbour( blur_image ); - downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + downsample_image = downsample_nearest_neighbour( blur_image ); + //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); images_pyramid.at( i ) = downsample_image.clone(); break; default: @@ -80,7 +80,7 @@ static void build_upsampled_prev_aligement( \ constexpr int repeat_factor = pyramid_scale_factor_prev_curr / tilesize_scale_factor_prev_curr; - printf("build_upsampled_prev_aligement with scale factor %d, repeat factor %d, tile size factor %d\n", \ + // printf("build_upsampled_prev_aligement with scale factor %d, repeat factor %d, tile size factor %d\n", \ pyramid_scale_factor_prev_curr, repeat_factor, tilesize_scale_factor_prev_curr ); int dst_height = src_height * repeat_factor; @@ -334,16 +334,16 @@ void align_image_level( \ { upsample_alignment_func_ptr( prev_aligement, upsampled_prev_aligement, num_tiles_h, num_tiles_w ); - printf("\n!!!!!Upsampled previous alignment\n"); - for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) - { - for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) - { - const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); - printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ - tile_row, tile_col, tile_start.first, tile_start.second); - } - } + // printf("\n!!!!!Upsampled previous alignment\n"); + // for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) + // { + // for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) + // { + // const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); + // printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ + // tile_row, tile_col, tile_start.first, tile_start.second); + // } + // } } @@ -395,6 +395,7 @@ void align_image_level( \ // print_tile( ref_img, curr_tile_size, ref_tile_row_start_idx_i, ref_tile_col_start_idx_i ); // Upsampled alignment at this tile + // Alignment are relative displacement in pixel value int prev_alignment_row_i = upsampled_prev_aligement.at( ref_tile_row_i ).at( ref_tile_col_i ).first; int prev_alignment_col_i = upsampled_prev_aligement.at( ref_tile_row_i ).at( ref_tile_col_i ).second; @@ -411,13 +412,13 @@ void align_image_level( \ { int before = alt_tile_row_start_idx_i; alt_tile_row_start_idx_i = alt_tile_row_idx_max; - printf("@@ change start x from %d to %d\n", before, alt_tile_row_idx_max); + // printf("@@ change start x from %d to %d\n", before, alt_tile_row_idx_max); } if ( alt_tile_col_start_idx_i > alt_tile_col_idx_max ) { int before = alt_tile_col_start_idx_i; alt_tile_col_start_idx_i = alt_tile_col_idx_max; - printf("@@ change start y from %d to %d\n", before, alt_tile_col_idx_max ); + // printf("@@ change start y from %d to %d\n", before, alt_tile_col_idx_max ); } // Because alternative image is padded with search radious. @@ -454,7 +455,7 @@ void align_image_level( \ min_distance_row_i = search_row_j; } - // // If same value, choose the one closer to the original tile location + // If same value, choose the one closer to the original tile location // if ( distance_j == min_distance_i && min_distance_row_i != -1 && min_distance_col_i != -1 ) // { // int prev_distance_row_2_ref = min_distance_row_i - search_radiou; @@ -468,7 +469,7 @@ void align_image_level( \ // // previous min distance idx is farther away from ref tile start location // if ( prev_distance_2_ref_sqr > curr_distance_2_ref_sqr ) // { - // printf("@@@ Same distance %d, choose closer one (%d, %d) instead of (%d, %d)\n", \ + // // printf("@@@ Same distance %d, choose closer one (%d, %d) instead of (%d, %d)\n", \ // distance_j, search_row_j, search_col_j, min_distance_row_i, min_distance_col_i); // min_distance_col_i = search_col_j; // min_distance_row_i = search_row_j; @@ -491,26 +492,26 @@ void align_image_level( \ } } - printf("\n!!!!!Min distance for each tile \n"); - for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) - { - for ( int tile_col = 0; tile_col < num_tiles_w; ++tile_col ) - { - printf("tile (%d, %d) distance %u\n", \ - tile_row, tile_col, distances.at( tile_row).at(tile_col ) ); - } - } + // printf("\n!!!!!Min distance for each tile \n"); + // for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) + // { + // for ( int tile_col = 0; tile_col < num_tiles_w; ++tile_col ) + // { + // printf("tile (%d, %d) distance %u\n", \ + // tile_row, tile_col, distances.at( tile_row).at(tile_col ) ); + // } + // } - printf("\n!!!!!Alignment at current level\n"); - for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) - { - for ( int tile_col = 0; tile_col < num_tiles_w; tile_col++ ) - { - const auto tile_start = curr_alignment.at( tile_row ).at( tile_col ); - printf("tile (%d, %d) -> start idx (%d, %d)\n", \ - tile_row, tile_col, tile_start.first, tile_start.second); - } - } + // printf("\n!!!!!Alignment at current level\n"); + // for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) + // { + // for ( int tile_col = 0; tile_col < num_tiles_w; tile_col++ ) + // { + // const auto tile_start = curr_alignment.at( tile_row ).at( tile_col ); + // printf("tile (%d, %d) -> start idx (%d, %d)\n", \ + // tile_row, tile_col, tile_start.first, tile_start.second); + // } + // } } @@ -555,6 +556,9 @@ void align::process( const hdrplus::burst& burst_images, \ printf("%s::%s align::process start\n", __FILE__, __func__ ); fflush(stdout); #endif + images_alignment.clear(); + images_alignment.resize( burst_images.num_images ); + // image pyramid per image, per pyramid level std::vector> per_grayimg_pyramid; @@ -576,7 +580,7 @@ void align::process( const hdrplus::burst& burst_images, \ this->inv_scale_factors ); } - #ifndef NDEBUG + // #ifndef NDEBUG // printf("%s::%s build image pyramid of size : ", __FILE__, __func__ ); // for ( int level_i = 0; level_i < num_levels; ++level_i ) // { @@ -584,7 +588,7 @@ void align::process( const hdrplus::burst& burst_images, \ // per_grayimg_pyramid[ 0 ][ level_i ].size().width ); // } // printf("\n"); fflush(stdout); - #endif + // #endif // print image pyramid // for ( int level_i; level_i < num_levels; ++level_i ) @@ -611,7 +615,11 @@ void align::process( const hdrplus::burst& burst_images, \ std::vector>> prev_alignment; for ( int level_i = num_levels - 1; level_i >= 0; level_i-- ) // 3,2,1,0 { - printf("\n\n########################align level %d\n", level_i ); + // make curr alignment as previous alignment + prev_alignment.swap( curr_alignment ); + curr_alignment.clear(); + + // printf("\n\n########################align level %d\n", level_i ); align_image_level( ref_grayimg_pyramid[ level_i ], // reference image at current level alt_grayimg_pyramid[ level_i ], // alternative image at current level @@ -623,11 +631,7 @@ void align::process( const hdrplus::burst& burst_images, \ grayimg_search_radious[ level_i ], // search radious distances[ level_i ] ); // L1/L2 distance - printf("@@@Alignment at level %d is h=%d, w=%d", level_i, curr_alignment.size(), curr_alignment.at(0).size() ); - - // make curr alignment as previous alignment - prev_alignment.swap( curr_alignment ); - curr_alignment.clear(); + // printf("@@@Alignment at level %d is h=%d, w=%d", level_i, curr_alignment.size(), curr_alignment.at(0).size() ); // Stop at second iteration // if ( level_i == num_levels - 3 ) @@ -635,6 +639,19 @@ void align::process( const hdrplus::burst& burst_images, \ } // for pyramid level + // Alignment at grayscale image + images_alignment.at( img_idx ).swap( curr_alignment ); + + // printf("\n!!!!!Alternative Image Alignment\n"); + // for ( int tile_row = 0; tile_row < images_alignment.at( img_idx ).size(); tile_row++ ) + // { + // for ( int tile_col = 0; tile_col < images_alignment.at( img_idx ).at(0).size(); tile_col++ ) + // { + // const auto tile_start = images_alignment.at( img_idx ).at( tile_row ).at( tile_col ); + // printf("tile (%d, %d) -> start idx (%d, %d)\n", \ + // tile_row, tile_col, tile_start.first, tile_start.second); + // } + // } } // for alternative image diff --git a/tests/test_align.cpp b/tests/test_align.cpp index 0ef73f9..43521b7 100644 --- a/tests/test_align.cpp +++ b/tests/test_align.cpp @@ -1,7 +1,8 @@ +#include #include "hdrplus/align.h" #include "hdrplus/bayer_image.h" #include "hdrplus/burst.h" -#include +#include "hdrplus/utility.h" void test_align_one_level(int argc, char** argv) { @@ -20,7 +21,52 @@ void test_align_one_level(int argc, char** argv) hdrplus::align align_module; align_module.process( burst_images, alignments ); -} + // Access alternative image tile in each channel + // Below code can be use in merging part + for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx ) + { + if ( img_idx == burst_images.reference_image_idx ) + { + continue; + } + + //const auto& grayscale_image_pad = burst_images.grayscale_images_pad.at( img_idx ); + const auto& bayer_image_pad = burst_images.bayer_images_pad.at( img_idx ); + const auto& alignment = alignments.at( img_idx ); + + // Create RGB channel + std::vector rggb_imgs( 4 ); + hdrplus::extract_rgb_fmom_bayer( bayer_image_pad, rggb_imgs.at(0), rggb_imgs.at(1), rggb_imgs.at(2), rggb_imgs.at(3) ); + + // Get tile of each channel with the alignments + int tilesize = 16; // tile size of grayscale image + int num_tiles_h = rggb_imgs.at(0).size().height / ( tilesize / 2 ) - 1; + int num_tiles_w = rggb_imgs.at(0).size().width / ( tilesize / 2 ) - 1; + + for ( int img_channel = 0; img_channel < rggb_imgs.size(); ++img_channel ) + { + for ( int tile_row_i = 0; tile_row_i < num_tiles_h; ++tile_row_i ) + { + for ( int tile_col_i = 0; tile_col_i < num_tiles_w; ++tile_col_i ) + { + int ref_tile_row_start_idx_i = tile_row_i * tilesize / 2; + int ref_tile_col_start_idx_i = tile_col_i * tilesize / 2; + + int alignment_row_i = alignment.at( tile_row_i ).at( tile_col_i ).first; + int alignment_col_i = alignment.at( tile_row_i ).at( tile_col_i ).second; + + // Alternative image tile i (tile_row_i, tile_col_i) left top pixel index (tile start location) + int alt_tile_row_start_idx_i = ref_tile_row_start_idx_i + alignment_row_i; + int alt_tile_col_start_idx_i = ref_tile_col_start_idx_i + alignment_col_i; + + printf("\nAlt img align channel %d tile [%d, %d]\n", img_channel, tile_row_i, tile_col_i ); + hdrplus::print_tile( rggb_imgs.at( img_channel ), tilesize, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i ); + } + } + } + } + +} // end of test_align_one_level int main(int argc, char** argv)