diff --git a/src/align.cpp b/src/align.cpp index 0a100b0..8f89ec0 100644 --- a/src/align.cpp +++ b/src/align.cpp @@ -12,6 +12,53 @@ namespace hdrplus { +// Function declration +static void build_per_grayimg_pyramid( \ + std::vector& images_pyramid, \ + const cv::Mat& src_image, \ + const std::vector& inv_scale_factors ); + + +template< int pyramid_scale_factor_prev_curr, int tilesize_scale_factor_prev_curr > +static void build_upsampled_prev_aligement( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + int num_tiles_h, int num_tiles_w ); + + +template< int tile_size > +static void build_alignment_consider_neighbour( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + const cv::Mat& ref_img, const cv::Mat& alt_img ); + + +template< typename data_type, typename return_type, int tile_size > +static unsigned long long l1_distance( const cv::Mat& img1, const cv::Mat& img2, \ + int img1_tile_row_start_idx, int img1_tile_col_start_idx, \ + int img2_tile_row_start_idx, int img2_tile_col_start_idx ); + + +template< typename data_type, typename return_type, int tile_size > +static return_type l2_distance( const cv::Mat& img1, const cv::Mat& img2, \ + int img1_tile_row_start_idx, int img1_tile_col_start_idx, \ + int img2_tile_row_start_idx, int img2_tile_col_start_idx ); + + +static void align_image_level( \ + const cv::Mat& ref_img, \ + const cv::Mat& alt_img, \ + std::vector>>& prev_aligement, \ + std::vector>>& curr_alignment, \ + int scale_factor_prev_curr, \ + int curr_tile_size, \ + int prev_tile_size, \ + int search_radiou, \ + int distance_type ); + + +// Function Implementations + // static function only visible within file static void build_per_grayimg_pyramid( \ @@ -43,23 +90,23 @@ static void build_per_grayimg_pyramid( \ downsample_image = src_image; break; case 2: - //printf("downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); + printf("(2) downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); // // Gaussian blur - cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + //cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); // // Downsample - downsample_image = downsample_nearest_neighbour( blur_image ); - //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + //downsample_image = downsample_nearest_neighbour( blur_image ); + downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); // Add images_pyramid.at( i ) = downsample_image.clone(); break; case 4: - printf("downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); - cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); - downsample_image = downsample_nearest_neighbour( blur_image ); - //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + printf("(4) downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); + //cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + //downsample_image = downsample_nearest_neighbour( blur_image ); + downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); images_pyramid.at( i ) = downsample_image.clone(); break; default: @@ -80,7 +127,7 @@ static void build_upsampled_prev_aligement( \ constexpr int repeat_factor = pyramid_scale_factor_prev_curr / tilesize_scale_factor_prev_curr; - // printf("build_upsampled_prev_aligement with scale factor %d, repeat factor %d, tile size factor %d\n", \ + printf("build_upsampled_prev_aligement with scale factor %d, repeat factor %d, tile size factor %d\n", \ pyramid_scale_factor_prev_curr, repeat_factor, tilesize_scale_factor_prev_curr ); int dst_height = src_height * repeat_factor; @@ -119,6 +166,127 @@ static void build_upsampled_prev_aligement( \ } +static bool operator==( const std::pair& lhs, const std::pair& rhs ) +{ + return lhs.first == rhs.first && lhs.second == rhs.second; +} + + +static bool operator!=( const std::pair& lhs, const std::pair& rhs ) +{ + return lhs.first != rhs.first || lhs.second != rhs.second; +} + + +template< int tile_size > +static void build_alignment_consider_neighbour( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + const cv::Mat& ref_img, const cv::Mat& alt_img ) +{ + int num_tiles_h = src_alignment.size(); + int num_tiles_w = src_alignment.at( 0 ).size(); + + // Distance function + unsigned long long (*distance_func_ptr)(const cv::Mat&, const cv::Mat&, int, int, int, int) = \ + &l1_distance; + + // Copy the alignment information + // Below double for loop will only replace the change one + dst_alignment = src_alignment; + + // Main part of the loop + for ( int tile_row_i = 1; tile_row_i < num_tiles_h - 1; tile_row_i++ ) + { + for ( int tile_col_i = 1; tile_col_i < num_tiles_w - 1; tile_col_i++ ) + { + const auto& curr_align_i = src_alignment[ tile_row_i ][ tile_col_i ]; + + // Container for nbr alignment pair + std::vector> nbrs_align_i; + + // Consider 4 neighbour's alignment + // Only compute distance if alignment is different + const auto& nbr1_align_i = src_alignment[ tile_row_i + 0 ][ tile_col_i - 1 ]; + if ( curr_align_i != nbr1_align_i ) nbrs_align_i.emplace_back( nbr1_align_i ); + + const auto& nbr2_align_i = src_alignment[ tile_row_i + 0 ][ tile_col_i + 1 ]; + if ( curr_align_i != nbr2_align_i ) nbrs_align_i.emplace_back( nbr2_align_i ); + + const auto& nbr3_align_i = src_alignment[ tile_row_i - 1 ][ tile_col_i + 0 ]; + if ( curr_align_i != nbr3_align_i ) nbrs_align_i.emplace_back( nbr3_align_i ); + + const auto& nbr4_align_i = src_alignment[ tile_row_i + 1 ][ tile_col_i + 0 ]; + if ( curr_align_i != nbr4_align_i ) nbrs_align_i.emplace_back( nbr4_align_i ); + + // If there is a nbr alignment that need to be considered. Compute distance + if ( ! nbrs_align_i.empty() ) + { + int ref_tile_row_start_idx_i = tile_row_i * tile_size / 2; + int ref_tile_col_start_idx_i = tile_col_i * tile_size / 2; + + // curr_align_i's distance + auto curr_align_i_distance = distance_func_ptr( + ref_img, alt_img, \ + ref_tile_row_start_idx_i, \ + ref_tile_col_start_idx_i, \ + ref_tile_row_start_idx_i + curr_align_i.first, \ + ref_tile_col_start_idx_i + curr_align_i.second ); + + for ( const auto& nbr_align_i : nbrs_align_i ) + { + auto nbr_align_i_distance = distance_func_ptr( + ref_img, alt_img, \ + ref_tile_row_start_idx_i, \ + ref_tile_col_start_idx_i, \ + ref_tile_row_start_idx_i + nbr_align_i.first, \ + ref_tile_col_start_idx_i + nbr_align_i.second ); + + if ( nbr_align_i_distance < curr_align_i_distance ) + { + printf("tile [%d, %d] update align, prev align (%d, %d) curr align (%d, %d), prev distance %d curr distance %d\n", \ + tile_row_i, tile_col_i, \ + curr_align_i.first, curr_align_i.second, \ + nbr_align_i.first, nbr_align_i.second, \ + int(curr_align_i_distance), int(nbr_align_i_distance) ); + + dst_alignment[ tile_row_i ][ tile_col_i ] = nbr_align_i; + curr_align_i_distance = nbr_align_i_distance; + } + + } + } + } + } + + // Border part of the loop + // TOP + // { + // int tile_row_i = 0; + // for ( int tile_col_i = 1; tile_col_i < num_tiles_w - 1; ++tile_col_i ) + // { + + // } + // } + + // TOP LEFT corner + + // RIGHT + + // TOP RIGHT corner + + // LEFT + + // BOTTOM LEFT corner + + // BOTTOM + + // BOTTOM RIGHT CORNER + + +} // end of build_alignment_consider_neighbour + + // Set tilesize as template argument for better compiler optimization result. template< typename data_type, typename return_type, int tile_size > static unsigned long long l1_distance( const cv::Mat& img1, const cv::Mat& img2, \ @@ -245,7 +413,7 @@ static return_type l2_distance( const cv::Mat& img1, const cv::Mat& img2, \ } -void align_image_level( \ +static void align_image_level( \ const cv::Mat& ref_img, \ const cv::Mat& alt_img, \ std::vector>>& prev_aligement, \ @@ -316,6 +484,16 @@ void align_image_level( \ } } + void (*alignment_nbr_func_ptr)( std::vector>>&, std::vector>>& dst_alignment, const cv::Mat&, const cv::Mat& ) = nullptr; + if ( curr_tile_size == 8 ) + { + alignment_nbr_func_ptr = &build_alignment_consider_neighbour<8>; + } + else if ( curr_tile_size == 16 ) + { + alignment_nbr_func_ptr = &build_alignment_consider_neighbour<16>; + } + int num_tiles_h = ref_img.size().height / (curr_tile_size / 2) - 1; int num_tiles_w = ref_img.size().width / (curr_tile_size / 2 ) - 1; @@ -332,18 +510,20 @@ void align_image_level( \ // Upsample previous level alignment else { - upsample_alignment_func_ptr( prev_aligement, upsampled_prev_aligement, num_tiles_h, num_tiles_w ); + std::vector>> upsampled_prev_aligement_tmp; + upsample_alignment_func_ptr( prev_aligement, upsampled_prev_aligement_tmp, num_tiles_h, num_tiles_w ); + alignment_nbr_func_ptr( upsampled_prev_aligement_tmp, upsampled_prev_aligement, ref_img, alt_img ); - // printf("\n!!!!!Upsampled previous alignment\n"); - // for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) - // { - // for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) - // { - // const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); - // printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ - // tile_row, tile_col, tile_start.first, tile_start.second); - // } - // } + printf("\n!!!!!Upsampled previous alignment\n"); + for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) + { + for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) + { + const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); + printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ + tile_row, tile_col, tile_start.first, tile_start.second); + } + } } @@ -492,60 +672,27 @@ void align_image_level( \ } } - // printf("\n!!!!!Min distance for each tile \n"); - // for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) - // { - // for ( int tile_col = 0; tile_col < num_tiles_w; ++tile_col ) - // { - // printf("tile (%d, %d) distance %u\n", \ - // tile_row, tile_col, distances.at( tile_row).at(tile_col ) ); - // } - // } - - // printf("\n!!!!!Alignment at current level\n"); - // for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) - // { - // for ( int tile_col = 0; tile_col < num_tiles_w; tile_col++ ) - // { - // const auto tile_start = curr_alignment.at( tile_row ).at( tile_col ); - // printf("tile (%d, %d) -> start idx (%d, %d)\n", \ - // tile_row, tile_col, tile_start.first, tile_start.second); - // } - // } - -} - - -static void build_per_pyramid_reftiles_start( \ - std::vector>>>& per_pyramid_reftiles_start, \ - const std::vector>& per_grayimg_pyramid, \ - const std::vector& grayimg_tile_sizes ) -{ - per_pyramid_reftiles_start.resize( per_grayimg_pyramid.at(0).size() ); - - // Every image pyramid level - for ( int level_i = 0; level_i < per_grayimg_pyramid.at(0).size(); level_i++ ) + printf("\n!!!!!Min distance for each tile \n"); + for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) { - int level_i_img_h = per_grayimg_pyramid.at(0).at( level_i ).size().height; - int level_i_img_w = per_grayimg_pyramid.at(0).at( level_i ).size().width; - - int level_i_tile_size = grayimg_tile_sizes.at( level_i ); - - int num_tiles_h = level_i_img_h / (level_i_tile_size / 2) - 1; - int num_tiles_w = level_i_img_w / (level_i_tile_size / 2) - 1; - - // Allocate memory - per_pyramid_reftiles_start.at( level_i ).resize( num_tiles_h, std::vector>( num_tiles_w ) ); + for ( int tile_col = 0; tile_col < num_tiles_w; ++tile_col ) + { + printf("tile (%d, %d) distance %u\n", \ + tile_row, tile_col, distances.at( tile_row).at(tile_col ) ); + } + } - for ( int tile_col_i = 0; tile_col_i < num_tiles_h; tile_col_i++ ) + printf("\n!!!!!Alignment at current level\n"); + for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ ) + { + for ( int tile_col = 0; tile_col < num_tiles_w; tile_col++ ) { - for ( int tile_row_j = 0; tile_row_j < num_tiles_w; tile_row_j++ ) - { - per_pyramid_reftiles_start.at( level_i ).at( tile_col_i ).at( tile_row_j ) \ - = std::make_pair( tile_col_i * level_i_tile_size, tile_row_j * level_i_tile_size ); - } + const auto tile_start = curr_alignment.at( tile_row ).at( tile_col ); + printf("tile (%d, %d) -> start idx (%d, %d)\n", \ + tile_row, tile_col, tile_start.first, tile_start.second); } } + } @@ -634,8 +781,8 @@ void align::process( const hdrplus::burst& burst_images, \ // printf("@@@Alignment at level %d is h=%d, w=%d", level_i, curr_alignment.size(), curr_alignment.at(0).size() ); // Stop at second iteration - // if ( level_i == num_levels - 3 ) - // break; + if ( level_i == num_levels - 2 ) + break; } // for pyramid level diff --git a/tests/test_align.cpp b/tests/test_align.cpp index 43521b7..c2a01a3 100644 --- a/tests/test_align.cpp +++ b/tests/test_align.cpp @@ -21,6 +21,8 @@ void test_align_one_level(int argc, char** argv) hdrplus::align align_module; align_module.process( burst_images, alignments ); + exit(1); + // Access alternative image tile in each channel // Below code can be use in merging part for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx ) @@ -39,9 +41,9 @@ void test_align_one_level(int argc, char** argv) hdrplus::extract_rgb_fmom_bayer( bayer_image_pad, rggb_imgs.at(0), rggb_imgs.at(1), rggb_imgs.at(2), rggb_imgs.at(3) ); // Get tile of each channel with the alignments - int tilesize = 16; // tile size of grayscale image - int num_tiles_h = rggb_imgs.at(0).size().height / ( tilesize / 2 ) - 1; - int num_tiles_w = rggb_imgs.at(0).size().width / ( tilesize / 2 ) - 1; + int tile_size = 16; // tile size of grayscale image + int num_tiles_h = rggb_imgs.at(0).size().height / ( tile_size / 2 ) - 1; + int num_tiles_w = rggb_imgs.at(0).size().width / ( tile_size / 2 ) - 1; for ( int img_channel = 0; img_channel < rggb_imgs.size(); ++img_channel ) { @@ -49,8 +51,8 @@ void test_align_one_level(int argc, char** argv) { for ( int tile_col_i = 0; tile_col_i < num_tiles_w; ++tile_col_i ) { - int ref_tile_row_start_idx_i = tile_row_i * tilesize / 2; - int ref_tile_col_start_idx_i = tile_col_i * tilesize / 2; + int ref_tile_row_start_idx_i = tile_row_i * tile_size / 2; + int ref_tile_col_start_idx_i = tile_col_i * tile_size / 2; int alignment_row_i = alignment.at( tile_row_i ).at( tile_col_i ).first; int alignment_col_i = alignment.at( tile_row_i ).at( tile_col_i ).second; @@ -60,7 +62,7 @@ void test_align_one_level(int argc, char** argv) int alt_tile_col_start_idx_i = ref_tile_col_start_idx_i + alignment_col_i; printf("\nAlt img align channel %d tile [%d, %d]\n", img_channel, tile_row_i, tile_col_i ); - hdrplus::print_tile( rggb_imgs.at( img_channel ), tilesize, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i ); + hdrplus::print_tile( rggb_imgs.at( img_channel ), tile_size, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i ); } } }