diff --git a/src/align.cpp b/src/align.cpp index 6681716..7391108 100644 --- a/src/align.cpp +++ b/src/align.cpp @@ -13,6 +13,53 @@ namespace hdrplus { +// Function declration +static void build_per_grayimg_pyramid( \ + std::vector& images_pyramid, \ + const cv::Mat& src_image, \ + const std::vector& inv_scale_factors ); + + +template< int pyramid_scale_factor_prev_curr, int tilesize_scale_factor_prev_curr > +static void build_upsampled_prev_aligement( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + int num_tiles_h, int num_tiles_w ); + + +template< int tile_size > +static void build_alignment_consider_neighbour( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + const cv::Mat& ref_img, const cv::Mat& alt_img ); + + +template< typename data_type, typename return_type, int tile_size > +static unsigned long long l1_distance( const cv::Mat& img1, const cv::Mat& img2, \ + int img1_tile_row_start_idx, int img1_tile_col_start_idx, \ + int img2_tile_row_start_idx, int img2_tile_col_start_idx ); + + +template< typename data_type, typename return_type, int tile_size > +static return_type l2_distance( const cv::Mat& img1, const cv::Mat& img2, \ + int img1_tile_row_start_idx, int img1_tile_col_start_idx, \ + int img2_tile_row_start_idx, int img2_tile_col_start_idx ); + + +static void align_image_level( \ + const cv::Mat& ref_img, \ + const cv::Mat& alt_img, \ + std::vector>>& prev_aligement, \ + std::vector>>& curr_alignment, \ + int scale_factor_prev_curr, \ + int curr_tile_size, \ + int prev_tile_size, \ + int search_radiou, \ + int distance_type ); + + +// Function Implementations + // static function only visible within file static void build_per_grayimg_pyramid( \ @@ -44,23 +91,23 @@ static void build_per_grayimg_pyramid( \ downsample_image = src_image; break; case 2: + // printf("(2) downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); // // Gaussian blur - cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + //cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); // // Downsample - downsample_image = downsample_nearest_neighbour( blur_image ); - //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + //downsample_image = downsample_nearest_neighbour( blur_image ); + downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); // Add images_pyramid.at( i ) = downsample_image.clone(); break; case 4: - cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); - - downsample_image = downsample_nearest_neighbour( blur_image ); - - //downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); + // printf("(4) downsample with gaussian sigma %.2f", inv_scale_factors[ i ] * 0.5 ); + //cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 ); + //downsample_image = downsample_nearest_neighbour( blur_image ); + downsample_image = downsample_nearest_neighbour( images_pyramid.at( i-1 ) ); images_pyramid.at( i ) = downsample_image.clone(); break; default: @@ -125,6 +172,127 @@ static void build_upsampled_prev_aligement( \ } +static bool operator==( const std::pair& lhs, const std::pair& rhs ) +{ + return lhs.first == rhs.first && lhs.second == rhs.second; +} + + +static bool operator!=( const std::pair& lhs, const std::pair& rhs ) +{ + return lhs.first != rhs.first || lhs.second != rhs.second; +} + + +template< int tile_size > +static void build_alignment_consider_neighbour( \ + std::vector>>& src_alignment, \ + std::vector>>& dst_alignment, + const cv::Mat& ref_img, const cv::Mat& alt_img ) +{ + int num_tiles_h = src_alignment.size(); + int num_tiles_w = src_alignment.at( 0 ).size(); + + // Distance function + unsigned long long (*distance_func_ptr)(const cv::Mat&, const cv::Mat&, int, int, int, int) = \ + &l1_distance; + + // Copy the alignment information + // Below double for loop will only replace the change one + dst_alignment = src_alignment; + + // Main part of the loop + for ( int tile_row_i = 1; tile_row_i < num_tiles_h - 1; tile_row_i++ ) + { + for ( int tile_col_i = 1; tile_col_i < num_tiles_w - 1; tile_col_i++ ) + { + const auto& curr_align_i = src_alignment[ tile_row_i ][ tile_col_i ]; + + // Container for nbr alignment pair + std::vector> nbrs_align_i; + + // Consider 4 neighbour's alignment + // Only compute distance if alignment is different + const auto& nbr1_align_i = src_alignment[ tile_row_i + 0 ][ tile_col_i - 1 ]; + if ( curr_align_i != nbr1_align_i ) nbrs_align_i.emplace_back( nbr1_align_i ); + + const auto& nbr2_align_i = src_alignment[ tile_row_i + 0 ][ tile_col_i + 1 ]; + if ( curr_align_i != nbr2_align_i ) nbrs_align_i.emplace_back( nbr2_align_i ); + + const auto& nbr3_align_i = src_alignment[ tile_row_i - 1 ][ tile_col_i + 0 ]; + if ( curr_align_i != nbr3_align_i ) nbrs_align_i.emplace_back( nbr3_align_i ); + + const auto& nbr4_align_i = src_alignment[ tile_row_i + 1 ][ tile_col_i + 0 ]; + if ( curr_align_i != nbr4_align_i ) nbrs_align_i.emplace_back( nbr4_align_i ); + + // If there is a nbr alignment that need to be considered. Compute distance + if ( ! nbrs_align_i.empty() ) + { + int ref_tile_row_start_idx_i = tile_row_i * tile_size / 2; + int ref_tile_col_start_idx_i = tile_col_i * tile_size / 2; + + // curr_align_i's distance + auto curr_align_i_distance = distance_func_ptr( + ref_img, alt_img, \ + ref_tile_row_start_idx_i, \ + ref_tile_col_start_idx_i, \ + ref_tile_row_start_idx_i + curr_align_i.first, \ + ref_tile_col_start_idx_i + curr_align_i.second ); + + for ( const auto& nbr_align_i : nbrs_align_i ) + { + auto nbr_align_i_distance = distance_func_ptr( + ref_img, alt_img, \ + ref_tile_row_start_idx_i, \ + ref_tile_col_start_idx_i, \ + ref_tile_row_start_idx_i + nbr_align_i.first, \ + ref_tile_col_start_idx_i + nbr_align_i.second ); + + if ( nbr_align_i_distance < curr_align_i_distance ) + { + printf("tile [%d, %d] update align, prev align (%d, %d) curr align (%d, %d), prev distance %d curr distance %d\n", \ + tile_row_i, tile_col_i, \ + curr_align_i.first, curr_align_i.second, \ + nbr_align_i.first, nbr_align_i.second, \ + int(curr_align_i_distance), int(nbr_align_i_distance) ); + + dst_alignment[ tile_row_i ][ tile_col_i ] = nbr_align_i; + curr_align_i_distance = nbr_align_i_distance; + } + + } + } + } + } + + // Border part of the loop + // TOP + // { + // int tile_row_i = 0; + // for ( int tile_col_i = 1; tile_col_i < num_tiles_w - 1; ++tile_col_i ) + // { + + // } + // } + + // TOP LEFT corner + + // RIGHT + + // TOP RIGHT corner + + // LEFT + + // BOTTOM LEFT corner + + // BOTTOM + + // BOTTOM RIGHT CORNER + + +} // end of build_alignment_consider_neighbour + + // Set tilesize as template argument for better compiler optimization result. template< typename data_type, typename return_type, int tile_size > static unsigned long long l1_distance( const cv::Mat& img1, const cv::Mat& img2, \ @@ -423,18 +591,20 @@ void align_image_level( \ // Upsample previous level alignment else { - upsample_alignment_func_ptr( prev_aligement, upsampled_prev_aligement, num_tiles_h, num_tiles_w ); + std::vector>> upsampled_prev_aligement_tmp; + upsample_alignment_func_ptr( prev_aligement, upsampled_prev_aligement_tmp, num_tiles_h, num_tiles_w ); + alignment_nbr_func_ptr( upsampled_prev_aligement_tmp, upsampled_prev_aligement, ref_img, alt_img ); - // printf("\n!!!!!Upsampled previous alignment\n"); - // for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) - // { - // for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) - // { - // const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); - // printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ - // tile_row, tile_col, tile_start.first, tile_start.second); - // } - // } + printf("\n!!!!!Upsampled previous alignment\n"); + for ( int tile_row = 0; tile_row < upsampled_prev_aligement.size(); tile_row++ ) + { + for ( int tile_col = 0; tile_col < upsampled_prev_aligement.at(0).size(); tile_col++ ) + { + const auto tile_start = upsampled_prev_aligement.at( tile_row ).at( tile_col ); + printf("up tile (%d, %d) -> start idx (%d, %d)\n", \ + tile_row, tile_col, tile_start.first, tile_start.second); + } + } } @@ -707,8 +877,8 @@ void align::process( const hdrplus::burst& burst_images, \ // printf("@@@Alignment at level %d is h=%d, w=%d", level_i, curr_alignment.size(), curr_alignment.at(0).size() ); // Stop at second iteration - // if ( level_i == num_levels - 3 ) - // break; + if ( level_i == num_levels - 2 ) + break; } // for pyramid level diff --git a/tests/test_align.cpp b/tests/test_align.cpp index cbe95c9..2689a8b 100644 --- a/tests/test_align.cpp +++ b/tests/test_align.cpp @@ -41,9 +41,9 @@ void test_align_one_level(int argc, char** argv) hdrplus::extract_rgb_from_bayer( bayer_image_pad, rggb_imgs.at(0), rggb_imgs.at(1), rggb_imgs.at(2), rggb_imgs.at(3) ); // Get tile of each channel with the alignments - int tilesize = 16; // tile size of grayscale image - int num_tiles_h = rggb_imgs.at(0).size().height / ( tilesize / 2 ) - 1; - int num_tiles_w = rggb_imgs.at(0).size().width / ( tilesize / 2 ) - 1; + int tile_size = 16; // tile size of grayscale image + int num_tiles_h = rggb_imgs.at(0).size().height / ( tile_size / 2 ) - 1; + int num_tiles_w = rggb_imgs.at(0).size().width / ( tile_size / 2 ) - 1; for ( int img_channel = 0; img_channel < int(rggb_imgs.size()); ++img_channel ) { @@ -51,8 +51,8 @@ void test_align_one_level(int argc, char** argv) { for ( int tile_col_i = 0; tile_col_i < num_tiles_w; ++tile_col_i ) { - int ref_tile_row_start_idx_i = tile_row_i * tilesize / 2; - int ref_tile_col_start_idx_i = tile_col_i * tilesize / 2; + int ref_tile_row_start_idx_i = tile_row_i * tile_size / 2; + int ref_tile_col_start_idx_i = tile_col_i * tile_size / 2; int alignment_row_i = alignment.at( tile_row_i ).at( tile_col_i ).first; int alignment_col_i = alignment.at( tile_row_i ).at( tile_col_i ).second; @@ -62,7 +62,7 @@ void test_align_one_level(int argc, char** argv) int alt_tile_col_start_idx_i = ref_tile_col_start_idx_i + alignment_col_i; printf("\nAlt img align channel %d tile [%d, %d]\n", img_channel, tile_row_i, tile_col_i ); - hdrplus::print_tile( rggb_imgs.at( img_channel ), tilesize, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i ); + hdrplus::print_tile( rggb_imgs.at( img_channel ), tile_size, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i ); } } }