#include #include #include #include #include // std::runtime_error #include // all opencv header #include "hdrplus/align.h" #include "hdrplus/burst.h" #include "hdrplus/utility.h" namespace hdrplus { // static function only visible within file static void build_per_grayimg_pyramid( \ std::vector& images_pyramid, \ const cv::Mat& src_image, \ const std::vector& inv_scale_factors ) { #ifndef NDEBUG printf("%s::%s build_per_grayimg_pyramid start with scale factor : ", __FILE__, __func__ ); for ( int i = 0; i < inv_scale_factors.size(); ++i ) { printf("%d ", inv_scale_factors.at( i )); } printf("\n"); #endif images_pyramid.resize( inv_scale_factors.size() ); cv::Mat blur_image; cv::Mat downsample_image; for ( int i = 0; i < inv_scale_factors.size(); ++i ) { printf("inv scale factor %d\n", inv_scale_factors.at( i ) ); switch ( inv_scale_factors[ i ] ) { case 1: images_pyramid[ images_pyramid.size() - i - 1 ] = src_image; // cv::Mat use reference count, will not create deep copy downsample_image = src_image; break; case 2: printf("gaussian blur 2 start\n"); fflush(stdout); // Gaussian blur cv::GaussianBlur( downsample_image, blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); printf("gaussian blur 2 done\n"); fflush(stdout); // Downsample downsample_image = downsample_nearest_neighbour( blur_image ); // Add images_pyramid[ images_pyramid.size() - i - 1 ] = downsample_image; break; case 4: printf("gaussian blur 4 start\n"); fflush(stdout); cv::GaussianBlur( downsample_image, blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); printf("gaussian blur 4 done\n"); fflush(stdout); downsample_image = downsample_nearest_neighbour( blur_image ); images_pyramid[ images_pyramid.size() - i - 1 ] = downsample_image; break; default: throw std::runtime_error("inv scale factor " + std::to_string( inv_scale_factors[ i ]) + "invalid" ); } printf("downsample size h=%d w=%d\n", \ downsample_image.size().height, downsample_image.size().width ); fflush(stdout); } } template< int stride > static void upsample_alignment_stride( \ std::vector>>& src_alignment, \ std::vector>>& dst_alignment ) { int src_height = src_alignment.size(); int src_width = src_alignment[ 0 ].size(); int dst_height = src_height * stride; int dst_width = src_width * stride; // Allocate data for dst_alignment dst_alignment.resize( dst_height, std::vector>( dst_width ) ); // Upsample alignment for ( int row_i = 0; row_i < src_height; row_i++ ) { for ( int col_i = 0; col_i < src_width; col_i++ ) { // Scale alignment std::pair align_i = src_alignment[ row_i ][ col_i ]; align_i.first *= stride; align_i.second *= stride; // repeat for ( int stride_row_i = 0; stride_row_i < stride; ++stride_row_i ) { for ( int stride_col_i = 0; stride_col_i < stride; ++stride_col_i ) { dst_alignment[ row_i + stride_row_i ][ col_i + stride_col_i ] = align_i; } } } } } template void print_tile( const cv::Mat& img, int tile_size, int start_idx_x, int start_idx_y ) { const T* img_ptr = (T*)img.data; int src_height = img.size().height; int src_width = img.size().width; int src_step = img.step1(); for ( int row = 0; row < tile_size; ++row ) { const T* img_ptr_row = img_ptr + row * src_step; for ( int col = 0; col < tile_size; ++col ) { printf("%d ", img_ptr_row[ col ] ); } printf("\n"); } printf("\n"); } void align_image_level( \ const cv::Mat& ref_img, \ const cv::Mat& alt_img, \ const std::vector>>& reftiles_start, \ std::vector>>& prev_aligement, \ std::vector>>& alignment, \ int scale_factor_prev_curr, \ int tile_size, \ int prev_tile_size, \ int search_radiou, \ int distance ) { #ifndef NDEBUG printf("%s::%s align_image_level : ", __FILE__, __func__ ); printf("scale_factor_prev_curr %d, tile_size %d, prev_tile_size %d, search_radiou %d, distance %d", \ scale_factor_prev_curr, tile_size, prev_tile_size, search_radiou, distance ); printf("\n"); #endif /* Basic infos */ int num_tiles_h = reftiles_start.size(); int num_tiles_w = reftiles_start.at( 0 ).size(); printf("num tile h %d, num tile w %d\n", num_tiles_h, num_tiles_w); /* Upsample pervious layer alignment */ std::vector>> upsampled_prev_aligement; // Coarsest level // prev_alignment is invalid / empty, construct alignment as (0,0) if ( prev_tile_size == -1 ) { upsampled_prev_aligement.resize( num_tiles_h, std::vector>( num_tiles_w, std::pair(0, 0) ) ); } // Upsample previous level alignment else { if ( scale_factor_prev_curr == 2 ) { // TODO: add choose from 3 neighbour upsample_alignment_stride<2>( prev_aligement, upsampled_prev_aligement ); } else if ( scale_factor_prev_curr == 4 ) { // TODO: add choose from 3 neighbour upsample_alignment_stride<4>( prev_aligement, upsampled_prev_aligement ); } else { throw std::runtime_error("Invalid scale factor" + std::to_string( scale_factor_prev_curr ) ); } } /* Pad alternative image */ cv::Mat alt_img_pad; cv::copyMakeBorder( alt_img, \ alt_img_pad, \ search_radiou, search_radiou, search_radiou, search_radiou, \ cv::BORDER_CONSTANT, cv::Scalar( UINT_LEAST16_MAX ) ); /* Iterate through all reference tile & compute distance */ for ( int ref_tile_row = 0; ref_tile_row < num_tiles_h; ref_tile_row++ ) { for ( int ref_tile_col = 0; ref_tile_col < num_tiles_w; ref_tile_col++ ) { // Upper left index of reference tile int ref_tile_idx_x = reftiles_start.at( ref_tile_row ).at( ref_tile_col ).first; int ref_tile_idx_y = reftiles_start.at( ref_tile_row ).at( ref_tile_col ).second; // Upsampled alignment at this tile // int prev_alignment_x = upsampled_prev_aligement.at( ref_tile_row ).at( ref_tile_col ).first; // int prev_alignment_y = upsampled_prev_aligement.at( ref_tile_row ).at( ref_tile_col ).second; // int alt_tile_idx_x = ref_tile_idx_x + prev_alignment_x; // int alt_tile_idx_y = ref_tile_idx_y + prev_alignment_y; printf("Ref img tile [%d, %d]\n", ref_tile_row, ref_tile_col ); print_tile( ref_img, 8, ref_tile_idx_x, ref_tile_idx_y ); } } } static void build_per_pyramid_reftiles_start( \ std::vector>>>& per_pyramid_reftiles_start, \ const std::vector>& per_grayimg_pyramid, \ const std::vector& grayimg_tile_sizes ) { per_pyramid_reftiles_start.resize( per_grayimg_pyramid.at(0).size() ); // Every image pyramid level for ( int level_i = 0; level_i < per_grayimg_pyramid.at(0).size(); level_i++ ) { int level_i_img_h = per_grayimg_pyramid.at(0).at( level_i ).size().height; int level_i_img_w = per_grayimg_pyramid.at(0).at( level_i ).size().width; int level_i_tile_size = grayimg_tile_sizes.at( level_i ); int num_tiles_h = level_i_img_h / (level_i_tile_size / 2) - 1; int num_tiles_w = level_i_img_w / (level_i_tile_size / 2) - 1; // Allocate memory per_pyramid_reftiles_start.at( level_i ).resize( num_tiles_h, std::vector>( num_tiles_w ) ); for ( int tile_col_i = 0; tile_col_i < num_tiles_h; tile_col_i++ ) { for ( int tile_row_j = 0; tile_row_j < num_tiles_w; tile_row_j++ ) { per_pyramid_reftiles_start.at( level_i ).at( tile_col_i ).at( tile_row_j ) \ = std::make_pair( tile_col_i * level_i_tile_size, tile_row_j * level_i_tile_size ); } } } } void align::process( const hdrplus::burst& burst_images, \ std::vector>>>& images_alignment ) { #ifndef NDEBUG printf("%s::%s align::process start\n", __FILE__, __func__ ); #endif // image pyramid per image, per pyramid level std::vector> per_grayimg_pyramid; per_grayimg_pyramid.resize( burst_images.num_images ); for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx ) { // per_grayimg_pyramid[ img_idx ][ 0 ] is the original image // per_grayimg_pyramid[ img_idx ][ 3 ] is the coarsest image build_per_grayimg_pyramid( per_grayimg_pyramid.at( img_idx ), \ burst_images.grayscale_images_pad.at( img_idx ), \ this->inv_scale_factors ); } #ifndef NDEBUG printf("%s::%s build image pyramid of size : ", __FILE__, __func__ ); for ( int level_i = 0; level_i < num_levels; ++level_i ) { printf("(%d, %d) ", per_grayimg_pyramid[ 0 ][ level_i ].size().height, per_grayimg_pyramid[ 0 ][ level_i ].size().width ); } printf("\n"); #endif // Tile starting location for each tile level std::vector>>> per_pyramid_reftiles_start; build_per_pyramid_reftiles_start( \ per_pyramid_reftiles_start, \ per_grayimg_pyramid, \ grayimg_tile_sizes ); // Align every image const std::vector& ref_grayimg_pyramid = per_grayimg_pyramid[ burst_images.reference_image_idx ]; for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx ) { // Do not align with reference image if ( img_idx == burst_images.reference_image_idx ) continue; const std::vector& alt_grayimg_pyramid = per_grayimg_pyramid[ img_idx ]; // Align every level from coarse to grain // level 0 : finest level, the original image // level 3 : coarsest level std::vector>> curr_alignment; std::vector>> prev_alignment; for ( int level_i = num_levels - 1; level_i >= 0; level_i-- ) { align_image_level( ref_grayimg_pyramid[ level_i ], // reference image at current level alt_grayimg_pyramid[ level_i ], // alternative image at current level per_pyramid_reftiles_start[ level_i ], // reference tile start location for current level prev_alignment, // previous layer alignment curr_alignment, // current layer alignment ( level_i == ( num_levels - 1 ) ? -1 : inv_scale_factors[ level_i ] ), // scale factor between previous layer and current layer. -1 if current layer is the coarsest layer grayimg_tile_sizes[ level_i ], // current level tile size ( level_i == ( num_levels - 1 ) ? -1 : grayimg_tile_sizes[ level_i + 1 ] ), // previous level tile size grayimg_search_radious[ level_i ], // search radious distances[ level_i ] ); // L1/L2 distance // make curr alignment as previous alignment prev_alignment.swap( curr_alignment ); curr_alignment.clear(); break; } // for pyramid level } // for alternative image } } // namespace hdrplus