diff --git a/include/hdrplus/align.h b/include/hdrplus/align.h index 60e87c4..41ae831 100644 --- a/include/hdrplus/align.h +++ b/include/hdrplus/align.h @@ -34,17 +34,5 @@ class align const int num_levels = 4; }; -void align_image_level( \ - const cv::Mat& ref_img, \ - const cv::Mat& alt_img, \ - const std::vector>>& reftiles_start, \ - std::vector>>& prev_aligement, \ - std::vector>>& alignment, \ - int scale_factor_prev_curr, \ - int tile_size, \ - int prev_tile_size, \ - int search_radiou, \ - int distance ); - } // namespace hdrplus diff --git a/include/hdrplus/utility.h b/include/hdrplus/utility.h index 67eb135..1f673aa 100644 --- a/include/hdrplus/utility.h +++ b/include/hdrplus/utility.h @@ -16,41 +16,7 @@ namespace hdrplus { -template -cv::Mat box_filter_2x2( const cv::Mat& src_image ) -{ - // https://stackoverflow.com/questions/34042112/opencv-mat-data-member-access - const T* src_image_ptr = (T*)src_image.data; - int src_height = src_image.size().height; - int src_width = src_image.size().width; - int src_step = src_image.step1(); - if ( src_height % 2 != 0 || src_width % 2 != 0 ) - { - throw std::runtime_error( std::string( __FILE__ ) + "::" + __func__ + " source image need to have size multiplier of 2\n" ); - } - - cv::Mat dst_image( src_height / 2, src_width / 2, src_image.type() ); - T* dst_image_ptr = (T*)dst_image.data; - int dst_step = dst_image.step1(); - - // -03 should be enough to optimize below code - for ( int row_i = 0; row_i < src_height; row_i += 2 ) - { - for ( int col_i = 0; col_i < src_width; col_i += 2 ) - { - T box_sum = src_image_ptr[ ( row_i + 0 ) * src_step + col_i + 0 ] + \ - src_image_ptr[ ( row_i + 0 ) * src_step + col_i + 1 ] + \ - src_image_ptr[ ( row_i + 1 ) * src_step + col_i + 0 ] + \ - src_image_ptr[ ( row_i + 1 ) * src_step + col_i + 1 ]; - T box_avg = ( box_sum + T(3) ) / 4; // take ceiling - dst_image_ptr[ ( row_i / 2 ) * dst_step + ( col_i / 2 ) ] = box_avg; - } - } - - // cv::Mat internally use reference count. Will not copy by value here - return dst_image; -} template cv::Mat box_filter_kxk( const cv::Mat& src_image ) @@ -60,38 +26,40 @@ cv::Mat box_filter_kxk( const cv::Mat& src_image ) int src_width = src_image.size().width; int src_step = src_image.step1(); - if ( src_height % kernel != 0 || src_width % kernel != 0 ) + if ( kernel <= 0 ) { - throw std::runtime_error( std::string( __FILE__ ) + "::" + __func__ + " source image need to have size multiplier of kernel\n" ); + throw std::runtime_error(std::string( __FILE__ ) + "::" + __func__ + " box filter only support kernel size >= 1"); } + // int(src_height / kernel) = floor(src_height / kernel) + // When input size is not multiplier of kernel, take floor cv::Mat dst_image( src_height / kernel, src_width / kernel, src_image.type() ); T* dst_image_ptr = (T*)dst_image.data; + int dst_height = dst_image.size().height; + int dst_width = dst_image.size().width; int dst_step = dst_image.step1(); - // -03 should be enough to optimize below code - for ( int row_i = 0; row_i < src_height; row_i += kernel ) + for ( int row_i = 0; row_i < dst_height; ++row_i ) { - for ( int col_i = 0; col_i < src_width; col_i += kernel ) + for ( int col_i = 0; col_i < dst_width; col_i++ ) { T box_sum = T(0); - //#pragma LOOP_UNROLL for ( int kernel_row_i = 0; kernel_row_i < kernel; ++kernel_row_i ) { //#pragma LOOP_UNROLL for ( int kernel_col_i = 0; kernel_col_i < kernel; ++kernel_col_i ) { - box_sum += src_image_ptr[ ( row_i + kernel_row_i ) * src_step + ( col_i + kernel_col_i ) ]; + box_sum += src_image_ptr[ ( row_i * kernel + kernel_row_i ) * src_step + ( col_i * kernel + kernel_col_i ) ]; } } - T box_avg = ( box_sum + T( kernel * kernel - 1 ) ) / ( kernel * kernel ); // take ceiling - dst_image_ptr[ ( row_i / kernel ) * dst_step + ( col_i / kernel ) ] = box_avg; + // Average by taking ceiling + T box_avg = box_sum / T( kernel * kernel ); + dst_image_ptr[ row_i * dst_step + col_i ] = box_avg; } } - // cv::Mat internally use reference count. Will not copy by value here return dst_image; } @@ -105,23 +73,23 @@ cv::Mat downsample_nearest_neighbour( const cv::Mat& src_image ) int src_step = src_image.step1(); // int(src_height / kernel) = floor(src_height / kernel) - cv::Mat dst_image( src_height / kernel, src_width / kernel, src_image.type() ); + // When input size is not multiplier of kernel, take floor + cv::Mat dst_image = cv::Mat( src_height / kernel, src_width / kernel, src_image.type() ); T* dst_image_ptr = (T*)dst_image.data; int dst_height = dst_image.size().height; int dst_width = dst_image.size().width; int dst_step = dst_image.step1(); // -03 should be enough to optimize below code - for ( int row_i = 0; row_i < dst_height; row_i += kernel ) + for ( int row_i = 0; row_i < dst_height; row_i++ ) { - for ( int col_i = 0; col_i < dst_width; col_i += kernel ) + for ( int col_i = 0; col_i < dst_width; col_i++ ) { dst_image_ptr[ row_i * dst_step + col_i ] = \ src_image_ptr[ (row_i * kernel) * src_step + (col_i * kernel) ]; } } - // cv::Mat internally use reference count. Will not copy by value (deepcopy) here return dst_image; } diff --git a/src/align.cpp b/src/align.cpp index 1f87f63..4575569 100644 --- a/src/align.cpp +++ b/src/align.cpp @@ -28,49 +28,37 @@ static void build_per_grayimg_pyramid( \ images_pyramid.resize( inv_scale_factors.size() ); - cv::Mat blur_image; - cv::Mat downsample_image; - for ( int i = 0; i < inv_scale_factors.size(); ++i ) { - printf("inv scale factor %d\n", inv_scale_factors.at( i ) ); + cv::Mat blur_image; + cv::Mat downsample_image; switch ( inv_scale_factors[ i ] ) { case 1: - images_pyramid[ images_pyramid.size() - i - 1 ] = src_image; + images_pyramid[ i ] = src_image.clone(); // cv::Mat use reference count, will not create deep copy downsample_image = src_image; break; case 2: - printf("gaussian blur 2 start\n"); fflush(stdout); - // Gaussian blur - cv::GaussianBlur( downsample_image, blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); - - printf("gaussian blur 2 done\n"); fflush(stdout); + cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); // Downsample downsample_image = downsample_nearest_neighbour( blur_image ); // Add - images_pyramid[ images_pyramid.size() - i - 1 ] = downsample_image; + images_pyramid.at( i ) = downsample_image.clone(); break; case 4: - printf("gaussian blur 4 start\n"); fflush(stdout); - cv::GaussianBlur( downsample_image, blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); - printf("gaussian blur 4 done\n"); fflush(stdout); - + cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 ); downsample_image = downsample_nearest_neighbour( blur_image ); - images_pyramid[ images_pyramid.size() - i - 1 ] = downsample_image; + images_pyramid.at( i ) = downsample_image.clone(); break; default: throw std::runtime_error("inv scale factor " + std::to_string( inv_scale_factors[ i ]) + "invalid" ); - } - - printf("downsample size h=%d w=%d\n", \ - downsample_image.size().height, downsample_image.size().width ); fflush(stdout); + } } } @@ -120,12 +108,44 @@ void print_tile( const cv::Mat& img, int tile_size, int start_idx_x, int start_i int src_width = img.size().width; int src_step = img.step1(); - for ( int row = 0; row < tile_size; ++row ) + for ( int row = start_idx_x; row < tile_size + start_idx_x; ++row ) { const T* img_ptr_row = img_ptr + row * src_step; - for ( int col = 0; col < tile_size; ++col ) + for ( int col = start_idx_y; col < tile_size + start_idx_y; ++col ) + { + printf("%u ", img_ptr_row[ col ] ); + } + printf("\n"); + } + printf("\n"); +} + + +template< typename T> +void print_img( const cv::Mat& img, int img_height = -1, int img_width = -1 ) +{ + const T* img_ptr = (T*)img.data; + if ( img_height == -1 && img_width == -1 ) + { + img_height = img.size().height; + img_width = img.size().width; + } + else + { + img_height = std::min( img.size().height, img_height ); + img_width = std::min( img.size().width, img_width ); + } + printf("Image size (h=%d, w=%d), Print range (h=0-%d, w=0-%d)]\n", \ + img.size().height, img.size().width, img_height, img_width ); + + int img_step = img.step1(); + + for ( int row = 0; row < img_height; ++row ) + { + const T* img_ptr_row = img_ptr + row * img_step; + for ( int col = 0; col < img_width; ++col ) { - printf("%d ", img_ptr_row[ col ] ); + printf("%u ", img_ptr_row[ col ]); } printf("\n"); } @@ -136,7 +156,6 @@ void print_tile( const cv::Mat& img, int tile_size, int start_idx_x, int start_i void align_image_level( \ const cv::Mat& ref_img, \ const cv::Mat& alt_img, \ - const std::vector>>& reftiles_start, \ std::vector>>& prev_aligement, \ std::vector>>& alignment, \ int scale_factor_prev_curr, \ @@ -145,18 +164,21 @@ void align_image_level( \ int search_radiou, \ int distance ) { + /* Basic infos */ + int num_tiles_h = ref_img.size().height / (tile_size / 2) - 1; + int num_tiles_w = ref_img.size().width / (tile_size / 2 ) - 1 ; + #ifndef NDEBUG - printf("%s::%s align_image_level : ", __FILE__, __func__ ); - printf("scale_factor_prev_curr %d, tile_size %d, prev_tile_size %d, search_radiou %d, distance %d", \ + printf("%s::%s start: \n", __FILE__, __func__ ); + printf(" scale_factor_prev_curr %d, tile_size %d, prev_tile_size %d, search_radiou %d, distance L%d, \n", \ scale_factor_prev_curr, tile_size, prev_tile_size, search_radiou, distance ); - printf("\n"); + printf(" ref img size h=%d w=%d, alt img size h=%d w=%d, \n", \ + ref_img.size().height, ref_img.size().width, alt_img.size().height, alt_img.size().width ); + printf(" num tile h %d, num tile w %d\n", num_tiles_h, num_tiles_w); #endif - /* Basic infos */ - int num_tiles_h = reftiles_start.size(); - int num_tiles_w = reftiles_start.at( 0 ).size(); - - printf("num tile h %d, num tile w %d\n", num_tiles_h, num_tiles_w); + printf("Reference image : \n"); + print_img( ref_img ); /* Upsample pervious layer alignment */ std::vector>> upsampled_prev_aligement; @@ -199,8 +221,8 @@ void align_image_level( \ for ( int ref_tile_col = 0; ref_tile_col < num_tiles_w; ref_tile_col++ ) { // Upper left index of reference tile - int ref_tile_idx_x = reftiles_start.at( ref_tile_row ).at( ref_tile_col ).first; - int ref_tile_idx_y = reftiles_start.at( ref_tile_row ).at( ref_tile_col ).second; + int ref_tile_idx_x = ref_tile_row * tile_size / 2; + int ref_tile_idx_y = ref_tile_col * tile_size / 2; // Upsampled alignment at this tile // int prev_alignment_x = upsampled_prev_aligement.at( ref_tile_row ).at( ref_tile_col ).first; @@ -209,7 +231,9 @@ void align_image_level( \ // int alt_tile_idx_x = ref_tile_idx_x + prev_alignment_x; // int alt_tile_idx_y = ref_tile_idx_y + prev_alignment_y; - printf("Ref img tile [%d, %d]\n", ref_tile_row, ref_tile_col ); + printf("Ref img tile [%d, %d] -> start [%d, %d]\n", \ + ref_tile_row, ref_tile_col, ref_tile_idx_x, ref_tile_idx_y ); + print_tile( ref_img, 8, ref_tile_idx_x, ref_tile_idx_y ); } } @@ -254,7 +278,7 @@ void align::process( const hdrplus::burst& burst_images, \ std::vector>>>& images_alignment ) { #ifndef NDEBUG - printf("%s::%s align::process start\n", __FILE__, __func__ ); + printf("%s::%s align::process start\n", __FILE__, __func__ ); fflush(stdout); #endif // image pyramid per image, per pyramid level @@ -277,16 +301,14 @@ void align::process( const hdrplus::burst& burst_images, \ printf("(%d, %d) ", per_grayimg_pyramid[ 0 ][ level_i ].size().height, per_grayimg_pyramid[ 0 ][ level_i ].size().width ); } - printf("\n"); + printf("\n"); fflush(stdout); #endif - // Tile starting location for each tile level - std::vector>>> per_pyramid_reftiles_start; - - build_per_pyramid_reftiles_start( \ - per_pyramid_reftiles_start, \ - per_grayimg_pyramid, \ - grayimg_tile_sizes ); + for ( int level_i; level_i < num_levels; ++level_i ) + { + printf("level %d img : \n" , level_i ); + print_img( per_grayimg_pyramid[ burst_images.reference_image_idx ][ level_i], 100, 100 ); + } // Align every image const std::vector& ref_grayimg_pyramid = per_grayimg_pyramid[ burst_images.reference_image_idx ]; @@ -308,7 +330,6 @@ void align::process( const hdrplus::burst& burst_images, \ align_image_level( ref_grayimg_pyramid[ level_i ], // reference image at current level alt_grayimg_pyramid[ level_i ], // alternative image at current level - per_pyramid_reftiles_start[ level_i ], // reference tile start location for current level prev_alignment, // previous layer alignment curr_alignment, // current layer alignment ( level_i == ( num_levels - 1 ) ? -1 : inv_scale_factors[ level_i ] ), // scale factor between previous layer and current layer. -1 if current layer is the coarsest layer diff --git a/src/bayer_image.cpp b/src/bayer_image.cpp index 773bd00..c5bfced 100644 --- a/src/bayer_image.cpp +++ b/src/bayer_image.cpp @@ -7,7 +7,7 @@ #include #include // exiv2 #include "hdrplus/bayer_image.h" -#include "hdrplus/utility.h" // box_filter_2x2 +#include "hdrplus/utility.h" // box_filter_kxk namespace hdrplus { @@ -58,7 +58,7 @@ bayer_image::bayer_image( const std::string& bayer_image_path ) raw_image = cv::Mat( height, width, CV_16U, libraw_processor->imgdata.rawdata.raw_image ).clone(); // changed the order of width and height // 2x2 box filter - grayscale_image = box_filter_2x2( raw_image ); + grayscale_image = box_filter_kxk( raw_image ); #ifndef NDEBUG printf("%s::%s read bayer image %s with\n width %zu\n height %zu\n iso %.3f\n white level %d\n black level %d %d %d %d\n", \ diff --git a/src/burst.cpp b/src/burst.cpp index 0fa6da0..c6156fe 100644 --- a/src/burst.cpp +++ b/src/burst.cpp @@ -84,7 +84,7 @@ burst::burst( const std::string& burst_path, const std::string& reference_image_ // cv::Mat use internal reference count bayer_images_pad.emplace_back( bayer_image_pad_i ); - grayscale_images_pad.emplace_back( box_filter_2x2( bayer_image_pad_i ) ); + grayscale_images_pad.emplace_back( box_filter_kxk( bayer_image_pad_i ) ); } #ifndef NDEBUG diff --git a/tests/test_utility.cpp b/tests/test_utility.cpp index be12959..fa6819c 100644 --- a/tests/test_utility.cpp +++ b/tests/test_utility.cpp @@ -19,12 +19,12 @@ void print_cvmat( cv::Mat image ) } } -void test_box_filter_2x2() +void test_downsample_nearest_neighbour( ) { - printf("\n###Test test_box_filter_2x2()###\n"); + printf("\n###Test test_box_filter_kxk()###\n"); // Intialize input data - int src_width = 10; - int src_height = 6; + int src_width = 12; + int src_height = 8; std::vector src_data( src_width, src_height ); for ( int i = 0; i < src_width * src_height; ++i ) @@ -38,12 +38,18 @@ void test_box_filter_2x2() printf("src cv::Mat is \n"); print_cvmat( src_image ); - cv::Mat dst_image = hdrplus::box_filter_2x2( src_image ); + cv::Mat dst_image = hdrplus::downsample_nearest_neighbour( src_image ); - printf("dst cv::Mat is \n"); + printf("dst cv::Mat downsample nn 2x2 is \n"); + print_cvmat( dst_image ); + + dst_image = hdrplus::downsample_nearest_neighbour( src_image ); + + printf("dst cv::Mat downsample nn 4x4 is \n"); print_cvmat( dst_image ); } + void test_box_filter_kxk() { printf("\n###Test test_box_filter_kxk()###\n"); @@ -77,6 +83,6 @@ void test_box_filter_kxk() int main() { - test_box_filter_2x2(); + test_downsample_nearest_neighbour(); test_box_filter_kxk(); } \ No newline at end of file