@ -43,19 +43,23 @@ static void build_per_grayimg_pyramid( \
downsample_image = src_image ;
break ;
case 2 :
// Gaussian blur
cv : : GaussianBlur ( images_pyramid . at ( i - 1 ) , blur_image , cv : : Size ( 0 , 0 ) , inv_scale_factors [ i ] / 2 ) ;
printf ( " downsample with gaussian sigma %.2f " , inv_scale_factors [ i ] * 0.5 ) ;
// // Gaussian blur
// cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 );
// Downsample
downsample_image = downsample_nearest_neighbour < uint16_t , 2 > ( blur_image ) ;
// // Downsample
// downsample_image = downsample_nearest_neighbour<uint16_t, 2>( blur_image );
downsample_image = downsample_nearest_neighbour < uint16_t , 2 > ( images_pyramid . at ( i - 1 ) ) ;
// Add
images_pyramid . at ( i ) = downsample_image . clone ( ) ;
break ;
case 4 :
cv : : GaussianBlur ( images_pyramid . at ( i - 1 ) , blur_image , cv : : Size ( 0 , 0 ) , inv_scale_factors [ i ] / 2 ) ;
downsample_image = downsample_nearest_neighbour < uint16_t , 4 > ( blur_image ) ;
printf ( " downsample with gaussian sigma %.2f " , inv_scale_factors [ i ] * 0.5 ) ;
// cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] * 0.5 );
// downsample_image = downsample_nearest_neighbour<uint16_t, 4>( blur_image );
downsample_image = downsample_nearest_neighbour < uint16_t , 4 > ( images_pyramid . at ( i - 1 ) ) ;
images_pyramid . at ( i ) = downsample_image . clone ( ) ;
break ;
default :
@ -68,21 +72,29 @@ static void build_per_grayimg_pyramid( \
template < int pyramid_scale_factor_prev_curr , int tilesize_scale_factor_prev_curr >
static void build_upsampled_prev_aligement ( \
std : : vector < std : : vector < std : : pair < int , int > > > & src_alignment , \
std : : vector < std : : vector < std : : pair < int , int > > > & dst_alignment )
std : : vector < std : : vector < std : : pair < int , int > > > & dst_alignment ,
int num_tiles_h , int num_tiles_w )
{
printf ( " build_upsampled_prev_aligement with scale factor %d and tile size factor %d \n " , \
pyramid_scale_factor_prev_curr , tilesize_scale_factor_prev_curr ) ;
int src_height = src_alignment . size ( ) ;
int src_width = src_alignment [ 0 ] . size ( ) ;
int repeat_factor = pyramid_scale_factor_prev_curr / tilesize_scale_factor_prev_curr ;
constexpr int repeat_factor = pyramid_scale_factor_prev_curr / tilesize_scale_factor_prev_curr ;
printf ( " build_upsampled_prev_aligement with scale factor %d, repeat factor %d, tile size factor %d \n " , \
pyramid_scale_factor_prev_curr , repeat_factor , tilesize_scale_factor_prev_curr ) ;
int dst_height = src_height * repeat_factor ;
int dst_width = src_width * repeat_factor ;
if ( dst_height > num_tiles_h | | dst_width > num_tiles_w )
{
throw std : : runtime_error ( " current level number of tiles smaller than upsampled tiles \n " ) ;
}
// Allocate data for dst_alignment
dst_alignment . resize ( dst_height , std : : vector < std : : pair < int , int > > ( dst_width ) ) ;
// NOTE: number of tiles h, number of tiles w might be different from dst_height, dst_width
// For tiles between num_tile_h and dst_height, use (0,0)
dst_alignment . resize ( num_tiles_h , std : : vector < std : : pair < int , int > > ( num_tiles_w , std : : pair < int , int > ( 0 , 0 ) ) ) ;
// Upsample alignment
for ( int row_i = 0 ; row_i < src_height ; row_i + + )
@ -99,7 +111,7 @@ static void build_upsampled_prev_aligement( \
{
for ( int repeat_col_i = 0 ; repeat_col_i < repeat_factor ; + + repeat_col_i )
{
dst_alignment [ row_i + repeat_row_i ] [ col_i + repeat_col_i ] = align_i ;
dst_alignment [ row_i * repeat_factor + repeat_row_i ] [ col_i * repeat_factor + repeat_col_i ] = align_i ;
}
}
}
@ -107,59 +119,6 @@ static void build_upsampled_prev_aligement( \
}
template < typename T >
void print_tile ( const cv : : Mat & img , int tile_size , int start_idx_row , int start_idx_col )
{
const T * img_ptr = ( T * ) img . data ;
int src_height = img . size ( ) . height ;
int src_width = img . size ( ) . width ;
int src_step = img . step1 ( ) ;
for ( int row = start_idx_row ; row < tile_size + start_idx_row ; + + row )
{
const T * img_ptr_row = img_ptr + row * src_step ;
for ( int col = start_idx_col ; col < tile_size + start_idx_col ; + + col )
{
printf ( " %u " , img_ptr_row [ col ] ) ;
}
printf ( " \n " ) ;
}
printf ( " \n " ) ;
}
template < typename T >
void print_img ( const cv : : Mat & img , int img_height = - 1 , int img_width = - 1 )
{
const T * img_ptr = ( T * ) img . data ;
if ( img_height = = - 1 & & img_width = = - 1 )
{
img_height = img . size ( ) . height ;
img_width = img . size ( ) . width ;
}
else
{
img_height = std : : min ( img . size ( ) . height , img_height ) ;
img_width = std : : min ( img . size ( ) . width , img_width ) ;
}
printf ( " Image size (h=%d, w=%d), Print range (h=0-%d, w=0-%d)] \n " , \
img . size ( ) . height , img . size ( ) . width , img_height , img_width ) ;
int img_step = img . step1 ( ) ;
for ( int row = 0 ; row < img_height ; + + row )
{
const T * img_ptr_row = img_ptr + row * img_step ;
for ( int col = 0 ; col < img_width ; + + col )
{
printf ( " %u " , img_ptr_row [ col ] ) ;
}
printf ( " \n " ) ;
}
printf ( " \n " ) ;
}
// Set tilesize as template argument for better compiler optimization result.
template < typename data_type , typename return_type , int tile_size >
static unsigned long long l1_distance ( const cv : : Mat & img1 , const cv : : Mat & img2 , \
@ -261,10 +220,10 @@ static return_type l2_distance( const cv::Mat& img1, const cv::Mat& img2, \
throw std : : runtime_error ( " l1 distance img2_tile_col_start_idx out of valid range \n " ) ;
}
printf ( " Search two tile with ref : \n " ) ;
print_tile < data_type > ( img1 , tile_size , img1_tile_row_start_idx , img1_tile_col_start_idx ) ;
printf ( " Search two tile with alt : \n " ) ;
print_tile < data_type > ( img2 , tile_size , img2_tile_row_start_idx , img2_tile_col_start_idx ) ;
// printf("Search two tile with ref : \n");
// print_tile<data_type>( img1, tile_size, img1_tile_row_start_idx, img1_tile_col_start_idx );
// printf("Search two tile with alt :\n");
// print_tile<data_type>( img2, tile_size, img2_tile_row_start_idx, img2_tile_col_start_idx );
return_type sum ( 0 ) ;
// TODO: add pragma unroll here
@ -325,7 +284,7 @@ void align_image_level( \
}
// Every level share the same upsample function
void ( * upsample_alignment_func_ptr ) ( std : : vector < std : : vector < std : : pair < int , int > > > & , std : : vector < std : : vector < std : : pair < int , int > > > & ) = nullptr ;
void ( * upsample_alignment_func_ptr ) ( std : : vector < std : : vector < std : : pair < int , int > > > & , std : : vector < std : : vector < std : : pair < int , int > > > & , int , int ) = nullptr ;
if ( scale_factor_prev_curr = = 2 )
{
if ( curr_tile_size / prev_tile_size = = 2 )
@ -357,8 +316,8 @@ void align_image_level( \
}
}
int num_tiles_ ref_ h = ref_img . size ( ) . height / ( curr_tile_size / 2 ) - 1 ;
int num_tiles_ ref_ w = ref_img . size ( ) . width / ( curr_tile_size / 2 ) - 1 ;
int num_tiles_ h = ref_img . size ( ) . height / ( curr_tile_size / 2 ) - 1 ;
int num_tiles_ w = ref_img . size ( ) . width / ( curr_tile_size / 2 ) - 1 ;
/* Upsample pervious layer alignment */
std : : vector < std : : vector < std : : pair < int , int > > > upsampled_prev_aligement ;
@ -367,19 +326,26 @@ void align_image_level( \
// prev_alignment is invalid / empty, construct alignment as (0,0)
if ( prev_tile_size = = - 1 )
{
upsampled_prev_aligement . resize ( num_tiles_ ref_ h, \
std : : vector < std : : pair < int , int > > ( num_tiles_ ref_ w, std : : pair < int , int > ( 0 , 0 ) ) ) ;
upsampled_prev_aligement . resize ( num_tiles_ h, \
std : : vector < std : : pair < int , int > > ( num_tiles_ w, std : : pair < int , int > ( 0 , 0 ) ) ) ;
}
// Upsample previous level alignment
else
{
upsample_alignment_func_ptr ( prev_aligement , upsampled_prev_aligement ) ;
}
upsample_alignment_func_ptr ( prev_aligement , upsampled_prev_aligement , num_tiles_h , num_tiles_w ) ;
/* Basic infos */
// NOTE: num_tile_h might be different from num_tile_ref_h
int num_tiles_h = upsampled_prev_aligement . size ( ) ;
int num_tiles_w = upsampled_prev_aligement . at ( 0 ) . size ( ) ;
printf ( " \n !!!!!Upsampled previous alignment \n " ) ;
for ( int tile_row = 0 ; tile_row < upsampled_prev_aligement . size ( ) ; tile_row + + )
{
for ( int tile_col = 0 ; tile_col < upsampled_prev_aligement . at ( 0 ) . size ( ) ; tile_col + + )
{
const auto tile_start = upsampled_prev_aligement . at ( tile_row ) . at ( tile_col ) ;
printf ( " up tile (%d, %d) -> start idx (%d, %d) \n " , \
tile_row , tile_col , tile_start . first , tile_start . second ) ;
}
}
}
# ifndef NDEBUG
printf ( " %s::%s start: \n " , __FILE__ , __func__ ) ;
@ -388,7 +354,6 @@ void align_image_level( \
printf ( " ref img size h=%d w=%d, alt img size h=%d w=%d, \n " , \
ref_img . size ( ) . height , ref_img . size ( ) . width , alt_img . size ( ) . height , alt_img . size ( ) . width ) ;
printf ( " num tile h (upsampled) %d, num tile w (upsampled) %d \n " , num_tiles_h , num_tiles_w ) ;
printf ( " num tile h (gray) %d, num tile w (gray) %d \n " , num_tiles_ref_h , num_tiles_ref_w ) ;
# endif
// allocate memory for current alignmenr
@ -401,13 +366,13 @@ void align_image_level( \
search_radiou , search_radiou , search_radiou , search_radiou , \
cv : : BORDER_CONSTANT , cv : : Scalar ( UINT_LEAST16_MAX ) ) ;
printf ( " Reference image h=%d, w=%d: \n " , ref_img . size ( ) . height , ref_img . size ( ) . width ) ;
print_img < uint16_t > ( ref_img ) ;
// printf("Reference image h=%d, w=%d: \n", ref_img.size().height, ref_img.size().width );
// print_img<uint16_t>( ref_img );
printf ( " Alter image pad h=%d, w=%d: \n " , alt_img_pad . size ( ) . height , alt_img_pad . size ( ) . width ) ;
print_img < uint16_t > ( alt_img_pad ) ;
// printf("Alter image pad h=%d, w=%d: \n", alt_img_pad.size().height, alt_img_pad.size().width );
// print_img<uint16_t>( alt_img_pad );
printf ( " !! enlarged tile size %d \n " , curr_tile_size + 2 * search_radiou ) ;
//printf("!! enlarged tile size %d\n", curr_tile_size + 2 * search_radiou );
int alt_tile_row_idx_max = alt_img_pad . size ( ) . height - ( curr_tile_size + 2 * search_radiou ) ;
int alt_tile_col_idx_max = alt_img_pad . size ( ) . width - ( curr_tile_size + 2 * search_radiou ) ;
@ -424,10 +389,10 @@ void align_image_level( \
int ref_tile_row_start_idx_i = ref_tile_row_i * curr_tile_size / 2 ;
int ref_tile_col_start_idx_i = ref_tile_col_i * curr_tile_size / 2 ;
printf ( " \n Ref img tile [%d, %d] -> start idx [%d, %d] (row, col) \n " , \
/ / printf ( " \n Ref img tile [%d, %d] -> start idx [%d, %d] (row, col) \n " , \
ref_tile_row_i , ref_tile_col_i , ref_tile_row_start_idx_i , ref_tile_col_start_idx_i ) ;
print_tile < uint16_t > ( ref_img , 8 , ref_tile_row_start_idx_i , ref_tile_col_start_idx_i ) ;
// printf("\nRef img tile [%d, %d]\n", ref_tile_row_i, ref_tile_col_i );
// print_tile<uint16_t>( ref_img, curr_tile_size, ref_tile_row_start_idx_i, ref_tile_col_start_idx_i );
// Upsampled alignment at this tile
int prev_alignment_row_i = upsampled_prev_aligement . at ( ref_tile_row_i ) . at ( ref_tile_col_i ) . first ;
@ -457,9 +422,10 @@ void align_image_level( \
// Because alternative image is padded with search radious.
// Using same coordinate with reference image will automatically considered search radious * 2
printf ( " Alt image tile [%d, %d]-> start idx [%d, %d] \n " ,
/ / printf ( " Alt image tile [%d, %d]-> start idx [%d, %d] \n " , \
ref_tile_row_i , ref_tile_col_i , alt_tile_row_start_idx_i , alt_tile_col_start_idx_i ) ;
print_tile < uint16_t > ( alt_img_pad , 16 , alt_tile_row_start_idx_i , alt_tile_col_start_idx_i ) ;
// printf("\nAlt image tile [%d, %d]\n", ref_tile_row_i, ref_tile_col_i );
// print_tile<uint16_t>( alt_img_pad, curr_tile_size + 2 * search_radiou, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i );
// Search based on L1/L2 distance
unsigned long long min_distance_i = ULONG_LONG_MAX ;
@ -469,16 +435,16 @@ void align_image_level( \
{
for ( int search_col_j = 0 ; search_col_j < ( search_radiou * 2 + 1 ) ; search_col_j + + )
{
printf ( " \n --->tile at [%d, %d] search (%d, %d) \n " , \
ref_tile_row_i , ref_tile_col_i , search_row_j , search_col_j ) ;
/ / printf ( " \n --->tile at [%d, %d] search (%d, %d) \n " , \
ref_tile_row_i , ref_tile_col_i , search_row_j - search_radiou , search_col_j - search_radiou ) ;
// TODO: currently distance is incorrect
unsigned long long distance_j = distance_func_ptr ( ref_img , alt_img_pad , \
ref_tile_row_start_idx_i , ref_tile_col_start_idx_i , \
alt_tile_row_start_idx_i + search_row_j , alt_tile_col_start_idx_i + search_col_j ) ;
printf ( " <---tile at [%d, %d] search (%d, %d), new dis %llu, old dis %llu \n " , \
ref_tile_row_i , ref_tile_col_i , search_row_j , search_col_j , distance_j , min_distance_i ) ;
/ / printf ( " <---tile at [%d, %d] search (%d, %d), new dis %llu, old dis %llu \n " , \
ref_tile_row_i , ref_tile_col_i , search_row_j - search_radiou , search_col_j - search_radiou , distance_j , min_distance_i ) ;
// If this is smaller distance
if ( distance_j < min_distance_i )
@ -488,30 +454,30 @@ void align_image_level( \
min_distance_row_i = search_row_j ;
}
// If same value, choose the one closer to the original tile location
if ( distance_j = = min_distance_i & & min_distance_row_i ! = - 1 & & min_distance_col_i ! = - 1 )
{
int prev_distance_row_2_ref = min_distance_row_i - search_radiou ;
int prev_distance_col_2_ref = min_distance_col_i - search_radiou ;
int curr_distance_row_2_ref = search_row_j - search_radiou ;
int curr_distance_col_2_ref = search_col_j - search_radiou ;
int prev_distance_2_ref_sqr = prev_distance_row_2_ref * prev_distance_row_2_ref + prev_distance_col_2_ref * prev_distance_col_2_ref ;
int curr_distance_2_ref_sqr = curr_distance_row_2_ref * curr_distance_row_2_ref + curr_distance_col_2_ref * curr_distance_col_2_ref ;
// previous min distance idx is farther away from ref tile start location
if ( prev_distance_2_ref_sqr > curr_distance_2_ref_sqr )
{
printf ( " @@@ Same distance , choose closer one (%d, %d) instead of (%d, %d)\n " , \
search_row_j , search_col_j , min_distance_row_i , min_distance_col_i ) ;
min_distance_col_i = search_col_j ;
min_distance_row_i = search_row_j ;
}
}
// // If same value, choose the one closer to the original tile location
// if ( distance_j == min_distance_i && min_distance_row_i != -1 && min_distance_col_i != -1 )
// {
// int prev_distance_row_2_ref = min_distance_row_i - search_radiou;
// int prev_distance_col_2_ref = min_distance_col_i - search_radiou;
// int curr_distance_row_2_ref = search_row_j - search_radiou;
// int curr_distance_col_2_ref = search_col_j - search_radiou;
// int prev_distance_2_ref_sqr = prev_distance_row_2_ref * prev_distance_row_2_ref + prev_distance_col_2_ref * prev_distance_col_2_ref;
// int curr_distance_2_ref_sqr = curr_distance_row_2_ref * curr_distance_row_2_ref + curr_distance_col_2_ref * curr_distance_col_2_ref;
// // previous min distance idx is farther away from ref tile start location
// if ( prev_distance_2_ref_sqr > curr_distance_2_ref_sqr )
// {
/ / printf ( " @@@ Same distance %d , choose closer one (%d, %d) instead of (%d, %d)\n " , \
// distance_j, search_row_j, search_col_j, min_distance_row_i, min_distance_col_i);
// min_distance_col_i = search_col_j;
// min_distance_row_i = search_row_j;
// }
// }
}
}
printf ( " tile at (%d, %d) alignment (%d, %d) \n " , \
/ / printf ( " tile at (%d, %d) alignment (%d, %d) \n " , \
ref_tile_row_i , ref_tile_col_i , min_distance_row_i , min_distance_col_i ) ;
int alignment_row_i = prev_alignment_row_i + min_distance_row_i - search_radiou ;
@ -525,7 +491,7 @@ void align_image_level( \
}
}
printf ( " Min distance for each tile \n " ) ;
printf ( " \n !!!!! Min distance for each tile \n " ) ;
for ( int tile_row = 0 ; tile_row < num_tiles_h ; tile_row + + )
{
for ( int tile_col = 0 ; tile_col < num_tiles_w ; + + tile_col )
@ -535,8 +501,7 @@ void align_image_level( \
}
}
printf ( " Alignment at current level \n " ) ;
printf ( " \n !!!!!Alignment at current level \n " ) ;
for ( int tile_row = 0 ; tile_row < num_tiles_h ; tile_row + + )
{
for ( int tile_col = 0 ; tile_col < num_tiles_w ; tile_col + + )
@ -593,6 +558,14 @@ void align::process( const hdrplus::burst& burst_images, \
// image pyramid per image, per pyramid level
std : : vector < std : : vector < cv : : Mat > > per_grayimg_pyramid ;
// printf("!!!!! ref bayer padded\n");
// print_img<uint16_t>( burst_images.bayer_images_pad.at( burst_images.reference_image_idx) );
// exit(1);
// printf("!!!!! ref gray padded\n");
// print_img<uint16_t>( burst_images.grayscale_images_pad.at( burst_images.reference_image_idx) );
// exit(1);
per_grayimg_pyramid . resize ( burst_images . num_images ) ;
for ( int img_idx = 0 ; img_idx < burst_images . num_images ; + + img_idx )
{
@ -604,20 +577,22 @@ void align::process( const hdrplus::burst& burst_images, \
}
# ifndef NDEBUG
printf ( " %s::%s build image pyramid of size : " , __FILE__ , __func__ ) ;
for ( int level_i = 0 ; level_i < num_levels ; + + level_i )
{
printf ( " (%d, %d) " , per_grayimg_pyramid [ 0 ] [ level_i ] . size ( ) . height ,
per_grayimg_pyramid [ 0 ] [ level_i ] . size ( ) . width ) ;
}
printf ( " \n " ) ; fflush ( stdout ) ;
// printf("%s::%s build image pyramid of size : ", __FILE__, __func__ );
// for ( int level_i = 0; level_i < num_levels; ++level_i )
// {
// printf("(%d, %d) ", per_grayimg_pyramid[ 0 ][ level_i ].size().height,
// per_grayimg_pyramid[ 0 ][ level_i ].size().width );
// }
// printf("\n"); fflush(stdout);
# endif
// print image pyramid
// for ( int level_i; level_i < num_levels; ++level_i )
// {
// printf(" level %d img : \n" , level_i );
// print_img<uint16_t>( per_grayimg_pyramid[ burst_images.reference_image_idx ][ level_i ], 100, 100 );
// printf(" \n\n!!!!! ref gray pyramid level %d img : \n" , level_i );
// print_img<uint16_t>( per_grayimg_pyramid[ burst_images.reference_image_idx ][ level_i ] );
// }
// exit(-1);
// Align every image
const std : : vector < cv : : Mat > & ref_grayimg_pyramid = per_grayimg_pyramid [ burst_images . reference_image_idx ] ;
@ -634,7 +609,7 @@ void align::process( const hdrplus::burst& burst_images, \
// level 3 : coarsest level
std : : vector < std : : vector < std : : pair < int , int > > > curr_alignment ;
std : : vector < std : : vector < std : : pair < int , int > > > prev_alignment ;
for ( int level_i = num_levels - 1 ; level_i > = 0 ; level_i - - )
for ( int level_i = num_levels - 1 ; level_i > = 0 ; level_i - - ) // 3,2,1,0
{
printf ( " \n \n ########################align level %d \n " , level_i ) ;
align_image_level (
@ -642,19 +617,22 @@ void align::process( const hdrplus::burst& burst_images, \
alt_grayimg_pyramid [ level_i ] , // alternative image at current level
prev_alignment , // previous layer alignment
curr_alignment , // current layer alignment
( level_i = = ( num_levels - 1 ) ? - 1 : inv_scale_factors [ level_i ] ) , // scale factor between previous layer and current layer. -1 if current layer is the coarsest layer
( level_i = = ( num_levels - 1 ) ? - 1 : inv_scale_factors [ level_i + 1 ] ) , // scale factor between previous layer and current layer. -1 if current layer is the coarsest layer , [-1, 4, 4, 2]
grayimg_tile_sizes [ level_i ] , // current level tile size
( level_i = = ( num_levels - 1 ) ? - 1 : grayimg_tile_sizes [ level_i + 1 ] ) , // previous level tile size
grayimg_search_radious [ level_i ] , // search radious
distances [ level_i ] ) ; // L1/L2 distance
printf ( " @@@Alignment at level %d is h=%d, w=%d " , level_i , curr_alignment . size ( ) , curr_alignment . at ( 0 ) . size ( ) ) ;
// make curr alignment as previous alignment
prev_alignment . swap ( curr_alignment ) ;
curr_alignment . clear ( ) ;
// Stop at second iteration
if ( level_i = = num_levels - 2 )
break ;
// if ( level_i == num_levels - 3 )
// break;
} // for pyramid level