HDRplus/src/align.cpp

#include <vector>
#include <string>
#include <limits>
#include <cstdio>
#include <utility> // std::make_pair
#include <stdexcept> // std::runtime_error
#include <opencv2/opencv.hpp> // all opencv header
#include "hdrplus/align.h"
#include "hdrplus/burst.h"
#include "hdrplus/utility.h"

namespace hdrplus
{


// static function only visible within file
static void build_per_grayimg_pyramid( \
    std::vector<cv::Mat>& images_pyramid, \
    const cv::Mat& src_image, \
    const std::vector<int>& inv_scale_factors )
{
    #ifndef NDEBUG
    printf("%s::%s build_per_grayimg_pyramid start with scale factor : ", __FILE__, __func__ );
    for ( int i = 0; i < inv_scale_factors.size(); ++i )
    {
        printf("%d ", inv_scale_factors.at( i ));
    }
    printf("\n");
    #endif

    images_pyramid.resize( inv_scale_factors.size() );

    for ( int i = 0; i < inv_scale_factors.size(); ++i )
    {
        cv::Mat blur_image;
        cv::Mat downsample_image;

        switch ( inv_scale_factors[ i ] )
        {
        case 1:
            images_pyramid[ i ] = src_image.clone();
            // cv::Mat use reference count, will not create deep copy
            downsample_image = src_image;
            break;
        case 2:
            // Gaussian blur
            cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 );
        
            // Downsample
            downsample_image = downsample_nearest_neighbour<uint16_t, 2>( blur_image );

            // Add
            images_pyramid.at( i ) = downsample_image.clone();

            break;
        case 4:
            cv::GaussianBlur( images_pyramid.at( i-1 ), blur_image, cv::Size(0, 0), inv_scale_factors[ i ] / 2 );
            downsample_image = downsample_nearest_neighbour<uint16_t, 4>( blur_image );
            images_pyramid.at( i ) = downsample_image.clone();
            break;
        default:
            throw std::runtime_error("inv scale factor " + std::to_string( inv_scale_factors[ i ]) + "invalid" );
        } 
    }
}


template< int stride >
static void upsample_alignment_stride( \
    std::vector<std::vector<std::pair<int, int>>>& src_alignment, \
    std::vector<std::vector<std::pair<int, int>>>& dst_alignment )
{
    int src_height = src_alignment.size();
    int src_width = src_alignment[ 0 ].size();

    int dst_height = src_height * stride;
    int dst_width = src_width * stride;

    // Allocate data for dst_alignment
    dst_alignment.resize( dst_height, std::vector<std::pair<int, int>>( dst_width ) );

    // Upsample alignment
    for ( int row_i = 0; row_i < src_height; row_i++ )
    {
        for ( int col_i = 0; col_i < src_width; col_i++ )
        {
            // Scale alignment
            std::pair<int, int> align_i = src_alignment[ row_i ][ col_i ];
            align_i.first *= stride;
            align_i.second *= stride;

            // repeat
            for ( int stride_row_i = 0; stride_row_i < stride; ++stride_row_i )
            {
                for ( int stride_col_i = 0; stride_col_i < stride; ++stride_col_i )
                {
                    dst_alignment[ row_i + stride_row_i ][ col_i + stride_col_i ] = align_i;
                }
            }
        }
    }
}


// Set tilesize as template argument for better compiler optimization result.
template< typename data_type, typename return_type, int tile_size >
static unsigned long long l1_distance( const cv::Mat& img1, const cv::Mat& img2, \
    int img1_tile_row_start_idx, int img1_tile_col_start_idx, \
    int img2_tile_row_start_idx, int img2_tile_col_start_idx )
{
    #define CUSTOME_ABS( x ) ( x ) > 0 ? ( x ) : - ( x )

    const data_type* img1_ptr = (const data_type*)img1.data;
    const data_type* img2_ptr = (const data_type*)img2.data;

    int img1_step = img1.step1();
    int img2_step = img2.step1();

    int img1_width = img1.size().width;
    int img1_height = img1.size().height;

    int img2_width = img2.size().width;
    int img2_height = img2.size().height;

    // Range check for safety
    if ( img1_tile_row_start_idx < 0 || img1_tile_row_start_idx > img1_height - tile_size )
    {
        throw std::runtime_error("l1 distance img1_tile_row_start_idx out of valid range\n");
    }

    if ( img1_tile_col_start_idx < 0 || img1_tile_col_start_idx > img1_width - tile_size )
    {
        throw std::runtime_error("l1 distance img1_tile_col_start_idx out of valid range\n");
    }

    if ( img2_tile_row_start_idx < 0 || img2_tile_row_start_idx > img2_height - tile_size )
    {
        throw std::runtime_error("l1 distance img2_tile_row_start_idx out of valid range\n");
    }

    if ( img2_tile_col_start_idx < 0 || img2_tile_col_start_idx > img2_width - tile_size )
    {
        throw std::runtime_error("l1 distance img2_tile_col_start_idx out of valid range\n");
    }

    return_type sum(0);
    // TODO: add pragma unroll here
    for ( int row_i = 0; row_i < tile_size; ++row_i )
    {
        const T* img1_ptr_row_i = img1_ptr + img1_tile_row_start_idx * img1_step + img1_tile_col_start_idx;
        const T* img2_ptr_row_i = img2_ptr + img2_tile_row_start_idx * img2_step + img2_tile_col_start_idx;

        for ( int col_i = 0; col_i < tile_size; ++col_i )
        {
            data_type l1 = CUSTOME_ABS( img1_ptr_row_i[ col_i ] - img2_ptr_row_i[ col_i ] );
            sum += l1;
        }
    }

    #undef CUSTOME_ABS

    return sum;
}


template< typename data_type, typename return_type, int tile_size >
static return_type l2_distance( const cv::Mat& img1, const cv::Mat& img2, \
    int img1_tile_row_start_idx, int img1_tile_col_start_idx, \
    int img2_tile_row_start_idx, int img2_tile_col_start_idx )
{
    #define CUSTOME_ABS( x ) ( x ) > 0 ? ( x ) : - ( x )

    const data_type* img1_ptr = (const data_type*)img1.data;
    const data_type* img2_ptr = (const data_type*)img2.data;

    int img1_step = img1.step1();
    int img2_step = img2.step1();

    int img1_width = img1.size().width;
    int img1_height = img1.size().height;

    int img2_width = img2.size().width;
    int img2_height = img2.size().height;

    // Range check for safety
    if ( img1_tile_row_start_idx < 0 || img1_tile_row_start_idx > img1_height - tile_size )
    {
        throw std::runtime_error("l1 distance img1_tile_row_start_idx out of valid range\n");
    }

    if ( img1_tile_col_start_idx < 0 || img1_tile_col_start_idx > img1_width - tile_size )
    {
        throw std::runtime_error("l1 distance img1_tile_col_start_idx out of valid range\n");
    }

    if ( img2_tile_row_start_idx < 0 || img2_tile_row_start_idx > img2_height - tile_size )
    {
        throw std::runtime_error("l1 distance img2_tile_row_start_idx out of valid range\n");
    }

    if ( img2_tile_col_start_idx < 0 || img2_tile_col_start_idx > img2_width - tile_size )
    {
        throw std::runtime_error("l1 distance img2_tile_col_start_idx out of valid range\n");
    }

    return_type sum(0);
    // TODO: add pragma unroll here
    for ( int row_i = 0; row_i < tile_size; ++row_i )
    {
        const data_type* img1_ptr_row_i = img1_ptr + img1_tile_row_start_idx * img1_step + img1_tile_col_start_idx;
        const data_type* img2_ptr_row_i = img2_ptr + img2_tile_row_start_idx * img2_step + img2_tile_col_start_idx;

        for ( int col_i = 0; col_i < tile_size; ++col_i )
        {
            data_type l1 = CUSTOME_ABS( img1_ptr_row_i[ col_i ] - img2_ptr_row_i[ col_i ] );
            sum += ( l1 * l1 );
        }
    }

    #undef CUSTOME_ABS

    return sum;
}


template <typename T>
void print_tile( const cv::Mat& img, int tile_size, int start_idx_row, int start_idx_col )
{
    const T* img_ptr = (T*)img.data;
    int src_height = img.size().height;
    int src_width  = img.size().width;
    int src_step   = img.step1();

    for ( int row = start_idx_row; row < tile_size + start_idx_row; ++row )
    {
        const T* img_ptr_row = img_ptr + row * src_step;
        for ( int col = start_idx_col; col < tile_size + start_idx_col; ++col )
        {
            printf("%u ", img_ptr_row[ col ] );
        }
        printf("\n");
    }
    printf("\n");
}


template< typename T>
void print_img( const cv::Mat& img, int img_height = -1, int img_width = -1 )
{
    const T* img_ptr = (T*)img.data;
    if ( img_height == -1 && img_width == -1 )
    {
        img_height = img.size().height;
        img_width = img.size().width;
    }
    else
    {
        img_height = std::min( img.size().height, img_height );
        img_width = std::min( img.size().width, img_width );
    }
    printf("Image size (h=%d, w=%d), Print range (h=0-%d, w=0-%d)]\n", \
        img.size().height, img.size().width, img_height, img_width );

    int img_step = img.step1();

    for ( int row = 0; row < img_height; ++row )
    {
        const T* img_ptr_row = img_ptr + row * img_step;
        for ( int col = 0; col < img_width; ++col )
        {
            printf("%u ", img_ptr_row[ col ]);
        }
        printf("\n");
    }
    printf("\n");
}


void align_image_level( \
    const cv::Mat& ref_img, \
    const cv::Mat& alt_img, \
    std::vector<std::vector<std::pair<int, int>>>& prev_aligement, \
    std::vector<std::vector<std::pair<int, int>>>& curr_alignment, \
    int scale_factor_prev_curr, \
    int curr_tile_size, \
    int prev_tile_size, \
    int search_radiou, \
    int distance_type )
{
    /* Basic infos */
    int num_tiles_h = ref_img.size().height / (curr_tile_size / 2) - 1;
    int num_tiles_w = ref_img.size().width / (curr_tile_size / 2 ) - 1 ;

    // Every align image level share the same distance function. 
    // Use function ptr to reduce if else overhead inside for loop
    unsigned long long (*distance_func_ptr)(const cv::Mat&, const cv::Mat&, int, int, int, int) = nullptr;

    if ( distance_type == 1 ) // l1 distance
    {
        if ( curr_tile_size == 8 )
        {
            distance_func_ptr = &l1_distance<uint16_t, unsigned long long, 8>;
        }
        else if ( curr_tile_size == 16 )
        {
            distance_func_ptr = &l1_distance<uint16_t, unsigned long long, 16>;
        }
    }
    else if ( distance_type == 2 ) // l2 distance
    {
        if ( curr_tile_size == 8 )
        {
            distance_func_ptr = &l2_distance<uint16_t, unsigned long long, 8>;
        }
        else if ( curr_tile_size == 16 )
        {
            distance_func_ptr = &l2_distance<uint16_t, unsigned long long, 16>;
        }
    }

    #ifndef NDEBUG
    printf("%s::%s start: \n", __FILE__, __func__ );
    printf("    scale_factor_prev_curr %d, tile_size %d, prev_tile_size %d, search_radiou %d, distance L%d, \n", \
        scale_factor_prev_curr, curr_tile_size, prev_tile_size, search_radiou, distance_type );
    printf("    ref img size h=%d w=%d, alt img size h=%d w=%d, \n", \
        ref_img.size().height, ref_img.size().width, alt_img.size().height, alt_img.size().width );
    printf("    num tile h %d, num tile w %d\n", num_tiles_h, num_tiles_w);
    #endif

    printf("Reference image h=%d, w=%d: \n", ref_img.size().height, ref_img.size().width );
    print_img<uint16_t>( ref_img );

    /* Upsample pervious layer alignment */
    std::vector<std::vector<std::pair<int, int>>> upsampled_prev_aligement;

    // Coarsest level
    // prev_alignment is invalid / empty, construct alignment as (0,0)
    if ( prev_tile_size == -1 )
    {
        printf("create empty prev alignment\n");
        upsampled_prev_aligement.resize( num_tiles_h, std::vector<std::pair<int, int>>( num_tiles_w, std::pair<int, int>(0, 0) ) );
    }
    // Upsample previous level alignment 
    else
    {
        if ( scale_factor_prev_curr == 2 )
        {
            // TODO: add choose from 3 neighbour
            upsample_alignment_stride<2>( prev_aligement, upsampled_prev_aligement );
        }
        else if ( scale_factor_prev_curr == 4 )
        {
            // TODO: add choose from 3 neighbour
            upsample_alignment_stride<4>( prev_aligement, upsampled_prev_aligement );
        }
        else
        {
            throw std::runtime_error("Invalid scale factor" + std::to_string( scale_factor_prev_curr ) );
        }
    }

    // allocate memory for current alignmenr
    curr_alignment.resize( num_tiles_h, std::vector<std::pair<int, int>>( num_tiles_w, std::pair<int, int>(0, 0) ) );

    /* Pad alternative image */
    cv::Mat alt_img_pad;
    cv::copyMakeBorder( alt_img, \
                        alt_img_pad, \
                        search_radiou, search_radiou, search_radiou, search_radiou, \
                        cv::BORDER_CONSTANT, cv::Scalar( UINT_LEAST16_MAX ) );

    printf("Alter image pad h=%d, w=%d: \n", alt_img_pad.size().height, alt_img_pad.size().width );
    print_img<uint16_t>( alt_img_pad );

    printf("!! enlarged tile size %d\n", curr_tile_size + 2 * search_radiou );

    int alt_tile_row_idx_max = alt_img_pad.size().height - ( curr_tile_size + 2 * search_radiou );
    int alt_tile_col_idx_max = alt_img_pad.size().width  - ( curr_tile_size + 2 * search_radiou );

    std::vector<std::vector<uint16_t>> distances( num_tiles_h, std::vector<uint16_t>( num_tiles_w, 0 ));

    /* Iterate through all reference tile & compute distance */
    for ( int ref_tile_row_i = 0; ref_tile_row_i < num_tiles_h; ref_tile_row_i++ )
    {
        for ( int ref_tile_col_i = 0; ref_tile_col_i < num_tiles_w; ref_tile_col_i++ )
        {
            // Upper left index of reference tile
            int ref_tile_row_start_idx_i = ref_tile_row_i * curr_tile_size / 2;
            int ref_tile_col_start_idx_i = ref_tile_col_i * curr_tile_size / 2;

            printf("\nRef img tile [%d, %d] -> start idx [%d, %d] (row, col)\n", \
                ref_tile_row_i, ref_tile_col_i, ref_tile_row_start_idx_i, ref_tile_col_start_idx_i );

            print_tile<uint16_t>( ref_img, 8, ref_tile_row_start_idx_i, ref_tile_col_start_idx_i );

            // Upsampled alignment at this tile
            int prev_alignment_row_i = upsampled_prev_aligement.at( ref_tile_row_i ).at( ref_tile_col_i ).first;
            int prev_alignment_col_i = upsampled_prev_aligement.at( ref_tile_row_i ).at( ref_tile_col_i ).second;

            // Alternative image tile start idx
            int alt_tile_row_start_idx_i = ref_tile_row_start_idx_i + prev_alignment_row_i;
            int alt_tile_col_start_idx_i = ref_tile_col_start_idx_i + prev_alignment_col_i;

            // Ensure alternative image tile within range
            if ( alt_tile_row_start_idx_i < 0 )
                alt_tile_row_start_idx_i = 0;
            if ( alt_tile_col_start_idx_i < 0 )
                alt_tile_col_start_idx_i = 0;
            if ( alt_tile_row_start_idx_i > alt_tile_row_idx_max )
            {
                int before = alt_tile_row_start_idx_i;
                alt_tile_row_start_idx_i = alt_tile_row_idx_max;
                printf("@@ change start x from %d to %d\n", before, alt_tile_row_idx_max);
            }
            if ( alt_tile_col_start_idx_i > alt_tile_col_idx_max )
            {
                int before = alt_tile_col_start_idx_i;
                alt_tile_col_start_idx_i = alt_tile_col_idx_max;
                printf("@@ change start y from %d to %d\n", before, alt_tile_col_idx_max );
            }

            // Because alternative image is padded with search radious. 
            // Using same coordinate with reference image will automatically considered search radious * 2
            printf("Alt image tile [%d, %d]-> start idx [%d, %d]\n", 
                ref_tile_row_i, ref_tile_col_i, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i );
            print_tile<uint16_t>( alt_img_pad, 16, alt_tile_row_start_idx_i, alt_tile_col_start_idx_i );

            // Search based on L1/L2 distance
            unsigned long long min_distance_i = ULONG_LONG_MAX;
            int min_distance_row_i = -1;
            int min_distance_col_i = -1;
            for ( int search_row_j = 0; search_row_j < ( search_radiou * 2 + 1 ); search_row_j++ )
            {
                for ( int search_col_j = 0; search_col_j < ( search_radiou * 2 + 1 ); search_col_j++ )
                {
                    unsigned long long distance_j = distance_func_ptr( ref_img, alt_img_pad, \
                        ref_tile_row_start_idx_i, ref_tile_col_start_idx_i, \
                        alt_tile_row_start_idx_i + search_row_j, alt_tile_col_start_idx_i + search_col_j );

                    printf("tile at (%d, %d) search (%d, %d), new dis %llu, old dis %llu\n", \
                        ref_tile_row_i, ref_tile_col_i, search_row_j, search_col_j, distance_j, min_distance_i );

                    // If this is smaller distance
                    if ( distance_j < min_distance_i )
                    {
                        min_distance_i = distance_j;
                        min_distance_col_i = search_col_j;
                        min_distance_row_i = search_row_j;
                    }
                }
            }

            printf("tile at (%d, %d) alignment (%d, %d)\n", \
                ref_tile_row_i, ref_tile_col_i, min_distance_row_i, min_distance_col_i );

            int alignment_row_i = prev_alignment_row_i + min_distance_row_i - search_radiou;
            int alignment_col_i = prev_alignment_col_i + min_distance_col_i - search_radiou;

            std::pair<int, int> alignment_i( alignment_row_i, alignment_col_i );

            // Add min_distance_i's corresbonding idx as min
            curr_alignment.at( ref_tile_row_i ).at( ref_tile_col_i ) = alignment_i;
            distances.at( ref_tile_row_i ).at( ref_tile_col_i ) = min_distance_i;
        }
    }

    printf("Min distance for each tile \n");
    for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ )
    {
        for ( int tile_col = 0; tile_col < num_tiles_w; ++tile_col )
        {
            printf("tile (%d, %d) distance %u\n", \
                tile_row, tile_col, distances.at( tile_row).at(tile_col ) );
        }
    }


    printf("Alignment at current level\n");
    for ( int tile_row = 0; tile_row < num_tiles_h; tile_row++ )
    {
        for ( int tile_col = 0; tile_col < num_tiles_w; tile_col++ )
        {
            const auto tile_start = curr_alignment.at( tile_row ).at( tile_col );
            printf("tile (%d, %d) -> start idx (%d, %d)\n", \
                tile_row, tile_col, tile_start.first, tile_start.second);
        }
    }

}


static void build_per_pyramid_reftiles_start( \
    std::vector<std::vector<std::vector<std::pair<int, int>>>>& per_pyramid_reftiles_start, \
    const std::vector<std::vector<cv::Mat>>& per_grayimg_pyramid, \
    const std::vector<int>& grayimg_tile_sizes )
{
    per_pyramid_reftiles_start.resize( per_grayimg_pyramid.at(0).size() );

    // Every image pyramid level
    for ( int level_i = 0; level_i < per_grayimg_pyramid.at(0).size(); level_i++ )
    {
        int level_i_img_h = per_grayimg_pyramid.at(0).at( level_i ).size().height;
        int level_i_img_w = per_grayimg_pyramid.at(0).at( level_i ).size().width;

        int level_i_tile_size = grayimg_tile_sizes.at( level_i );

        int num_tiles_h = level_i_img_h / (level_i_tile_size / 2) - 1;
        int num_tiles_w = level_i_img_w / (level_i_tile_size / 2) - 1;

        // Allocate memory
        per_pyramid_reftiles_start.at( level_i ).resize( num_tiles_h, std::vector<std::pair<int, int>>( num_tiles_w ) );

        for ( int tile_col_i = 0; tile_col_i < num_tiles_h; tile_col_i++ )
        {
            for ( int tile_row_j = 0; tile_row_j < num_tiles_w; tile_row_j++ )
            {
                per_pyramid_reftiles_start.at( level_i ).at( tile_col_i ).at( tile_row_j ) \
                    = std::make_pair<int, int>( tile_col_i * level_i_tile_size, tile_row_j * level_i_tile_size );
            }
        }
    }
}


void align::process( const hdrplus::burst& burst_images, \
                     std::vector<std::vector<std::vector<std::pair<int, int>>>>& images_alignment )
{
    #ifndef NDEBUG
    printf("%s::%s align::process start\n", __FILE__, __func__ ); fflush(stdout);
    #endif

    // image pyramid per image, per pyramid level
    std::vector<std::vector<cv::Mat>> per_grayimg_pyramid;

    per_grayimg_pyramid.resize( burst_images.num_images );
    for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx )
    {
        // per_grayimg_pyramid[ img_idx ][ 0 ] is the original image
        // per_grayimg_pyramid[ img_idx ][ 3 ] is the coarsest image
        build_per_grayimg_pyramid( per_grayimg_pyramid.at( img_idx ), \
                                   burst_images.grayscale_images_pad.at( img_idx ), \
                                   this->inv_scale_factors );
    }

    #ifndef NDEBUG
    printf("%s::%s build image pyramid of size : ", __FILE__, __func__ );
    for ( int level_i = 0; level_i < num_levels; ++level_i )
    {
        printf("(%d, %d) ", per_grayimg_pyramid[ 0 ][ level_i ].size().height,
                            per_grayimg_pyramid[ 0 ][ level_i ].size().width );
    }
    printf("\n"); fflush(stdout);
    #endif

    // for ( int level_i; level_i < num_levels; ++level_i )
    // {
    //     printf("level %d img : \n" , level_i );
    //     print_img<uint16_t>( per_grayimg_pyramid[ burst_images.reference_image_idx ][ level_i], 100, 100 );
    // }

    // Align every image
    const std::vector<cv::Mat>& ref_grayimg_pyramid = per_grayimg_pyramid[ burst_images.reference_image_idx ];
    for ( int img_idx = 0; img_idx < burst_images.num_images; ++img_idx )
    {
        // Do not align with reference image
        if ( img_idx == burst_images.reference_image_idx )
            continue;

        const std::vector<cv::Mat>& alt_grayimg_pyramid = per_grayimg_pyramid[ img_idx ];

        // Align every level from coarse to grain
        // level 0 : finest level, the original image
        // level 3 : coarsest level
        std::vector<std::vector<std::pair<int, int>>> curr_alignment;
        std::vector<std::vector<std::pair<int, int>>> prev_alignment;
        for ( int level_i = num_levels - 1; level_i >= 0; level_i-- )
        {
            align_image_level(
                ref_grayimg_pyramid[ level_i ],    // reference image at current level
                alt_grayimg_pyramid[ level_i ],    // alternative image at current level
                prev_alignment,                    // previous layer alignment
                curr_alignment,                    // current layer alignment
                ( level_i == ( num_levels - 1 ) ? -1 : inv_scale_factors[ level_i ] ),                 // scale factor between previous layer and current layer. -1 if current layer is the coarsest layer
                grayimg_tile_sizes[ level_i ],     // current level tile size
                ( level_i == ( num_levels - 1 ) ? -1 : grayimg_tile_sizes[ level_i + 1 ] ), // previous level tile size
                grayimg_search_radious[ level_i ], // search radious
                distances[ level_i ] );            // L1/L2 distance
           
            // make curr alignment as previous alignment
            prev_alignment.swap( curr_alignment );
            curr_alignment.clear();

            break;
        } // for pyramid level


    } // for alternative image

}

} // namespace hdrplus