API for burst

.gitignore

@@ -49,3 +49,4 @@ dataset/
 # Unit test
 tests/test_bayer_image
 tests/test_utility
+tests/test_burst

CMakeLists.txt

@@ -6,8 +6,8 @@ project(hdrplus)
 # set c++ standard
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -march=native -O3")
-set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -Wall -march=native -O3")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -march=native -O3 -funroll-loops")
+set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS}   -Wall -march=native -O3 -funroll-loops")
 
 # Default build with release
 if(NOT CMAKE_BUILD_TYPE)
@@ -44,7 +44,7 @@ add_library(${PROJECT_NAME} SHARED
   src/hdrplus_pipeline.cpp
   src/merge.cpp )
 
-target_link_libraries(${PROJECT_NAME} PRIVATE
+target_link_libraries(${PROJECT_NAME} PUBLIC
   ${OpenCV_LIBS}
   ${LIBRAW_LIBRARY} )
 
@@ -60,10 +60,15 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/tests )
 
 add_executable( test_bayer_image 
   tests/test_bayer_image.cpp )
-target_link_libraries( test_bayer_image ${PROJECT_NAME} )
+target_link_libraries( test_bayer_image 
+  ${PROJECT_NAME} )
 
 add_executable( test_utility 
   tests/test_utility.cpp )
 target_link_libraries( test_utility 
-  ${OpenCV_LIBS} 
+  ${PROJECT_NAME} )
+
+add_executable( test_burst 
+  tests/test_burst.cpp )
+target_link_libraries( test_burst 
   ${PROJECT_NAME} )

include/hdrplus/align.h

@@ -3,6 +3,7 @@
 #include <vector>
 #include <utility> // std::pair
 #include <opencv2/opencv.hpp> // all opencv header
+#include "hdrplus/burst.h"
 
 namespace hdrplus
 {
@@ -11,9 +12,10 @@ class align
 {
     public:
         align() = default;
-        void process( std::vector<cv::Mat>& bayer_images, \
-                      int reference_frame_idx,\
+        void process( hdrplus::burst& burst_images, \
                       std::vector<std::vector<std::vector<std::pair<int, int>>>>& aligements );
+
+        const std::vector<int> inv_scale_factors = { 1, 2, 2, 4 };
 };
 
 } // namespace hdrplus

include/hdrplus/bayer_image.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <string>
+#include <memory> // std::shared_ptr
 #include <opencv2/opencv.hpp> // all opencv header
 #include <libraw/libraw.h>
 
@@ -11,14 +12,14 @@ class bayer_image
 {
     public:
         explicit bayer_image( const std::string& bayer_image_path );
-        ~bayer_image();
+        ~bayer_image() = default;
 
-        LibRaw libraw_processor;
+        std::shared_ptr<LibRaw> libraw_processor;
         cv::Mat raw_image;
         cv::Mat grayscale_image;
-        size_t width;
-        size_t height;
-        size_t white_level;
+        int width;
+        int height;
+        int white_level;
 };
 
 } // namespace hdrplus

include/hdrplus/burst.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <vector>
+#include <string>
 #include <opencv2/opencv.hpp> // all opencv header
 #include "hdrplus/bayer_image.h"
 
@@ -9,7 +10,18 @@ namespace hdrplus
 
 class burst
 {
+    public:
+        explicit burst( const std::string& burst_path, const std::string& reference_image_path );
+        ~burst() = default;
+
+        std::string reference_image_path;
+        std::string burst_path;
+        int reference_image_idx;
+        std::vector<std::string> bayer_image_paths;
+
         std::vector<hdrplus::bayer_image> bayer_images;
+        std::vector<cv::Mat> grayscale_images_pad;
+
 };
 
 } // namespace hdrplus

include/hdrplus/utility.h

@@ -9,7 +9,7 @@ namespace hdrplus
 {
 
 template <typename T>
-cv::Mat box_filter_2x2( cv::Mat src_image )
+cv::Mat box_filter_2x2( const cv::Mat& src_image )
 {
     // https://stackoverflow.com/questions/34042112/opencv-mat-data-member-access
     const T* src_image_ptr = (T*)src_image.data;
@@ -24,8 +24,6 @@ cv::Mat box_filter_2x2( cv::Mat src_image )
 
     cv::Mat dst_image( src_height / 2, src_width / 2, src_image.type() );
     T* dst_image_ptr = (T*)dst_image.data;
-    //int dst_height = dst_image.size().height;
-    //int dst_width  = dst_image.size().width;
     int dst_step = dst_image.step1();
 
     // -03 should be enough to optimize below code
@@ -46,4 +44,81 @@ cv::Mat box_filter_2x2( cv::Mat src_image )
     return dst_image;
 }
 
+template <typename T, int kernel>
+cv::Mat box_filter_kxk( cv::Mat src_image )
+{
+    const T* src_image_ptr = (T*)src_image.data;
+    int src_height = src_image.size().height;
+    int src_width  = src_image.size().width;
+    int src_step   = src_image.step1();
+
+    if ( src_height % kernel != 0 || src_width % kernel != 0 )
+    {
+        throw std::runtime_error( std::string( __FILE__ ) + "::" + __func__ + " source image need to have size multiplier of kernel\n" );
+    }
+
+    cv::Mat dst_image( src_height / kernel, src_width / kernel, src_image.type() );
+    T* dst_image_ptr = (T*)dst_image.data;
+    int dst_step = dst_image.step1();
+
+    // -03 should be enough to optimize below code
+    for ( int row_i = 0; row_i < src_height; row_i += kernel )
+    {
+        for ( int col_i = 0; col_i < src_width; col_i += kernel )
+        {
+            T box_sum = T(0);
+
+            #pragma GCC unroll kernel
+            for ( int kernel_row_i = 0; kernel_row_i < kernel; ++kernel_row_i )
+            {
+                #pragma GCC unroll kernel
+                for ( int kernel_col_i = 0; kernel_col_i < kernel; ++kernel_col_i )
+                {
+                    box_sum += src_image_ptr[ ( row_i + kernel_row_i ) * src_step + ( col_i + kernel_col_i ) ];
+                }
+            }
+
+            T box_avg = ( box_sum + T( kernel * kernel - 1 ) ) / ( kernel * kernel ); // take ceiling
+            dst_image_ptr[ ( row_i / kernel ) * dst_step + ( col_i / kernel ) ] = box_avg;
+        }
+    }
+
+    // cv::Mat internally use reference count. Will not copy by value here
+    return dst_image;
+}
+
+
+template <typename T, int kernel>
+cv::Mat downsample_nearest_neighbour( cv::Mat src_image )
+{
+    const T* src_image_ptr = (T*)src_image.data;
+    int src_height = src_image.size().height;
+    int src_width  = src_image.size().width;
+    int src_step   = src_image.step1();
+
+    if ( src_height % kernel != 0 || src_width % kernel != 0 )
+    {
+        throw std::runtime_error( std::string( __FILE__ ) + "::" + __func__ + " source image need to have size multiplier of kernel size\n" );
+    }
+
+    cv::Mat dst_image( src_height / kernel, src_width / kernel, src_image.type() );
+    T* dst_image_ptr = (T*)dst_image.data;
+    int dst_step = dst_image.step1();
+
+    // -03 should be enough to optimize below code
+    for ( int row_i = 0; row_i < src_height; row_i += kernel )
+    {
+        for ( int col_i = 0; col_i < src_width; col_i += kernel )
+        {
+            dst_image_ptr[ ( row_i / kernel ) * dst_step + ( col_i / kernel ) ] = \
+                src_image_ptr[ row_i * src_step + col_i ];
+        }
+    }
+
+    // cv::Mat internally use reference count. Will not copy by value here
+    return dst_image;
+}
+
+cv::Mat gaussian_blur( cv::Mat src_image, double sigma );
+
 } // namespace hdrplus

src/bayer_image.cpp

@@ -1,5 +1,6 @@
 #include <string>
 #include <cstdio>
+#include <memory> // std::shared_ptr
 #include <stdexcept> // std::runtime_error
 #include <opencv2/opencv.hpp> // all opencv header
 #include <libraw/libraw.h>
@@ -10,42 +11,40 @@ namespace hdrplus
 
 bayer_image::bayer_image( const std::string& bayer_image_path )
 {
+    libraw_processor = std::make_shared<LibRaw>();
+
     // Open RAW image file
     int return_code;
-    if ( ( return_code = libraw_processor.open_file( bayer_image_path.c_str() ) ) != LIBRAW_SUCCESS )
+    if ( ( return_code = libraw_processor->open_file( bayer_image_path.c_str() ) ) != LIBRAW_SUCCESS )
     {
-        libraw_processor.recycle();
+        libraw_processor->recycle();
         throw std::runtime_error("Error opening file " + bayer_image_path + libraw_strerror( return_code ));
     }
 
     // Unpack the raw image
-    if ( ( return_code = libraw_processor.unpack() ) != LIBRAW_SUCCESS )
+    if ( ( return_code = libraw_processor->unpack() ) != LIBRAW_SUCCESS )
     {
         throw std::runtime_error("Error unpack file " + bayer_image_path + libraw_strerror( return_code ));
     }
 
     // Get image basic info
-    width = size_t( libraw_processor.imgdata.rawdata.sizes.raw_width );
-    height = size_t( libraw_processor.imgdata.rawdata.sizes.raw_height );
-    white_level = size_t( libraw_processor.imgdata.rawdata.color.maximum );
+    width = int( libraw_processor->imgdata.rawdata.sizes.raw_width );
+    height = int( libraw_processor->imgdata.rawdata.sizes.raw_height );
+    white_level = int( libraw_processor->imgdata.rawdata.color.maximum );
 
     #ifndef NDEBUG
-    printf("%s::%s read bayer image %d with width %zu height %zu", \
+    printf("%s::%s read bayer image %s with width %zu height %zu\n", \
         __FILE__, __func__, bayer_image_path.c_str(), width, height );
+    fflush( stdout );
     #endif
 
     // Create CV mat
     // https://answers.opencv.org/question/105972/de-bayering-a-cr2-image/
     // https://www.libraw.org/node/2141
-    raw_image = cv::Mat( width, height, CV_16U, libraw_processor.imgdata.rawdata.raw_image );
+    raw_image = cv::Mat( width, height, CV_16U, libraw_processor->imgdata.rawdata.raw_image ).clone();
 
     // 2x2 box filter
     grayscale_image = box_filter_2x2<uint16_t>( raw_image );
 }
 
-bayer_image::~bayer_image()
-{
-    libraw_processor.recycle();
-}
-
 }

src/burst.cpp

@@ -1,7 +1,74 @@
+#include <cstdio>
+#include <string>
 #include <opencv2/opencv.hpp> // all opencv header
 #include "hdrplus/burst.h"
 
 namespace hdrplus
 {
 
+burst::burst( const std::string& burst_path, const std::string& reference_image_path ) 
+    : reference_image_path( reference_image_path ), burst_path( burst_path )
+{
+    // Search through the input path directory to get all input image path
+    cv::glob( burst_path + "/*.dng", bayer_image_paths, false );
+
+    // Find reference image path in input directory
+    // reference image path need to be absolute path
+    reference_image_idx = -1;
+    for ( int i = 0; i < bayer_image_paths.size(); ++i )
+    {
+        if ( bayer_image_paths[ i ] == reference_image_path )
+        {
+            reference_image_idx = i;
+        }
+    }
+
+    if ( reference_image_idx == -1 )
+    {
+        throw std::runtime_error("Error unable to locate reference image " + reference_image_path );
+    }
+
+    #ifndef NDEBUG
+    for ( const auto& bayer_image_path_i : bayer_image_paths )
+    {
+        printf("%s::%s Find image %s\n", \
+            __FILE__, __func__, bayer_image_path_i.c_str());
+    }
+
+    printf("%s::%s reference image idx %d\n", \
+        __FILE__, __func__, reference_image_idx );
+    #endif
+
+    // Get source bayer image
+    // Downsample original bayer image by 2x2 box filter
+    for ( const auto& bayer_image_path_i : bayer_image_paths )
+    {
+        bayer_images.emplace_back( bayer_image_path_i );
+    }
+
+    // Pad gray scale image
+    int pad_height = bayer_images[ 0 ].height % 32;
+    int pad_width  = bayer_images[ 0 ].width  % 32;
+    // https://docs.opencv.org/3.4/dc/da3/tutorial_copyMakeBorder.html
+    for ( const auto& bayer_image_i : bayer_images )
+    {
+        cv::Mat grayscale_image_pad_i;
+        cv::copyMakeBorder( bayer_image_i.grayscale_image, \
+                            grayscale_image_pad_i, \
+                            0, pad_height, 0, pad_width, \
+                            cv::BORDER_REFLECT );
+        // cv::Mat use internal reference count
+        grayscale_images_pad.emplace_back( grayscale_image_pad_i );
+    } 
+
+    #ifndef NDEBUG
+    printf("%s::%s Pad image from (%d, %d) -> (%d, %d)\n", \
+        __FILE__, __func__, \
+        bayer_images[ 0 ].grayscale_image.size().height, \
+        bayer_images[ 0 ].grayscale_image.size().width, \
+        grayscale_images_pad[ 0 ].size().height, \
+        grayscale_images_pad[ 0 ].size().width );
+    #endif
+}
+
 } // namespace hdrplus

tests/test_bayer_image.cpp

@@ -11,5 +11,11 @@ int main( int argc, char** argv )
 
     hdrplus::bayer_image raw_bayer_image( argv[1] );
 
-    printf("Image of shape h=%d w=%d\n", raw_bayer_image.height, raw_bayer_image.width );
+    printf("Raw image of shape h=%d w=%d\n", \
+        raw_bayer_image.raw_image.size().height, \
+        raw_bayer_image.raw_image.size().width );
+
+    printf("Gray image of shape h=%d, w=%d\n", \
+        raw_bayer_image.grayscale_image.size().height, \
+        raw_bayer_image.grayscale_image.size().width );
 }

tests/test_burst.cpp

@@ -0,0 +1,21 @@
+#include <cstdio>
+#include <string>
+#include "hdrplus/burst.h"
+
+int main( int argc, char** argv )
+{
+    if ( argc != 3 )
+    {
+        printf("Usage: ./test_burst BURST_FOLDER_PATH(no / at end) REFERENCE_IMAGE_PATH\n");
+        exit(1);
+    }
+
+    hdrplus::burst burst_image( argv[1], argv[2] );
+    printf("number of image in burst %d\n", burst_image.bayer_images.size() );
+    printf("grayscale image shape (h=%d, w=%d)\n", \
+        burst_image.bayer_images[ 0 ].grayscale_image.size().height,
+        burst_image.bayer_images[ 0 ].grayscale_image.size().width );
+    printf("grayscale image pad shape (h=%d, w=%d)\n", 
+        burst_image.grayscale_images_pad[ 0 ].size().height, \
+        burst_image.grayscale_images_pad[ 0 ].size().width );
+}

tests/test_utility.cpp

@@ -21,6 +21,7 @@ void print_cvmat( cv::Mat image )
 
 void test_box_filter_2x2()
 {
+    printf("\n###Test test_box_filter_2x2()###\n");
     // Intialize input data
     int src_width = 10;
     int src_height = 6;
@@ -43,7 +44,39 @@ void test_box_filter_2x2()
     print_cvmat<uint16_t>( dst_image );
 }
 
+void test_box_filter_kxk()
+{
+    printf("\n###Test test_box_filter_kxk()###\n");
+    // Intialize input data
+    int src_width = 12;
+    int src_height = 8;
+    std::vector<uint16_t> src_data( src_width, src_height );
+
+    for ( int i = 0; i < src_width * src_height; ++i )
+    {
+        src_data[ i ] = i+1;
+    }
+
+    // Create input cv::mat
+    cv::Mat src_image( src_height, src_width, CV_16U, src_data.data() );
+
+    printf("src cv::Mat is \n");
+    print_cvmat<uint16_t>( src_image );
+
+    cv::Mat dst_image = hdrplus::box_filter_kxk<uint16_t, 2>( src_image );
+
+    printf("dst cv::Mat 2x2 is \n");
+    print_cvmat<uint16_t>( dst_image );
+
+    dst_image = hdrplus::box_filter_kxk<uint16_t, 4>( src_image );
+
+    printf("dst cv::Mat 4x4 is \n");
+    print_cvmat<uint16_t>( dst_image );
+}
+
+
 int main()
 {
     test_box_filter_2x2();
+    test_box_filter_kxk();
 }