Support brotli compressed boxes in JPEG XL

.github/workflows/on_PR_windows_matrix.yml

@@ -125,6 +125,7 @@ jobs:
             gtest:p
             libiconv:p
             zlib:p
+            brotli:p
             curl:p
 
       - name: Build
@@ -175,6 +176,9 @@ jobs:
             libxslt-devel
             python38-lxml
             zlib-devel
+            libbrotlicommon1
+            libbrotlidec1
+            libbrotli-devel
       - name: Build
         run: |
           cmake -GNinja \

CMakeLists.txt

@@ -28,7 +28,7 @@ option( EXIV2_ENABLE_LENSDATA         "Build including lens data"
 option( EXIV2_ENABLE_DYNAMIC_RUNTIME  "Use dynamic runtime (used for static libs)"            ON  )
 option( EXIV2_ENABLE_WEBREADY         "Build webready support into library"                   OFF )
 option( EXIV2_ENABLE_CURL             "USE Libcurl for HttpIo (WEBREADY)"                     OFF )
-option( EXIV2_ENABLE_BMFF             "Build with BMFF support"                               ON  )
+option( EXIV2_ENABLE_BMFF             "Build with BMFF support (brotli recommended)"          ON  )
 
 option( EXIV2_BUILD_SAMPLES           "Build sample applications"                             OFF )
 option( EXIV2_BUILD_EXIV2_COMMAND     "Build exiv2 command-line executable"                   ON  )

ci/install_dependencies.sh

@@ -30,39 +30,39 @@ distro_id=$(grep '^ID=' /etc/os-release|awk -F = '{print $2}'|sed 's/\"//g')
 
 case "$distro_id" in
     'fedora')
-        dnf -y --refresh install gcc-c++ clang cmake make expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which dos2unix glibc-langpack-en diffutils
+        dnf -y --refresh install gcc-c++ clang cmake make expat-devel zlib-devel brotli-devel libssh-devel libcurl-devel gtest-devel which dos2unix glibc-langpack-en diffutils
         ;;
 
     'debian')
         apt-get update
-        apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils
+        apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libbrotli-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils
         debian_build_gtest
         ;;
 
     'arch')
         pacman --noconfirm -Syu
-        pacman --noconfirm -S gcc clang cmake make expat zlib libssh curl gtest dos2unix which diffutils
+        pacman --noconfirm -S gcc clang cmake make expat zlib brotli libssh curl gtest dos2unix which diffutils
         ;;
 
     'ubuntu')
         apt-get update
-        apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils
+        apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libbrotli-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils
         debian_build_gtest
         ;;
 
     'alpine')
         apk update
-        apk add gcc g++ clang cmake make expat-dev zlib-dev libssh-dev curl-dev gtest gtest-dev gmock libintl gettext-dev which dos2unix bash libxml2-utils diffutils 
+        apk add gcc g++ clang cmake make expat-dev zlib-dev brotli-dev libssh-dev curl-dev gtest gtest-dev gmock libintl gettext-dev which dos2unix bash libxml2-utils diffutils 
         ;;
 
     'centos'|'rhel')
         dnf clean all
-        dnf -y install gcc-c++ clang cmake make expat-devel zlib-devel libssh-devel libcurl-devel which dos2unix
+        dnf -y install gcc-c++ clang cmake make expat-devel zlib-devel brotli-devel libssh-devel libcurl-devel which dos2unix
         ;;
 
     'opensuse-tumbleweed')
         zypper --non-interactive refresh
-        zypper --non-interactive install gcc-c++ clang cmake make libexpat-devel zlib-devel libssh-devel curl libcurl-devel git which dos2unix libxml2-tools
+        zypper --non-interactive install gcc-c++ clang cmake make libexpat-devel zlib-devel libbrotli-devel libssh-devel curl libcurl-devel git which dos2unix libxml2-tools
         pushd /tmp
           curl -LO https://github.com/google/googletest/archive/release-1.8.0.tar.gz
           tar xzf   release-1.8.0.tar.gz

cmake/FindBROTLI.cmake

@@ -0,0 +1,41 @@
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+###########################################################################
+include(FindPackageHandleStandardArgs)
+
+find_path(BROTLI_INCLUDE_DIR "brotli/decode.h")
+
+find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon)
+find_library(BROTLIDEC_LIBRARY NAMES brotlidec)
+
+find_package_handle_standard_args(BROTLI
+    FOUND_VAR
+      BROTLI_FOUND
+    REQUIRED_VARS
+      BROTLIDEC_LIBRARY
+      BROTLICOMMON_LIBRARY
+      BROTLI_INCLUDE_DIR
+    FAIL_MESSAGE
+      "Could NOT find BROTLI"
+)
+
+set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR})
+set(BROTLI_LIBRARIES ${BROTLICOMMON_LIBRARY} ${BROTLIDEC_LIBRARY})

cmake/config.h.cmake

@@ -47,9 +47,12 @@
 // Define if you have the <sys/mman.h> header file.
 #cmakedefine EXV_HAVE_SYS_MMAN_H
 
-// Define if you have are using the zlib library.
+// Define if you have the zlib library.
 #cmakedefine EXV_HAVE_LIBZ
 
+// Define if you have the brotli library.
+#cmakedefine EXV_HAVE_BROTLI
+
 /* Define if you have (Exiv2/xmpsdk) Adobe XMP Toolkit. */
 #cmakedefine EXV_HAVE_XMP_TOOLKIT
 

cmake/findDependencies.cmake

@@ -46,6 +46,10 @@ if( EXIV2_ENABLE_PNG )
     find_package( ZLIB REQUIRED )
 endif( )
 
+if( EXIV2_ENABLE_BMFF )
+    find_package( BROTLI )
+endif( )
+
 if( EXIV2_ENABLE_WEBREADY )
     if( EXIV2_ENABLE_CURL )
         find_package(CURL REQUIRED)

cmake/generateConfigFile.cmake

@@ -20,6 +20,7 @@ else()
 endif()
 set(EXV_HAVE_ICONV       ${ICONV_FOUND})
 set(EXV_HAVE_LIBZ        ${ZLIB_FOUND})
+set(EXV_HAVE_BROTLI      ${BROTLI_FOUND})
 
 check_cxx_symbol_exists(mmap        sys/mman.h     EXV_HAVE_MMAP )
 check_cxx_symbol_exists(munmap      sys/mman.h     EXV_HAVE_MUNMAP )

cmake/printSummary.cmake

@@ -51,6 +51,9 @@ else()
     OptionOutput( "XMP metadata support:               " EXIV2_ENABLE_XMP            )
 endif()
 OptionOutput( "Building BMFF support:              " EXIV2_ENABLE_BMFF               )
+if    ( EXIV2_ENABLE_BMFF )
+    OptionOutput( "Brotli support for JPEG XL:         " BROTLI_FOUND                )
+endif()
 OptionOutput( "Native language support:            " EXIV2_ENABLE_NLS                )
 OptionOutput( "Nikon lens database:                " EXIV2_ENABLE_LENSDATA           )
 OptionOutput( "Building webready support:          " EXIV2_ENABLE_WEBREADY           )

conanfile.py

@@ -21,23 +21,25 @@ class Exiv2Conan(ConanFile):
         self.options['gtest'].shared = False
 
     def requirements(self):
-        self.requires('zlib/1.2.12')
+        self.requires('zlib/1.2.13')
+
+        self.requires('brotli/1.0.9')
 
         if self.options.webready:
-            self.requires('libcurl/7.80.0')
+            self.requires('libcurl/7.85.0')
 
         if os_info.is_windows and self.options.iconv:
-            self.requires('libiconv/1.16')
+            self.requires('libiconv/1.17')
 
         if self.options.unitTests:
-            self.requires('gtest/1.10.0')
+            self.requires('gtest/1.12.1')
             if self.settings.build_type == "Debug":
                 self.options['gtest'].debug_postfix = ''
 
         if self.options.xmp:
             self.requires('XmpSdk/2016.7@piponazo/stable') # from conan-piponazo
         else:
-            self.requires('expat/2.4.8')
+            self.requires('expat/2.4.9')
 
     def imports(self):
         self.copy('*.dll', dst='bin', src='bin')

include/exiv2/bmffimage.hpp

@@ -146,6 +146,13 @@ class EXIV2API BmffImage : public Image {
   static bool fullBox(uint32_t box);
   static std::string uuidName(const Exiv2::DataBuf& uuid);
 
+  /*!
+    @brief Wrapper around brotli to uncompress JXL brob content.
+   */
+#ifdef EXV_HAVE_BROTLI
+  static void brotliUncompress(const byte* compressedBuf, size_t compressedBufSize, DataBuf& arr);
+#endif
+
 };  // class BmffImage
 
 // *****************************************************************************

src/CMakeLists.txt

@@ -232,6 +232,11 @@ if( EXIV2_ENABLE_PNG )
 	target_link_libraries( exiv2lib PRIVATE ZLIB::ZLIB)
 endif()
 
+if( EXIV2_ENABLE_BMFF AND BROTLI_FOUND )
+    target_link_libraries( exiv2lib PRIVATE ${BROTLI_LIBRARIES})
+    target_include_directories(exiv2lib PRIVATE ${BROTLI_INCLUDE_DIRS})
+endif()
+
 if( EXIV2_ENABLE_NLS )
     target_link_libraries(exiv2lib PRIVATE ${Intl_LIBRARIES})
     target_include_directories(exiv2lib PRIVATE ${Intl_INCLUDE_DIRS})

src/bmffimage.cpp

@@ -15,6 +15,10 @@
 #include "tiffimage_int.hpp"
 #include "types.hpp"
 
+#ifdef EXV_HAVE_BROTLI
+#include <brotli/decode.h>  // for JXL brob
+#endif
+
 // + standard includes
 #include <cinttypes>
 #include <cstdio>
@@ -51,8 +55,9 @@
 #define TAG_cmt3 0x434D5433 /**< "CMT3" canonID */
 #define TAG_cmt4 0x434D5434 /**< "CMT4" gpsID */
 #define TAG_colr 0x636f6c72 /**< "colr" Colour information */
-#define TAG_exif 0x45786966 /**< "Exif" Used by JXL*/
+#define TAG_exif 0x45786966 /**< "Exif" Used by JXL */
-#define TAG_xml 0x786d6c20  /**< "xml " Used by JXL*/
+#define TAG_xml 0x786d6c20  /**< "xml " Used by JXL */
+#define TAG_brob 0x62726f62 /**< "brob" Used by JXL (brotli box) */
 #define TAG_thmb 0x54484d42 /**< "THMB" Canon thumbnail */
 #define TAG_prvw 0x50525657 /**< "PRVW" Canon preview image */
 
@@ -158,6 +163,55 @@ std::string BmffImage::uuidName(const Exiv2::DataBuf& uuid) {
   return "";
 }
 
+#ifdef EXV_HAVE_BROTLI
+void BmffImage::brotliUncompress(const byte* compressedBuf, size_t compressedBufSize, DataBuf& arr) {
+  BrotliDecoderState* decoder = NULL;
+  decoder = BrotliDecoderCreateInstance(NULL, NULL, NULL);
+  if (!decoder) {
+    throw Error(ErrorCode::kerMallocFailed);
+  }
+
+  size_t uncompressedLen = compressedBufSize * 2;  // just a starting point
+  BrotliDecoderResult result;
+  int dos = 0;
+  size_t available_in = compressedBufSize;
+  const byte* next_in = compressedBuf;
+  size_t available_out;
+  byte* next_out;
+  size_t total_out = 0;
+
+  do {
+    arr.alloc(uncompressedLen);
+    available_out = uncompressedLen - total_out;
+    next_out = arr.data() + total_out;
+    result = BrotliDecoderDecompressStream(decoder, &available_in, &next_in, &available_out, &next_out, &total_out);
+    if (result == BROTLI_DECODER_RESULT_SUCCESS) {
+      arr.resize(total_out);
+    } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) {
+      uncompressedLen *= 2;
+      // DoS protection - can't be bigger than 128k
+      if (uncompressedLen > 131072) {
+        if (++dos > 1)
+          break;
+        uncompressedLen = 131072;
+      }
+    } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
+      // compressed input buffer in incomplete
+      throw Error(ErrorCode::kerFailedToReadImageData);
+    } else {
+      // something bad happened
+      throw Error(ErrorCode::kerErrorMessage, BrotliDecoderErrorString(BrotliDecoderGetErrorCode(decoder)));
+    }
+  } while (result != BROTLI_DECODER_RESULT_SUCCESS);
+
+  BrotliDecoderDestroyInstance(decoder);
+
+  if (result != BROTLI_DECODER_RESULT_SUCCESS) {
+    throw Error(ErrorCode::kerFailedToReadImageData);
+  }
+}
+#endif
+
 uint64_t BmffImage::boxHandler(std::ostream& out /* = std::cout*/, Exiv2::PrintStructureOption option /* = kpsNone */,
                                uint64_t pbox_end, size_t depth) {
   const size_t address = io_->tell();
@@ -452,6 +506,31 @@ uint64_t BmffImage::boxHandler(std::ostream& out /* = std::cout*/, Exiv2::PrintS
     case TAG_xml:
       parseXmp(buffer_size, io_->tell());
       break;
+    case TAG_brob: {
+      enforce(data.size() >= 4, Exiv2::ErrorCode::kerCorruptedMetadata);
+      uint32_t realType = data.read_uint32(0, endian_);
+      if (bTrace) {
+        out << "type: " << toAscii(realType);
+      }
+#ifdef EXV_HAVE_BROTLI
+      DataBuf arr;
+      brotliUncompress(data.c_data(4), data.size() - 4, arr);
+      if (realType == TAG_exif) {
+        uint32_t offset = arr.read_uint32(0, endian_) + 4;
+        enforce(offset + 4 < arr.size(), Exiv2::ErrorCode::kerCorruptedMetadata);
+        Internal::TiffParserWorker::decode(exifData(), iptcData(), xmpData(), arr.c_data(offset), arr.size() - offset,
+                                           Internal::Tag::root, Internal::TiffMapping::findDecoder);
+      } else if (realType == TAG_xml) {
+        arr.resize(arr.size() + 1);
+        arr.write_uint8(arr.size() - 1, 0);  // ensure xmp is null terminated!
+        try {
+          Exiv2::XmpParser::decode(xmpData(), std::string(arr.c_str()));
+        } catch (...) {
+          throw Error(ErrorCode::kerFailedToReadImageData);
+        }
+      }
+#endif
+    } break;
     case TAG_thmb:
       switch (version) {
         case 0:  // JPEG

src/version.cpp

@@ -298,6 +298,7 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector<std::regex>& key
   int have_unistd_h = 0;
   int have_sys_mman = 0;
   int have_libz = 0;
+  int have_brotli = 0;
   int have_xmptoolkit = 0;
   int adobe_xmpsdk = 0;
   int have_bool = 0;
@@ -380,6 +381,10 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector<std::regex>& key
   have_libz = 1;
 #endif
 
+#ifdef EXV_HAVE_BROTLI
+  have_brotli = 1;
+#endif
+
 #ifdef EXV_HAVE_XMP_TOOLKIT
   have_xmptoolkit = 1;
 #endif
@@ -472,6 +477,7 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector<std::regex>& key
   output(os, keys, "have_unistd_h", have_unistd_h);
   output(os, keys, "have_sys_mman", have_sys_mman);
   output(os, keys, "have_libz", have_libz);
+  output(os, keys, "have_brotli", have_brotli);
   output(os, keys, "have_xmptoolkit", have_xmptoolkit);
   output(os, keys, "adobe_xmpsdk", adobe_xmpsdk);
   output(os, keys, "have_bool", have_bool);