diff --git a/.github/workflows/on_PR_windows_matrix.yml b/.github/workflows/on_PR_windows_matrix.yml index 876ff381..491c2607 100644 --- a/.github/workflows/on_PR_windows_matrix.yml +++ b/.github/workflows/on_PR_windows_matrix.yml @@ -125,6 +125,7 @@ jobs: gtest:p libiconv:p zlib:p + brotli:p curl:p - name: Build @@ -175,6 +176,9 @@ jobs: libxslt-devel python38-lxml zlib-devel + libbrotlicommon1 + libbrotlidec1 + libbrotli-devel - name: Build run: | cmake -GNinja \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 147a392a..2e57478b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ option( EXIV2_ENABLE_LENSDATA "Build including lens data" option( EXIV2_ENABLE_DYNAMIC_RUNTIME "Use dynamic runtime (used for static libs)" ON ) option( EXIV2_ENABLE_WEBREADY "Build webready support into library" OFF ) option( EXIV2_ENABLE_CURL "USE Libcurl for HttpIo (WEBREADY)" OFF ) -option( EXIV2_ENABLE_BMFF "Build with BMFF support" ON ) +option( EXIV2_ENABLE_BMFF "Build with BMFF support (brotli recommended)" ON ) option( EXIV2_BUILD_SAMPLES "Build sample applications" OFF ) option( EXIV2_BUILD_EXIV2_COMMAND "Build exiv2 command-line executable" ON ) diff --git a/ci/install_dependencies.sh b/ci/install_dependencies.sh index db890039..acef7180 100755 --- a/ci/install_dependencies.sh +++ b/ci/install_dependencies.sh @@ -30,39 +30,39 @@ distro_id=$(grep '^ID=' /etc/os-release|awk -F = '{print $2}'|sed 's/\"//g') case "$distro_id" in 'fedora') - dnf -y --refresh install gcc-c++ clang cmake make expat-devel zlib-devel libssh-devel libcurl-devel gtest-devel which dos2unix glibc-langpack-en diffutils + dnf -y --refresh install gcc-c++ clang cmake make expat-devel zlib-devel brotli-devel libssh-devel libcurl-devel gtest-devel which dos2unix glibc-langpack-en diffutils ;; 'debian') apt-get update - apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils + apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libbrotli-dev libssh-dev libcurl4-openssl-dev libgtest-dev libxml2-utils debian_build_gtest ;; 'arch') pacman --noconfirm -Syu - pacman --noconfirm -S gcc clang cmake make expat zlib libssh curl gtest dos2unix which diffutils + pacman --noconfirm -S gcc clang cmake make expat zlib brotli libssh curl gtest dos2unix which diffutils ;; 'ubuntu') apt-get update - apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils + apt-get install -y cmake g++ clang make libexpat1-dev zlib1g-dev libbrotli-dev libssh-dev libcurl4-openssl-dev libgtest-dev google-mock libxml2-utils debian_build_gtest ;; 'alpine') apk update - apk add gcc g++ clang cmake make expat-dev zlib-dev libssh-dev curl-dev gtest gtest-dev gmock libintl gettext-dev which dos2unix bash libxml2-utils diffutils + apk add gcc g++ clang cmake make expat-dev zlib-dev brotli-dev libssh-dev curl-dev gtest gtest-dev gmock libintl gettext-dev which dos2unix bash libxml2-utils diffutils ;; 'centos'|'rhel') dnf clean all - dnf -y install gcc-c++ clang cmake make expat-devel zlib-devel libssh-devel libcurl-devel which dos2unix + dnf -y install gcc-c++ clang cmake make expat-devel zlib-devel brotli-devel libssh-devel libcurl-devel which dos2unix ;; 'opensuse-tumbleweed') zypper --non-interactive refresh - zypper --non-interactive install gcc-c++ clang cmake make libexpat-devel zlib-devel libssh-devel curl libcurl-devel git which dos2unix libxml2-tools + zypper --non-interactive install gcc-c++ clang cmake make libexpat-devel zlib-devel libbrotli-devel libssh-devel curl libcurl-devel git which dos2unix libxml2-tools pushd /tmp curl -LO https://github.com/google/googletest/archive/release-1.8.0.tar.gz tar xzf release-1.8.0.tar.gz diff --git a/cmake/FindBROTLI.cmake b/cmake/FindBROTLI.cmake new file mode 100644 index 00000000..0ed08550 --- /dev/null +++ b/cmake/FindBROTLI.cmake @@ -0,0 +1,41 @@ +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) 1998 - 2020, Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +########################################################################### +include(FindPackageHandleStandardArgs) + +find_path(BROTLI_INCLUDE_DIR "brotli/decode.h") + +find_library(BROTLICOMMON_LIBRARY NAMES brotlicommon) +find_library(BROTLIDEC_LIBRARY NAMES brotlidec) + +find_package_handle_standard_args(BROTLI + FOUND_VAR + BROTLI_FOUND + REQUIRED_VARS + BROTLIDEC_LIBRARY + BROTLICOMMON_LIBRARY + BROTLI_INCLUDE_DIR + FAIL_MESSAGE + "Could NOT find BROTLI" +) + +set(BROTLI_INCLUDE_DIRS ${BROTLI_INCLUDE_DIR}) +set(BROTLI_LIBRARIES ${BROTLICOMMON_LIBRARY} ${BROTLIDEC_LIBRARY}) diff --git a/cmake/config.h.cmake b/cmake/config.h.cmake index c42d9ed8..50f39613 100644 --- a/cmake/config.h.cmake +++ b/cmake/config.h.cmake @@ -47,9 +47,12 @@ // Define if you have the header file. #cmakedefine EXV_HAVE_SYS_MMAN_H -// Define if you have are using the zlib library. +// Define if you have the zlib library. #cmakedefine EXV_HAVE_LIBZ +// Define if you have the brotli library. +#cmakedefine EXV_HAVE_BROTLI + /* Define if you have (Exiv2/xmpsdk) Adobe XMP Toolkit. */ #cmakedefine EXV_HAVE_XMP_TOOLKIT diff --git a/cmake/findDependencies.cmake b/cmake/findDependencies.cmake index 9315f43c..fb10ccc0 100644 --- a/cmake/findDependencies.cmake +++ b/cmake/findDependencies.cmake @@ -46,6 +46,10 @@ if( EXIV2_ENABLE_PNG ) find_package( ZLIB REQUIRED ) endif( ) +if( EXIV2_ENABLE_BMFF ) + find_package( BROTLI ) +endif( ) + if( EXIV2_ENABLE_WEBREADY ) if( EXIV2_ENABLE_CURL ) find_package(CURL REQUIRED) diff --git a/cmake/generateConfigFile.cmake b/cmake/generateConfigFile.cmake index 91b9554b..81081b5a 100644 --- a/cmake/generateConfigFile.cmake +++ b/cmake/generateConfigFile.cmake @@ -20,6 +20,7 @@ else() endif() set(EXV_HAVE_ICONV ${ICONV_FOUND}) set(EXV_HAVE_LIBZ ${ZLIB_FOUND}) +set(EXV_HAVE_BROTLI ${BROTLI_FOUND}) check_cxx_symbol_exists(mmap sys/mman.h EXV_HAVE_MMAP ) check_cxx_symbol_exists(munmap sys/mman.h EXV_HAVE_MUNMAP ) diff --git a/cmake/printSummary.cmake b/cmake/printSummary.cmake index f1e6a033..b58e9729 100644 --- a/cmake/printSummary.cmake +++ b/cmake/printSummary.cmake @@ -51,6 +51,9 @@ else() OptionOutput( "XMP metadata support: " EXIV2_ENABLE_XMP ) endif() OptionOutput( "Building BMFF support: " EXIV2_ENABLE_BMFF ) +if ( EXIV2_ENABLE_BMFF ) + OptionOutput( "Brotli support for JPEG XL: " BROTLI_FOUND ) +endif() OptionOutput( "Native language support: " EXIV2_ENABLE_NLS ) OptionOutput( "Nikon lens database: " EXIV2_ENABLE_LENSDATA ) OptionOutput( "Building webready support: " EXIV2_ENABLE_WEBREADY ) diff --git a/conanfile.py b/conanfile.py index b9bed818..700028f3 100644 --- a/conanfile.py +++ b/conanfile.py @@ -21,23 +21,25 @@ class Exiv2Conan(ConanFile): self.options['gtest'].shared = False def requirements(self): - self.requires('zlib/1.2.12') + self.requires('zlib/1.2.13') + + self.requires('brotli/1.0.9') if self.options.webready: - self.requires('libcurl/7.80.0') + self.requires('libcurl/7.85.0') if os_info.is_windows and self.options.iconv: - self.requires('libiconv/1.16') + self.requires('libiconv/1.17') if self.options.unitTests: - self.requires('gtest/1.10.0') + self.requires('gtest/1.12.1') if self.settings.build_type == "Debug": self.options['gtest'].debug_postfix = '' if self.options.xmp: self.requires('XmpSdk/2016.7@piponazo/stable') # from conan-piponazo else: - self.requires('expat/2.4.8') + self.requires('expat/2.4.9') def imports(self): self.copy('*.dll', dst='bin', src='bin') diff --git a/include/exiv2/bmffimage.hpp b/include/exiv2/bmffimage.hpp index bde619cb..6956cfc6 100644 --- a/include/exiv2/bmffimage.hpp +++ b/include/exiv2/bmffimage.hpp @@ -146,6 +146,13 @@ class EXIV2API BmffImage : public Image { static bool fullBox(uint32_t box); static std::string uuidName(const Exiv2::DataBuf& uuid); + /*! + @brief Wrapper around brotli to uncompress JXL brob content. + */ +#ifdef EXV_HAVE_BROTLI + static void brotliUncompress(const byte* compressedBuf, size_t compressedBufSize, DataBuf& arr); +#endif + }; // class BmffImage // ***************************************************************************** diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 39039f44..de7ccc28 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -232,6 +232,11 @@ if( EXIV2_ENABLE_PNG ) target_link_libraries( exiv2lib PRIVATE ZLIB::ZLIB) endif() +if( EXIV2_ENABLE_BMFF AND BROTLI_FOUND ) + target_link_libraries( exiv2lib PRIVATE ${BROTLI_LIBRARIES}) + target_include_directories(exiv2lib PRIVATE ${BROTLI_INCLUDE_DIRS}) +endif() + if( EXIV2_ENABLE_NLS ) target_link_libraries(exiv2lib PRIVATE ${Intl_LIBRARIES}) target_include_directories(exiv2lib PRIVATE ${Intl_INCLUDE_DIRS}) diff --git a/src/bmffimage.cpp b/src/bmffimage.cpp index d208ba65..837bcf0f 100644 --- a/src/bmffimage.cpp +++ b/src/bmffimage.cpp @@ -15,6 +15,10 @@ #include "tiffimage_int.hpp" #include "types.hpp" +#ifdef EXV_HAVE_BROTLI +#include // for JXL brob +#endif + // + standard includes #include #include @@ -51,8 +55,9 @@ #define TAG_cmt3 0x434D5433 /**< "CMT3" canonID */ #define TAG_cmt4 0x434D5434 /**< "CMT4" gpsID */ #define TAG_colr 0x636f6c72 /**< "colr" Colour information */ -#define TAG_exif 0x45786966 /**< "Exif" Used by JXL*/ -#define TAG_xml 0x786d6c20 /**< "xml " Used by JXL*/ +#define TAG_exif 0x45786966 /**< "Exif" Used by JXL */ +#define TAG_xml 0x786d6c20 /**< "xml " Used by JXL */ +#define TAG_brob 0x62726f62 /**< "brob" Used by JXL (brotli box) */ #define TAG_thmb 0x54484d42 /**< "THMB" Canon thumbnail */ #define TAG_prvw 0x50525657 /**< "PRVW" Canon preview image */ @@ -158,6 +163,55 @@ std::string BmffImage::uuidName(const Exiv2::DataBuf& uuid) { return ""; } +#ifdef EXV_HAVE_BROTLI +void BmffImage::brotliUncompress(const byte* compressedBuf, size_t compressedBufSize, DataBuf& arr) { + BrotliDecoderState* decoder = NULL; + decoder = BrotliDecoderCreateInstance(NULL, NULL, NULL); + if (!decoder) { + throw Error(ErrorCode::kerMallocFailed); + } + + size_t uncompressedLen = compressedBufSize * 2; // just a starting point + BrotliDecoderResult result; + int dos = 0; + size_t available_in = compressedBufSize; + const byte* next_in = compressedBuf; + size_t available_out; + byte* next_out; + size_t total_out = 0; + + do { + arr.alloc(uncompressedLen); + available_out = uncompressedLen - total_out; + next_out = arr.data() + total_out; + result = BrotliDecoderDecompressStream(decoder, &available_in, &next_in, &available_out, &next_out, &total_out); + if (result == BROTLI_DECODER_RESULT_SUCCESS) { + arr.resize(total_out); + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + uncompressedLen *= 2; + // DoS protection - can't be bigger than 128k + if (uncompressedLen > 131072) { + if (++dos > 1) + break; + uncompressedLen = 131072; + } + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + // compressed input buffer in incomplete + throw Error(ErrorCode::kerFailedToReadImageData); + } else { + // something bad happened + throw Error(ErrorCode::kerErrorMessage, BrotliDecoderErrorString(BrotliDecoderGetErrorCode(decoder))); + } + } while (result != BROTLI_DECODER_RESULT_SUCCESS); + + BrotliDecoderDestroyInstance(decoder); + + if (result != BROTLI_DECODER_RESULT_SUCCESS) { + throw Error(ErrorCode::kerFailedToReadImageData); + } +} +#endif + uint64_t BmffImage::boxHandler(std::ostream& out /* = std::cout*/, Exiv2::PrintStructureOption option /* = kpsNone */, uint64_t pbox_end, size_t depth) { const size_t address = io_->tell(); @@ -452,6 +506,31 @@ uint64_t BmffImage::boxHandler(std::ostream& out /* = std::cout*/, Exiv2::PrintS case TAG_xml: parseXmp(buffer_size, io_->tell()); break; + case TAG_brob: { + enforce(data.size() >= 4, Exiv2::ErrorCode::kerCorruptedMetadata); + uint32_t realType = data.read_uint32(0, endian_); + if (bTrace) { + out << "type: " << toAscii(realType); + } +#ifdef EXV_HAVE_BROTLI + DataBuf arr; + brotliUncompress(data.c_data(4), data.size() - 4, arr); + if (realType == TAG_exif) { + uint32_t offset = arr.read_uint32(0, endian_) + 4; + enforce(offset + 4 < arr.size(), Exiv2::ErrorCode::kerCorruptedMetadata); + Internal::TiffParserWorker::decode(exifData(), iptcData(), xmpData(), arr.c_data(offset), arr.size() - offset, + Internal::Tag::root, Internal::TiffMapping::findDecoder); + } else if (realType == TAG_xml) { + arr.resize(arr.size() + 1); + arr.write_uint8(arr.size() - 1, 0); // ensure xmp is null terminated! + try { + Exiv2::XmpParser::decode(xmpData(), std::string(arr.c_str())); + } catch (...) { + throw Error(ErrorCode::kerFailedToReadImageData); + } + } +#endif + } break; case TAG_thmb: switch (version) { case 0: // JPEG diff --git a/src/version.cpp b/src/version.cpp index 04f5fcb7..3ae9cc80 100644 --- a/src/version.cpp +++ b/src/version.cpp @@ -298,6 +298,7 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector& key int have_unistd_h = 0; int have_sys_mman = 0; int have_libz = 0; + int have_brotli = 0; int have_xmptoolkit = 0; int adobe_xmpsdk = 0; int have_bool = 0; @@ -380,6 +381,10 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector& key have_libz = 1; #endif +#ifdef EXV_HAVE_BROTLI + have_brotli = 1; +#endif + #ifdef EXV_HAVE_XMP_TOOLKIT have_xmptoolkit = 1; #endif @@ -472,6 +477,7 @@ void Exiv2::dumpLibraryInfo(std::ostream& os, const std::vector& key output(os, keys, "have_unistd_h", have_unistd_h); output(os, keys, "have_sys_mman", have_sys_mman); output(os, keys, "have_libz", have_libz); + output(os, keys, "have_brotli", have_brotli); output(os, keys, "have_xmptoolkit", have_xmptoolkit); output(os, keys, "adobe_xmpsdk", adobe_xmpsdk); output(os, keys, "have_bool", have_bool);