Support for ZSTD

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.14)
 
-project(libXISF VERSION 0.2.5 LANGUAGES CXX C
+project(libXISF VERSION 0.2.8 LANGUAGES CXX C
     HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF
     DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.")
 
@@ -94,6 +94,12 @@ endif(USE_BUNDLED_LIBS)
 
 set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
 
+pkg_check_modules(ZSTD libzstd IMPORTED_TARGET)
+if(ZSTD_FOUND)
+    target_compile_definitions(XISF PRIVATE HAVE_ZSTD)
+    target_link_libraries(XISF PUBLIC PkgConfig::ZSTD)
+endif(ZSTD_FOUND)
+
 if(BUILD_SHARED_LIBS)
     target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY)
 else(BUILD_SHARED_LIBS)
@@ -108,6 +114,12 @@ install(FILES ${XISF_PUBLIC_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libxisf.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 install(TARGETS XISF LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
+list(JOIN PC_LIBS_REQUIRE ", " PC_REQUIRE_STR2)
+if(NOT PC_REQUIRE_STR2 STREQUAL "")
+    string(REPLACE "lz4" "liblz4" PC_REQUIRE_STR ${PC_REQUIRE_STR2})
+endif()
+list(TRANSFORM PC_LIBS_REQUIRE PREPEND "-l")
+list(JOIN PC_LIBS_REQUIRE " " PC_LIBS_STR)
 configure_file(libxisf.pc.in libxisf.pc @ONLY)
 
 #testing

libxisf.cpp

@@ -27,6 +27,9 @@
 #include <lz4hc.h>
 #include <pugixml.hpp>
 #include <zlib.h>
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
 #include "streambuffer.h"
 
 namespace LibXISF
@@ -142,6 +145,15 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding)
         if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0)
             throw Error("LZ4 decompression failed");
         break;
+    case ZSTD:
+#ifdef HAVE_ZSTD
+        data.resize(uncompressedSize);
+        if(ZSTD_isError(ZSTD_decompress(data.data(), data.size(), tmp.constData(), tmp.size())))
+            throw Error("ZSTD decompression failed");
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
     }
 
     byteUnshuffle(data, byteShuffling);
@@ -192,7 +204,32 @@ void DataBlock::compress(int sampleFormatSize)
         data.resize(compSize);
         break;
     }
+    case ZSTD:
+    {
+#ifdef HAVE_ZSTD
+        size_t compSize = 0;
+        data.resize(ZSTD_compressBound(uncompressedSize));
+        compSize = ZSTD_compress(data.data(), data.size(), tmp.data(), tmp.size(), compressLevel < 0 ? ZSTD_CLEVEL_DEFAULT : compressLevel);
+        if(ZSTD_isError(compSize))
+            throw Error("ZSTD compression failed");
+
+        data.resize(compSize);
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
     }
+    }
+}
+
+bool DataBlock::CompressionCodecSupported(CompressionCodec codec)
+{
+    (void)codec;
+#ifndef HAVE_ZSTD
+    if(codec == ZSTD)
+        return false;
+#endif
+    return true;
 }
 
 Property::Property(const String &_id, const char *_value) :
@@ -541,6 +578,7 @@ private:
     FITSKeyword parseFITSKeyword(const pugi::xml_node &node);
     ColorFilterArray parseCFA(const pugi::xml_node &node);
     Image parseImage(const pugi::xml_node &node);
+    void readAttachment(DataBlock &dataBlock);
 
     std::unique_ptr<std::istream> _io;
     std::unique_ptr<StreamBuffer> _buffer;
@@ -595,10 +633,7 @@ const Image& XISFReaderPrivate::getImage(uint32_t n, bool readPixels)
     Image &img = _images[n];
     if(img._dataBlock.attachmentPos && readPixels)
     {
-        _io->seekg(img._dataBlock.attachmentPos);
+        readAttachment(img._dataBlock);
-        ByteArray data(img._dataBlock.attachmentSize);
-        _io->read(data.data(), data.size());
-        img._dataBlock.decompress(data);
     }
     return img;
 }
@@ -665,6 +700,10 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock &
             dataBlock.codec = DataBlock::LZ4HC;
         else if(compression[0].find("lz4") == 0)
             dataBlock.codec = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+        else if(compression[0].find("zstd") == 0)
+            dataBlock.codec = DataBlock::ZSTD;
+#endif
         else
             throw Error("Unknown compression codec");
 
@@ -734,16 +773,9 @@ Property XISFReaderPrivate::parseProperty(const pugi::xml_node &node)
     {
         DataBlock dataBlock = parseDataBlock(node);
         if(dataBlock.attachmentPos)
-        {
+            readAttachment(dataBlock);
-            data.resize(dataBlock.attachmentSize);
+
-            _io->seekg(dataBlock.attachmentPos);
+        data = dataBlock.data;
-            _io->read(data.data(), dataBlock.attachmentSize);
-            dataBlock.decompress(data);
-        }
-        else
-        {
-            data = dataBlock.data;
-        }
     }
 
     deserializeVariant(node, property.value, data);
@@ -812,6 +844,9 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     if(node.child("ICCProfile"))
     {
         DataBlock icc = parseDataBlock(node.child("ICCProfile"));
+        if(icc.attachmentPos)
+            readAttachment(icc);
+
         image._iccProfile = icc.data;
     }
 
@@ -821,6 +856,14 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     return image;
 }
 
+void XISFReaderPrivate::readAttachment(DataBlock &dataBlock)
+{
+    ByteArray data(dataBlock.attachmentSize);
+    _io->seekg(dataBlock.attachmentPos);
+    _io->read(data.data(), dataBlock.attachmentSize);
+    dataBlock.decompress(data);
+}
+
 class  XISFWriterPrivate
 {
 public:
@@ -986,6 +1029,12 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
         codec = "lz4";
     else if(dataBlock.codec == DataBlock::LZ4HC)
         codec = "lz4hc";
+    else if(dataBlock.codec == DataBlock::ZSTD)
+#ifdef HAVE_ZSTD
+        codec = "zstd";
+#else
+        throw Error("ZSTD support not compiled");
+#endif
 
     if(dataBlock.byteShuffling > 1)
         codec += "+sh";
@@ -1147,6 +1196,10 @@ struct Init
                 compressionCodecOverride = DataBlock::LZ4HC;
             else if(compression.find("lz4") == 0)
                 compressionCodecOverride = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+            else if(compression.find("zstd") == 0)
+                compressionCodecOverride = DataBlock::ZSTD;
+#endif
 
             if(compression.find("+sh") != std::string::npos)
                 byteShuffleOverride = true;

libxisf.h

@@ -172,7 +172,8 @@ struct LIBXISF_EXPORT DataBlock
         None,
         Zlib,
         LZ4,
-        LZ4HC
+        LZ4HC,
+        ZSTD
     };
     bool embedded = false;
     uint32_t byteShuffling = 0;
@@ -184,6 +185,8 @@ struct LIBXISF_EXPORT DataBlock
     ByteArray data;
     void decompress(const ByteArray &input, const std::string &encoding = "");
     void compress(int sampleFormatSize);
+    /// ZSTD compression can be disabled at compile time
+    static bool CompressionCodecSupported(CompressionCodec codec);
 };
 
 struct LIBXISF_EXPORT Property

libxisf.pc.in

@@ -7,7 +7,6 @@ Name: @PROJECT_NAME@
 Description: @CMAKE_PROJECT_DESCRIPTION@
 URL: @CMAKE_PROJECT_HOMEPAGE_URL@
 Version: @PROJECT_VERSION@
-Requires.private: @PC_LIBS_REQUIRE@
+Requires.private: @PC_REQUIRE_STR@
 Cflags: -I"${includedir}"
 Libs: -L"${libdir}" -lXISF
-Libs.private: -L"${libdir}" @PC_LIBS_REQUIRE@

test/benchmark.cpp

@@ -78,6 +78,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
     image.setByteshuffling(true);
     {
         image.setCompression(DataBlock::Zlib);
@@ -109,6 +120,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
 }
 
 void benchmark()