Add F32Matrix and F64Matrix into toString

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.14)
 
-project(libXISF VERSION 0.2.5 LANGUAGES CXX C
+project(libXISF VERSION 0.2.11 LANGUAGES CXX C
     HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF
     DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.")
 
@@ -94,6 +94,12 @@ endif(USE_BUNDLED_LIBS)
 
 set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
 
+pkg_check_modules(ZSTD libzstd IMPORTED_TARGET)
+if(ZSTD_FOUND)
+    target_compile_definitions(XISF PRIVATE HAVE_ZSTD)
+    target_link_libraries(XISF PUBLIC PkgConfig::ZSTD)
+endif(ZSTD_FOUND)
+
 if(BUILD_SHARED_LIBS)
     target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY)
 else(BUILD_SHARED_LIBS)
@@ -108,6 +114,12 @@ install(FILES ${XISF_PUBLIC_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libxisf.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 install(TARGETS XISF LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
+list(JOIN PC_LIBS_REQUIRE ", " PC_REQUIRE_STR2)
+if(NOT PC_REQUIRE_STR2 STREQUAL "")
+    string(REPLACE "lz4" "liblz4" PC_REQUIRE_STR ${PC_REQUIRE_STR2})
+endif()
+list(TRANSFORM PC_LIBS_REQUIRE PREPEND "-l")
+list(JOIN PC_LIBS_REQUIRE " " PC_LIBS_STR)
 configure_file(libxisf.pc.in libxisf.pc @ONLY)
 
 #testing

README.md

@@ -9,7 +9,7 @@ To compile simply run these commands
 ```
 cmake -B build -S .
 cmake --build build --parallel
-cmake --install .
+cmake --install build
 ```
 
 By default it use bundled libraries. If you wish to use external libraries you will may add

libxisf.cpp

@@ -27,6 +27,9 @@
 #include <lz4hc.h>
 #include <pugixml.hpp>
 #include <zlib.h>
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
 #include "streambuffer.h"
 
 namespace LibXISF
@@ -83,12 +86,12 @@ static void byteShuffle(ByteArray &data, int itemSize)
     {
         ByteArray &input = data;
         ByteArray output(input.size());
-        int num = input.size() / itemSize;
+        size_t num = input.size() / itemSize;
         char *s = output.data();
         for(int i=0; i<itemSize; i++)
         {
             const char *u = input.constData() + i;
-            for(int o=0; o<num; o++, s++, u += itemSize)
+            for(size_t o=0; o<num; o++, s++, u += itemSize)
                 *s = *u;
         }
         memcpy(s, input.constData() + num * itemSize, input.size() % itemSize);
@@ -102,12 +105,12 @@ static void byteUnshuffle(ByteArray &data, int itemSize)
     {
         ByteArray &input = data;
         ByteArray output(input.size());
-        int num = input.size() / itemSize;
+        size_t num = input.size() / itemSize;
         const char *s = input.constData();
         for(int i=0; i<itemSize; i++)
         {
             char *u = output.data() + i;
-            for(int o=0; o<num; o++, s++, u += itemSize)
+            for(size_t o=0; o<num; o++, s++, u += itemSize)
                 *u = *s;
         }
         memcpy(output.data() + num * itemSize, s, input.size() % itemSize);
@@ -142,6 +145,15 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding)
         if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0)
             throw Error("LZ4 decompression failed");
         break;
+    case ZSTD:
+#ifdef HAVE_ZSTD
+        data.resize(uncompressedSize);
+        if(ZSTD_isError(ZSTD_decompress(data.data(), data.size(), tmp.constData(), tmp.size())))
+            throw Error("ZSTD decompression failed");
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
     }
 
     byteUnshuffle(data, byteShuffling);
@@ -192,8 +204,33 @@ void DataBlock::compress(int sampleFormatSize)
         data.resize(compSize);
         break;
     }
+    case ZSTD:
+    {
+#ifdef HAVE_ZSTD
+        size_t compSize = 0;
+        data.resize(ZSTD_compressBound(uncompressedSize));
+        compSize = ZSTD_compress(data.data(), data.size(), tmp.data(), tmp.size(), compressLevel < 0 ? ZSTD_CLEVEL_DEFAULT : compressLevel);
+        if(ZSTD_isError(compSize))
+            throw Error("ZSTD compression failed");
+
+        data.resize(compSize);
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
     }
     }
+}
+
+bool DataBlock::CompressionCodecSupported(CompressionCodec codec)
+{
+    (void)codec;
+#ifndef HAVE_ZSTD
+    if(codec == ZSTD)
+        return false;
+#endif
+    return true;
+}
 
 Property::Property(const String &_id, const char *_value) :
     id(_id),
@@ -352,6 +389,10 @@ bool Image::addFITSKeywordAsProperty(const String &name, const String &value)
     {
         auto &c = fitsNameToPropertyIdTypeConvert.at(name);
         Property prop(c.first, variantFromString(c.second, value));
+
+        if(name == "APTDIA" || name == "FOCALLEN")
+            prop.value.value<LibXISF::Float32>() /= 1000.0f;
+
         updateProperty(prop);
         return true;
     }
@@ -541,6 +582,7 @@ private:
     FITSKeyword parseFITSKeyword(const pugi::xml_node &node);
     ColorFilterArray parseCFA(const pugi::xml_node &node);
     Image parseImage(const pugi::xml_node &node);
+    void readAttachment(DataBlock &dataBlock);
 
     std::unique_ptr<std::istream> _io;
     std::unique_ptr<StreamBuffer> _buffer;
@@ -595,10 +637,7 @@ const Image& XISFReaderPrivate::getImage(uint32_t n, bool readPixels)
     Image &img = _images[n];
     if(img._dataBlock.attachmentPos && readPixels)
     {
-        _io->seekg(img._dataBlock.attachmentPos);
+        readAttachment(img._dataBlock);
-        ByteArray data(img._dataBlock.attachmentSize);
-        _io->read(data.data(), data.size());
-        img._dataBlock.decompress(data);
     }
     return img;
 }
@@ -665,6 +704,10 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock &
             dataBlock.codec = DataBlock::LZ4HC;
         else if(compression[0].find("lz4") == 0)
             dataBlock.codec = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+        else if(compression[0].find("zstd") == 0)
+            dataBlock.codec = DataBlock::ZSTD;
+#endif
         else
             throw Error("Unknown compression codec");
 
@@ -734,17 +777,10 @@ Property XISFReaderPrivate::parseProperty(const pugi::xml_node &node)
     {
         DataBlock dataBlock = parseDataBlock(node);
         if(dataBlock.attachmentPos)
-        {
+            readAttachment(dataBlock);
-            data.resize(dataBlock.attachmentSize);
+
-            _io->seekg(dataBlock.attachmentPos);
-            _io->read(data.data(), dataBlock.attachmentSize);
-            dataBlock.decompress(data);
-        }
-        else
-        {
         data = dataBlock.data;
     }
-    }
 
     deserializeVariant(node, property.value, data);
 
@@ -812,6 +848,9 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     if(node.child("ICCProfile"))
     {
         DataBlock icc = parseDataBlock(node.child("ICCProfile"));
+        if(icc.attachmentPos)
+            readAttachment(icc);
+
         image._iccProfile = icc.data;
     }
 
@@ -821,6 +860,14 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     return image;
 }
 
+void XISFReaderPrivate::readAttachment(DataBlock &dataBlock)
+{
+    ByteArray data(dataBlock.attachmentSize);
+    _io->seekg(dataBlock.attachmentPos);
+    _io->read(data.data(), dataBlock.attachmentSize);
+    dataBlock.decompress(data);
+}
+
 class  XISFWriterPrivate
 {
 public:
@@ -914,7 +961,7 @@ void XISFWriterPrivate::writeHeader()
         offset += image._dataBlock.data.size();
     }
 
-    uint32_t headerSize = size - sizeof(signature);
+    uint32_t headerSize = header.size() - sizeof(signature);
     header.resize(size, 0);
     header.replace(8, sizeof(uint32_t), (const char*)&headerSize, sizeof(uint32_t));
 
@@ -986,6 +1033,12 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
         codec = "lz4";
     else if(dataBlock.codec == DataBlock::LZ4HC)
         codec = "lz4hc";
+    else if(dataBlock.codec == DataBlock::ZSTD)
+#ifdef HAVE_ZSTD
+        codec = "zstd";
+#else
+        throw Error("ZSTD support not compiled");
+#endif
 
     if(dataBlock.byteShuffling > 1)
         codec += "+sh";
@@ -1147,6 +1200,10 @@ struct Init
                 compressionCodecOverride = DataBlock::LZ4HC;
             else if(compression.find("lz4") == 0)
                 compressionCodecOverride = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+            else if(compression.find("zstd") == 0)
+                compressionCodecOverride = DataBlock::ZSTD;
+#endif
 
             if(compression.find("+sh") != std::string::npos)
                 byteShuffleOverride = true;

libxisf.h

@@ -172,7 +172,8 @@ struct LIBXISF_EXPORT DataBlock
         None,
         Zlib,
         LZ4,
-        LZ4HC
+        LZ4HC,
+        ZSTD
     };
     bool embedded = false;
     uint32_t byteShuffling = 0;
@@ -184,6 +185,8 @@ struct LIBXISF_EXPORT DataBlock
     ByteArray data;
     void decompress(const ByteArray &input, const std::string &encoding = "");
     void compress(int sampleFormatSize);
+    /// ZSTD compression can be disabled at compile time
+    static bool CompressionCodecSupported(CompressionCodec codec);
 };
 
 struct LIBXISF_EXPORT Property

libxisf.pc.in

@@ -7,7 +7,6 @@ Name: @PROJECT_NAME@
 Description: @CMAKE_PROJECT_DESCRIPTION@
 URL: @CMAKE_PROJECT_HOMEPAGE_URL@
 Version: @PROJECT_VERSION@
-Requires.private: @PC_LIBS_REQUIRE@
+Requires.private: @PC_REQUIRE_STR@
 Cflags: -I"${includedir}"
 Libs: -L"${libdir}" -lXISF
-Libs.private: -L"${libdir}" @PC_LIBS_REQUIRE@

streambuffer.cpp

@@ -89,7 +89,7 @@ std::streamsize StreamBuffer::xsgetn(char_type *s, std::streamsize n)
     std::streamsize len = egptr() - gptr();
     if(len > 0)
     {
-        std::streamsize c = std::min(n, len);
+        std::streamsize c = n < len ? n : len;
         std::memcpy(s, gptr(), c);
         gbump(c);
         ret = c;

test/benchmark.cpp

@@ -78,6 +78,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
     image.setByteshuffling(true);
     {
         image.setCompression(DataBlock::Zlib);
@@ -109,6 +120,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
 }
 
 void benchmark()

utils.cpp

@@ -52,7 +52,7 @@ void sha1(uint8_t *data, size_t len, uint8_t *hash)
     nlen += 64 - nlen % 64;
     tmp.resize(nlen, 0);
 
-    size_t ml = len * 8;
+    uint64_t ml = len * 8;
     tmp[nlen - 1] = ml & 0xff;
     tmp[nlen - 2] = ml >>  8 & 0xff;
     tmp[nlen - 3] = ml >> 16 & 0xff;

variant.cpp

@@ -108,8 +108,8 @@ static std::map<Variant::Type, const char*> idToType = {
     {Variant::Type::UI32Matrix,    "UI32Matrix"},
     {Variant::Type::I64Matrix,     "I64Matrix"},
     {Variant::Type::UI64Matrix,    "UI64Matrix"},
-    {Variant::Type::F32Matrix,     "I8Matrix"},
+    {Variant::Type::F32Matrix,     "F32Matrix"},
-    {Variant::Type::F64Matrix,     "UI8Matrix"},
+    {Variant::Type::F64Matrix,     "F64Matrix"},
 };
 
 template<typename T>
@@ -196,9 +196,12 @@ void deserializeVariant(const pugi::xml_node &node, Variant &variant, const Byte
     std::string type = node.attribute("type").as_string();
     Variant::Type typeId = typeToId[type];
 
-    if(typeId == Variant::Type::String && !node.attribute("location"))
+    if(typeId == Variant::Type::String)
     {
+        if(!node.attribute("location"))
             variant.setValue(node.text().as_string());
+        else
+            variant.setValue(String(data.constData(), data.size()));
     }
     else if(node.attribute("value"))
     {
@@ -400,6 +403,7 @@ Variant variantFromString(Variant::Type type, const String &str)
 
 Variant::Type Variant::type() const
 {
+    int idx = _value.index();
     return (Variant::Type)_value.index();
 }
 
@@ -430,6 +434,28 @@ String Variant::toString() const
         return ss.str();
     };
 
+    auto matrixString = [](auto matrix) {
+        std::stringstream ss;
+        ss << "{";
+        for(int i=0; i<matrix.rows(); i++)
+        {
+            ss << "{";
+            for(int o=0; o<matrix.cols(); o++)
+            {
+                char str[128] = {0};
+                char *end = str + sizeof(str);
+                std::to_chars(str, end, matrix(i, o));
+                ss << str;
+                if(o < matrix.cols() - 1)
+                    ss << ",";
+            }
+            ss << "}";
+            if(i < matrix.rows() - 1)
+                ss << ",";
+        }
+        return ss.str();
+    };
+
     switch(type())
     {
     case Variant::Type::Int8: toChars<Int8>(_value, str, end); break;
@@ -454,6 +480,8 @@ String Variant::toString() const
     case Variant::Type::UI64Vector: string = vectorString(std::get<UI64Vector>(_value)); break;
     case Variant::Type::F32Vector: string = vectorString(std::get<F32Vector>(_value)); break;
     case Variant::Type::F64Vector: string = vectorString(std::get<F64Vector>(_value)); break;
+    case Variant::Type::F32Matrix: string = matrixString(std::get<F32Matrix>(_value)); break;
+    case Variant::Type::F64Matrix: string = matrixString(std::get<F64Matrix>(_value)); break;
     case Variant::Type::String: string = std::get<String>(_value); break;
     case Variant::Type::TimePoint:
     {