Add conflicts with libxisf0

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.14)
 
-project(libXISF VERSION 0.2.5 LANGUAGES CXX C
+project(libXISF VERSION 0.2.12 LANGUAGES CXX C
     HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF
     DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.")
 
@@ -94,6 +94,12 @@ endif(USE_BUNDLED_LIBS)
 
 set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
 
+pkg_check_modules(ZSTD libzstd IMPORTED_TARGET)
+if(ZSTD_FOUND)
+    target_compile_definitions(XISF PRIVATE HAVE_ZSTD)
+    target_link_libraries(XISF PUBLIC PkgConfig::ZSTD)
+endif(ZSTD_FOUND)
+
 if(BUILD_SHARED_LIBS)
     target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY)
 else(BUILD_SHARED_LIBS)
@@ -108,6 +114,12 @@ install(FILES ${XISF_PUBLIC_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/libxisf.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 install(TARGETS XISF LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
+list(JOIN PC_LIBS_REQUIRE ", " PC_REQUIRE_STR2)
+if(NOT PC_REQUIRE_STR2 STREQUAL "")
+    string(REPLACE "lz4" "liblz4" PC_REQUIRE_STR ${PC_REQUIRE_STR2})
+endif()
+list(TRANSFORM PC_LIBS_REQUIRE PREPEND "-l")
+list(JOIN PC_LIBS_REQUIRE " " PC_LIBS_STR)
 configure_file(libxisf.pc.in libxisf.pc @ONLY)
 
 #testing

README.md

@@ -9,7 +9,7 @@ To compile simply run these commands
 ```
 cmake -B build -S .
 cmake --build build --parallel
-cmake --install .
+cmake --install build
 ```
 
 By default it use bundled libraries. If you wish to use external libraries you will may add

debian/changelog

@@ -0,0 +1,80 @@
+libxisf (0.2.12-ubuntu1) focal; urgency=medium
+
+  * Fix bug in writing ICCProfile
+  * Fix handling big +2GiB images
+
+ -- Dušan Poizl <nou@nouspiro.space>  Sat, 23 Mar 2024 09:25:46 +0100
+
+libxisf (0.2.11-ubuntu1) focal; urgency=medium
+
+  * Fix not loading String properties
+  * Convert aperture and focal length from mm to meters
+
+ -- Dušan Poizl <nou@nouspiro.space>  Fri, 05 Jan 2024 16:18:32 +0100
+
+libxisf (0.2.10-ubuntu1) focal; urgency=medium
+
+  * Fix writing incorrect header size
+
+ -- Dušan Poizl <nou@nouspiro.space>  Tue, 17 Oct 2023 20:33:05 +0200
+
+libxisf (0.2.9-ubuntu1) focal; urgency=medium
+
+  * Add support for ZSTD
+
+ -- Dušan Poizl <nou@nouspiro.space>  Mon, 14 Aug 2023 10:32:30 +0200
+
+libxisf (0.2.6-ubuntu1) focal; urgency=medium
+
+  * Add method for ICC profile
+  * pkg-config file support
+
+ -- Dušan Poizl <nou@nouspiro.space>  Fri, 02 Jun 2023 09:16:59 +0200
+
+libxisf (0.2.4-ubuntu1) focal; urgency=medium
+
+  * Add getThumbnail method
+
+ -- Dušan Poizl <nou@nouspiro.space>  Tue, 19 May 2023 19:23:34 +0200
+
+libxisf (0.2.3-ubuntu1) focal; urgency=medium
+
+  * Fix Color filter array
+
+ -- Dušan Poizl <nou@nouspiro.space>  Tue, 11 Apr 2023 12:39:41 +0200
+
+libxisf (0.2.0-ubuntu2) focal; urgency=medium
+
+  * Fix lintian errors
+
+ -- Dušan Poizl <nou@nouspiro.space>  Sun, 19 Mar 2023 22:10:39 +0100
+
+libxisf (0.2.0-ubuntu1) focal; urgency=medium
+
+  * Rewrite to remove Qt as dependency
+
+ -- Dušan Poizl <nou@nouspiro.space>  Fri, 10 Mar 2023 22:10:01 +0100
+
+libxisf (0.1.3-ubuntu1) focal; urgency=medium
+
+  * Write missing pixelStorage attribute
+
+ -- Dušan Poizl <nou@nouspiro.space>  Fri, 17 Feb 2023 16:57:30 +0100
+
+libxisf (0.1.2-ubuntu1) focal; urgency=medium
+
+  * Add qtbase5-dev as dependency to libxisf-dev
+
+ -- Dušan Poizl <nou@nouspiro.space>  Thu, 09 Feb 2023 21:28:45 +0100
+
+libxisf (0.1.1-ubuntu1) focal; urgency=medium
+
+  * Fixed packaging
+
+ -- Dušan Poizl <nou@nouspiro.space>  Wed, 08 Feb 2023 18:57:50 +0100
+
+libxisf (0.1.0-ubuntu1) focal; urgency=low
+
+  * Initial release.
+
+ -- Dušan Poizl <nou@nouspiro.space>  Tue, 07 Feb 2023 10:47:08 +0100

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -0,0 +1,45 @@
+Source: libxisf
+Priority: optional
+Maintainer: Dušan Poizl <nou@nouspiro.space>
+Build-Depends: debhelper (>= 9),
+               cdbs,
+               cmake,
+               pkg-config,
+               liblz4-dev,
+               libpugixml-dev,
+               zlib1g-dev,
+               libzstd-dev
+Standards-Version: 4.5.0
+Section: libs
+Homepage: https://gitea.nouspiro.space/nou/libXISF
+Vcs-Git: https://gitea.nouspiro.space/nou/libXISF.git
+
+Package: libxisf-dev
+Section: libdevel
+Architecture: any
+Multi-Arch: same
+Depends: libxisf (= ${binary:Version}), ${misc:Depends}
+Description: Library to load and save XISF images development files
+ This package contain development files.
+ .
+ XISF is native format of PixInsight astroprocessing suite.
+ Extensible Image Serialization Format (XISF) is the native file format
+ of PixInsight. It is a free, open format for storage, management and
+ interchange of digital images and associated data.
+ .
+ This C++ library implements XISF specification 1.0 which can be found at
+ https://pixinsight.com/doc/docs/XISF-1.0-spec/XISF-1.0-spec.html
+
+Package: libxisf
+Architecture: any
+Multi-Arch: same
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Conflicts: libxisf0
+Description: Library to load and save XISF images
+ XISF is native format of PixInsight astroprocessing suite.
+ Extensible Image Serialization Format (XISF) is the native file format
+ of PixInsight. It is a free, open format for storage, management and
+ interchange of digital images and associated data.
+ .
+ This C++ library implements XISF specification 1.0 which can be found at
+ https://pixinsight.com/doc/docs/XISF-1.0-spec/XISF-1.0-spec.html

debian/copyright

@@ -0,0 +1,97 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: libxisf
+Source: https://gitea.nouspiro.space/nou/libXISF
+
+Files: * debian/*
+Copyright: (c) 2023 <Dušan Poizl nou@nouspiro.space>
+License: GPL-3+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete texts of the GNU General Public Licenses
+ version 2 and 3 can be found in '/usr/share/common-licenses/GPL-2' and
+ '/usr/share/common-licenses/GPL-3'.
+
+Files: lz4/*
+Copyright: (c) 2011-2020, Yann Collet
+License: BSD-2-clause
+ LZ4 Library
+ Copyright (c) 2011-2020, Yann Collet
+ All rights reserved.
+ .
+ Redistribution and use in source and binary forms, with or without modification,
+ are permitted provided that the following conditions are met:
+ .
+ * Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+ .
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Files: zlib/*
+Copyright: (c) 1995-2022 Jean-loup Gailly and Mark Adler
+License: Zlib
+ This software is provided 'as-is', without any express or implied
+ warranty.  In no event will the authors be held liable for any damages
+ arising from the use of this software.
+ .
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+ .
+ 1. The origin of this software must not be misrepresented; you must not
+    claim that you wrote the original software. If you use this software
+    in a product, an acknowledgment in the product documentation would be
+    appreciated but is not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+ .
+ Jean-loup Gailly        Mark Adler
+ jloup@gzip.org          madler@alumni.caltech.edu
+
+Files: pugixml/*
+Copyright: (c) 2006-2022 Arseny Kapoulkine
+License: MIT
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+ .
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.

debian/libxisf-dev.install

@@ -0,0 +1,2 @@
+usr/include
+usr/lib/*/*.so

debian/libxisf.install

@@ -0,0 +1 @@
+usr/lib/*/*.so.*

debian/rules

@@ -0,0 +1,16 @@
+#!/usr/bin/make -f
+
+export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
+DPKG_EXPORT_BUILDFLAGS = 1
+include /usr/share/dpkg/buildflags.mk
+CFLAGS+=$(CPPFLAGS)
+CXXFLAGS+=$(CPPFLAGS)
+
+%:
+	dh $@
+
+override_dh_auto_configure:
+	dh_auto_configure -- -DCMAKE_LIBRARY_ARCHITECTURE="$(DEB_HOST_MULTIARCH)" -DUSE_BUNDLED_LIBS=Off

libxisf.cpp

@@ -27,6 +27,9 @@
 #include <lz4hc.h>
 #include <pugixml.hpp>
 #include <zlib.h>
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
 #include "streambuffer.h"
 
 namespace LibXISF
@@ -46,6 +49,7 @@ static std::unordered_map<Image::ColorSpace, String> colorSpaceToString;
 static DataBlock::CompressionCodec compressionCodecOverride = DataBlock::None;
 static bool byteShuffleOverride = false;
 static int compressionLevelOverride = -1;
+const size_t GiB = 1073741824;
 
 static const std::unordered_map<String, std::pair<String, Variant::Type>> fitsNameToPropertyIdTypeConvert = {
     {"OBSERVER", {"Observer:Name", Variant::Type::String}},
@@ -83,12 +87,12 @@ static void byteShuffle(ByteArray &data, int itemSize)
     {
         ByteArray &input = data;
         ByteArray output(input.size());
-        int num = input.size() / itemSize;
+        size_t num = input.size() / itemSize;
         char *s = output.data();
         for(int i=0; i<itemSize; i++)
         {
             const char *u = input.constData() + i;
-            for(int o=0; o<num; o++, s++, u += itemSize)
+            for(size_t o=0; o<num; o++, s++, u += itemSize)
                 *s = *u;
         }
         memcpy(s, input.constData() + num * itemSize, input.size() % itemSize);
@@ -102,12 +106,12 @@ static void byteUnshuffle(ByteArray &data, int itemSize)
     {
         ByteArray &input = data;
         ByteArray output(input.size());
-        int num = input.size() / itemSize;
+        size_t num = input.size() / itemSize;
         const char *s = input.constData();
         for(int i=0; i<itemSize; i++)
         {
             char *u = output.data() + i;
-            for(int o=0; o<num; o++, s++, u += itemSize)
+            for(size_t o=0; o<num; o++, s++, u += itemSize)
                 *u = *s;
         }
         memcpy(output.data() + num * itemSize, s, input.size() % itemSize);
@@ -124,6 +128,9 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding)
     else if(encoding == "base16")
         tmp.decodeHex();
 
+    if(subblocks.size() == 0)
+        subblocks.push_back({tmp.size(), uncompressedSize});
+
     switch(codec)
     {
     case None:
@@ -132,17 +139,54 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding)
     case Zlib:
     {
         data.resize(uncompressedSize);
-        uLongf size = uncompressedSize;
+        const char *srcPtr = tmp.constData();
-        ::uncompress((Bytef*)data.data(), &size, (Bytef*)tmp.data(), tmp.size());
+        char *dstPtr = data.data();
+        for(auto &block : subblocks)
+        {
+            uLongf size = block.second;
+            ::uncompress((Bytef*)dstPtr, &size, (const Bytef*)srcPtr, block.first);
+            srcPtr += block.first;
+            dstPtr += block.second;
+        }
         break;
     }
     case LZ4:
     case LZ4HC:
+    {
         data.resize(uncompressedSize);
-        if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0)
+        const char *srcPtr = tmp.constData();
+        char *dstPtr = data.data();
+        for(auto &block : subblocks)
+        {
+            if(LZ4_decompress_safe(srcPtr, dstPtr, block.first, block.second) < 0)
                 throw Error("LZ4 decompression failed");
+            srcPtr += block.first;
+            dstPtr += block.second;
+        }
         break;
     }
+    case ZSTD:
+#ifdef HAVE_ZSTD
+    {
+        data.resize(uncompressedSize);
+        const char *srcPtr = tmp.constData();
+        char *dstPtr = data.data();
+        for(auto &block : subblocks)
+        {
+            if(ZSTD_isError(ZSTD_decompress(dstPtr, block.second, srcPtr, block.first)))
+                throw Error("ZSTD decompression failed");
+            srcPtr += block.first;
+            dstPtr += block.second;
+        }
+
+    }
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
+    }
+
+    subblocks.clear();
 
     byteUnshuffle(data, byteShuffling);
     attachmentPos = 0;
@@ -169,31 +213,78 @@ void DataBlock::compress(int sampleFormatSize)
         break;
     case Zlib:
     {
-        data.resize(compressBound(uncompressedSize));
+        int64_t size = tmp.size();
-        uLongf compressedSize = data.size();
+        int64_t compSize = 0;
-        if(::compress2((Bytef*)data.data(), &compressedSize, (Bytef*)tmp.data(), tmp.size(), compressLevel) != Z_OK)
+        int64_t inPtr = 0;
+        while(inPtr < size)
+        {
+            int64_t inSize = UINT32_MAX < size - inPtr ? UINT32_MAX : size - inPtr;
+            data.resize(compSize + compressBound(inSize));
+            uLongf outSize = data.size() - compSize;
+            if(::compress2((Bytef*)data.data() + compSize, &outSize, (const Bytef*)tmp.constData() + inPtr, inSize, compressLevel) != Z_OK)
                 throw Error("Zlib compression failed");
-        data.resize(compressedSize);
+
+            compSize += outSize;
+            inPtr += inSize;
+            subblocks.push_back({outSize, inSize});
+        }
+        data.resize(compSize);
         break;
     }
     case LZ4:
     case LZ4HC:
     {
-        int compSize = 0;
+        int64_t size = tmp.size();
-        data.resize(LZ4_compressBound(tmp.size()));
+        int64_t compSize = 0;
-        if(codec == LZ4)
+        int64_t inPtr = 0;
-            compSize = LZ4_compress_default(tmp.constData(), data.data(), tmp.size(), data.size());
+        while(inPtr < size)
-        else
+        {
-            compSize = LZ4_compress_HC(tmp.constData(), data.data(), tmp.size(), data.size(), compressLevel < 0 ? LZ4HC_CLEVEL_DEFAULT : compressLevel);
+            int64_t inSize = LZ4_MAX_INPUT_SIZE < size - inPtr ? LZ4_MAX_INPUT_SIZE : size - inPtr;
+            data.resize(compSize + LZ4_compressBound(inSize));
+            int outSize = 0;
 
-        if(compSize <= 0)
+            if(codec == LZ4)
+                outSize = LZ4_compress_default(tmp.constData() + inPtr, data.data() + compSize, inSize, data.size() - compSize);
+            else
+                outSize = LZ4_compress_HC(tmp.constData() + inPtr, data.data() + compSize, inSize, data.size() - compSize, compressLevel < 0 ? LZ4HC_CLEVEL_DEFAULT : compressLevel);
+
+            if(outSize <= 0)
                 throw Error("LZ4 compression failed");
 
+            compSize += outSize;
+            inPtr += inSize;
+            subblocks.push_back({outSize, inSize});
+        }
         data.resize(compSize);
         break;
     }
+    case ZSTD:
+    {
+#ifdef HAVE_ZSTD
+        size_t compSize = 0;
+        data.resize(ZSTD_compressBound(uncompressedSize));
+        compSize = ZSTD_compress(data.data(), data.size(), tmp.data(), tmp.size(), compressLevel < 0 ? ZSTD_CLEVEL_DEFAULT : compressLevel);
+        if(ZSTD_isError(compSize))
+            throw Error("ZSTD compression failed");
+
+        data.resize(compSize);
+#else
+        throw Error("ZSTD support not compiled");
+#endif
+        break;
     }
     }
+}
+
+bool DataBlock::CompressionCodecSupported(CompressionCodec codec)
+{
+    (void)codec;
+#ifndef HAVE_ZSTD
+    if(codec == ZSTD)
+        return false;
+#endif
+    return true;
+}
 
 Property::Property(const String &_id, const char *_value) :
     id(_id),
@@ -352,6 +443,10 @@ bool Image::addFITSKeywordAsProperty(const String &name, const String &value)
     {
         auto &c = fitsNameToPropertyIdTypeConvert.at(name);
         Property prop(c.first, variantFromString(c.second, value));
+
+        if(name == "APTDIA" || name == "FOCALLEN")
+            prop.value.value<LibXISF::Float32>() /= 1000.0f;
+
         updateProperty(prop);
         return true;
     }
@@ -541,6 +636,7 @@ private:
     FITSKeyword parseFITSKeyword(const pugi::xml_node &node);
     ColorFilterArray parseCFA(const pugi::xml_node &node);
     Image parseImage(const pugi::xml_node &node);
+    void readAttachment(DataBlock &dataBlock);
 
     std::unique_ptr<std::istream> _io;
     std::unique_ptr<StreamBuffer> _buffer;
@@ -595,10 +691,7 @@ const Image& XISFReaderPrivate::getImage(uint32_t n, bool readPixels)
     Image &img = _images[n];
     if(img._dataBlock.attachmentPos && readPixels)
     {
-        _io->seekg(img._dataBlock.attachmentPos);
+        readAttachment(img._dataBlock);
-        ByteArray data(img._dataBlock.attachmentSize);
-        _io->read(data.data(), data.size());
-        img._dataBlock.decompress(data);
     }
     return img;
 }
@@ -665,10 +758,14 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock &
             dataBlock.codec = DataBlock::LZ4HC;
         else if(compression[0].find("lz4") == 0)
             dataBlock.codec = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+        else if(compression[0].find("zstd") == 0)
+            dataBlock.codec = DataBlock::ZSTD;
+#endif
         else
             throw Error("Unknown compression codec");
 
-        dataBlock.uncompressedSize = std::stoul(compression[1]);
+        dataBlock.uncompressedSize = std::stoull(compression[1]);
 
         if(compression[0].find("+sh") != std::string::npos)
         {
@@ -677,6 +774,18 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock &
             else
                 throw Error("Missing byte shuffling size");
         }
+
+        if(node.attribute("subblocks"))
+        {
+            std::vector<std::string> subblocks = splitString(node.attribute("subblocks").as_string(), ':');
+            for(auto &block : subblocks)
+            {
+                size_t pos = 0;
+                size_t comp = std::stoull(block, &pos);
+                size_t deco = std::stoull(block.substr(pos+1));
+                dataBlock.subblocks.push_back({comp, deco});
+            }
+        }
     }
 }
 
@@ -698,8 +807,8 @@ DataBlock XISFReaderPrivate::parseDataBlock(const pugi::xml_node &node)
     }
     else if(location.size() >= 3 && location[0] == "attachment")
     {
-        dataBlock.attachmentPos = std::stoul(location[1]);
+        dataBlock.attachmentPos = std::stoull(location[1]);
-        dataBlock.attachmentSize = std::stoul(location[2]);
+        dataBlock.attachmentSize = std::stoull(location[2]);
     }
     else
     {
@@ -734,17 +843,10 @@ Property XISFReaderPrivate::parseProperty(const pugi::xml_node &node)
     {
         DataBlock dataBlock = parseDataBlock(node);
         if(dataBlock.attachmentPos)
-        {
+            readAttachment(dataBlock);
-            data.resize(dataBlock.attachmentSize);
+
-            _io->seekg(dataBlock.attachmentPos);
-            _io->read(data.data(), dataBlock.attachmentSize);
-            dataBlock.decompress(data);
-        }
-        else
-        {
         data = dataBlock.data;
     }
-    }
 
     deserializeVariant(node, property.value, data);
 
@@ -782,9 +884,9 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
 
     std::vector<std::string> geometry = splitString(node.attribute("geometry").as_string(), ':');
     if(geometry.size() != 3)throw Error("We support only 2D images");
-    image._width = std::stoul(geometry[0]);
+    image._width = std::stoull(geometry[0]);
-    image._height = std::stoul(geometry[1]);
+    image._height = std::stoull(geometry[1]);
-    image._channelCount = std::stoul(geometry[2]);
+    image._channelCount = std::stoull(geometry[2]);
     if(!image._width || !image._height || !image._channelCount)throw Error("Invalid image geometry");
 
     std::vector<std::string> bounds = splitString(node.attribute("bounds").as_string(), ':');
@@ -812,6 +914,9 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     if(node.child("ICCProfile"))
     {
         DataBlock icc = parseDataBlock(node.child("ICCProfile"));
+        if(icc.attachmentPos)
+            readAttachment(icc);
+
         image._iccProfile = icc.data;
     }
 
@@ -821,6 +926,22 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node)
     return image;
 }
 
+void XISFReaderPrivate::readAttachment(DataBlock &dataBlock)
+{
+    ByteArray data(dataBlock.attachmentSize);
+    _io->seekg(dataBlock.attachmentPos);
+    size_t size = dataBlock.attachmentSize;
+    char *ptr = data.data();
+    while(size > 0)
+    {
+        size_t s = std::min(size, GiB);
+        _io->read(ptr, s);
+        size -= s;
+        ptr += s;
+    }
+    dataBlock.decompress(data);
+}
+
 class  XISFWriterPrivate
 {
 public:
@@ -835,6 +956,7 @@ private:
     void writePropertyElement(pugi::xml_node &node, const Property &property);
     void writeFITSKeyword(pugi::xml_node &node, const FITSKeyword &keyword);
     void writeMetadata(pugi::xml_node &node);
+    void updateImageAttachmentPos(pugi::xml_node &root, size_t offset);
     ByteArray _xisfHeader;
     ByteArray _attachmentsData;
     std::vector<Image> _images;
@@ -866,7 +988,15 @@ void XISFWriterPrivate::save(std::ostream &io)
 
     for(auto &image : _images)
     {
-        io.write(image._dataBlock.data.constData(), image._dataBlock.data.size());
+        const char *ptr = image._dataBlock.data.constData();
+        size_t size = image._dataBlock.data.size();
+        while(size > 0)
+        {
+            size_t s = std::min(size, GiB);
+            io.write(ptr, s);
+            ptr += s;
+            size -= s;
+        }
     }
 }
 
@@ -897,25 +1027,26 @@ void XISFWriterPrivate::writeHeader()
 
     writeMetadata(root);
 
+    uint32_t size = 0;
+    std::string header;
+    while(true)
+    {
         std::stringstream xml;
         xml.write(signature, sizeof(signature));
         doc.save(xml, "", pugi::format_raw);
-
+        header = xml.str();
-    std::string header = xml.str();
+        if(size != header.size())
-    uint32_t size = header.size();
-
-    uint32_t offset = 0;
-    std::string replace = "attachment:2147483648";
-    for(auto &image : _images)
         {
-        std::string blockPos = std::string("attachment:") + std::to_string(size + offset);
+            size = header.size();
-        size_t pos = header.find(replace, 32);
+            updateImageAttachmentPos(root, size);
-        header.replace(pos, replace.size(), blockPos);
+        }
-        offset += image._dataBlock.data.size();
+        else
+        {
+            break;
+        }
     }
 
-    uint32_t headerSize = size - sizeof(signature);
+    uint32_t headerSize = header.size() - sizeof(signature);
-    header.resize(size, 0);
     header.replace(8, sizeof(uint32_t), (const char*)&headerSize, sizeof(uint32_t));
 
     _xisfHeader = ByteArray(header.c_str(), header.size());
@@ -955,7 +1086,8 @@ void XISFWriterPrivate::writeImageElement(pugi::xml_node &node, const Image &ima
     if(image._iccProfile.size())
     {
         ByteArray base64 = image._iccProfile;
-        base64.decodeBase64();
+        base64.encodeBase64();
+        base64.append('\0');
         pugi::xml_node icc_node = image_node.append_child("ICCProfile");
         icc_node.append_attribute("location").set_value("inline:base64");
         icc_node.append_child(pugi::node_pcdata).set_value(base64.data());
@@ -974,7 +1106,10 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
     }
     else
     {
-        std::string attachment = "attachment:2147483648:" + std::to_string(dataBlock.data.size());
+        std::string attachment = "attachment:";
+        if(dataBlock.attachmentPos == 0) attachment += "99999";
+        else attachment += std::to_string(dataBlock.attachmentPos);
+        attachment += ":" + std::to_string(dataBlock.data.size());
         image_node.append_attribute("location").set_value(attachment.c_str());
     }
 
@@ -986,6 +1121,12 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
         codec = "lz4";
     else if(dataBlock.codec == DataBlock::LZ4HC)
         codec = "lz4hc";
+    else if(dataBlock.codec == DataBlock::ZSTD)
+#ifdef HAVE_ZSTD
+        codec = "zstd";
+#else
+        throw Error("ZSTD support not compiled");
+#endif
 
     if(dataBlock.byteShuffling > 1)
         codec += "+sh";
@@ -998,6 +1139,19 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
 
         image_node.append_attribute("compression").set_value(codec.c_str());
     }
+
+    if(!dataBlock.subblocks.empty())
+    {
+        std::string subblocks;
+        for(auto i = dataBlock.subblocks.begin(); i != dataBlock.subblocks.end(); i++)
+        {
+            if(i != dataBlock.subblocks.begin())
+                subblocks += ":";
+
+            subblocks += std::to_string(i->first) + "," + std::to_string(i->second);
+        }
+        image_node.append_attribute("subblocks").set_value(subblocks.c_str());
+    }
 }
 
 void XISFWriterPrivate::writePropertyElement(pugi::xml_node &node, const Property &property)
@@ -1028,6 +1182,19 @@ void XISFWriterPrivate::writeMetadata(pugi::xml_node &node)
     writePropertyElement(metadata, Property("XISF:CreatorApplication", "LibXISF"));
 }
 
+void XISFWriterPrivate::updateImageAttachmentPos(pugi::xml_node &root, size_t offset)
+{
+    pugi::xpath_node_set imageNodes = root.select_nodes("//Image");
+    int i = 0;
+    for(auto &image : _images)
+    {
+        pugi::xml_node node = imageNodes[i++].node();
+        std::string location = "attachment:" + std::to_string(offset) + ":" + std::to_string(image._dataBlock.data.size());
+        offset += image._dataBlock.data.size();
+        node.attribute("location").set_value(location.c_str());
+    }
+}
+
 XISFReader::XISFReader()
 {
     p = new XISFReaderPrivate;
@@ -1147,6 +1314,10 @@ struct Init
                 compressionCodecOverride = DataBlock::LZ4HC;
             else if(compression.find("lz4") == 0)
                 compressionCodecOverride = DataBlock::LZ4;
+#ifdef HAVE_ZSTD
+            else if(compression.find("zstd") == 0)
+                compressionCodecOverride = DataBlock::ZSTD;
+#endif
 
             if(compression.find("+sh") != std::string::npos)
                 byteShuffleOverride = true;

libxisf.h

@@ -172,18 +172,22 @@ struct LIBXISF_EXPORT DataBlock
         None,
         Zlib,
         LZ4,
-        LZ4HC
+        LZ4HC,
+        ZSTD
     };
     bool embedded = false;
     uint32_t byteShuffling = 0;
     uint64_t attachmentPos = 0;
     uint64_t attachmentSize = 0;
     uint64_t uncompressedSize = 0;
+    std::vector<std::pair<uint64_t, uint64_t>> subblocks;
     CompressionCodec codec = None;
     int compressLevel = -1;
     ByteArray data;
     void decompress(const ByteArray &input, const std::string &encoding = "");
     void compress(int sampleFormatSize);
+    /// ZSTD compression can be disabled at compile time
+    static bool CompressionCodecSupported(CompressionCodec codec);
 };
 
 struct LIBXISF_EXPORT Property

libxisf.pc.in

@@ -7,7 +7,6 @@ Name: @PROJECT_NAME@
 Description: @CMAKE_PROJECT_DESCRIPTION@
 URL: @CMAKE_PROJECT_HOMEPAGE_URL@
 Version: @PROJECT_VERSION@
-Requires.private: @PC_LIBS_REQUIRE@
+Requires.private: @PC_REQUIRE_STR@
 Cflags: -I"${includedir}"
 Libs: -L"${libdir}" -lXISF
-Libs.private: -L"${libdir}" @PC_LIBS_REQUIRE@

streambuffer.cpp

@@ -89,7 +89,7 @@ std::streamsize StreamBuffer::xsgetn(char_type *s, std::streamsize n)
     std::streamsize len = egptr() - gptr();
     if(len > 0)
     {
-        std::streamsize c = std::min(n, len);
+        std::streamsize c = n < len ? n : len;
         std::memcpy(s, gptr(), c);
         gbump(c);
         ret = c;

test/benchmark.cpp

@@ -78,6 +78,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression   \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
     image.setByteshuffling(true);
     {
         image.setCompression(DataBlock::Zlib);
@@ -109,6 +120,17 @@ void benchmarkType(float avg, float stdDev)
         std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
                   << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
     }
+    if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
+    {
+        image.setCompression(DataBlock::ZSTD);
+        timer.start();
+        XISFWriter writer;
+        writer.writeImage(image);
+        ByteArray xisfImage;
+        writer.save(xisfImage);
+        std::cout << "ZSTD compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
+                  << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
+    }
 }
 
 void benchmark()

utils.cpp

@@ -52,7 +52,7 @@ void sha1(uint8_t *data, size_t len, uint8_t *hash)
     nlen += 64 - nlen % 64;
     tmp.resize(nlen, 0);
 
-    size_t ml = len * 8;
+    uint64_t ml = len * 8;
     tmp[nlen - 1] = ml & 0xff;
     tmp[nlen - 2] = ml >>  8 & 0xff;
     tmp[nlen - 3] = ml >> 16 & 0xff;

variant.cpp

@@ -108,8 +108,8 @@ static std::map<Variant::Type, const char*> idToType = {
     {Variant::Type::UI32Matrix,    "UI32Matrix"},
     {Variant::Type::I64Matrix,     "I64Matrix"},
     {Variant::Type::UI64Matrix,    "UI64Matrix"},
-    {Variant::Type::F32Matrix,     "I8Matrix"},
+    {Variant::Type::F32Matrix,     "F32Matrix"},
-    {Variant::Type::F64Matrix,     "UI8Matrix"},
+    {Variant::Type::F64Matrix,     "F64Matrix"},
 };
 
 template<typename T>
@@ -196,9 +196,12 @@ void deserializeVariant(const pugi::xml_node &node, Variant &variant, const Byte
     std::string type = node.attribute("type").as_string();
     Variant::Type typeId = typeToId[type];
 
-    if(typeId == Variant::Type::String && !node.attribute("location"))
+    if(typeId == Variant::Type::String)
     {
+        if(!node.attribute("location"))
             variant.setValue(node.text().as_string());
+        else
+            variant.setValue(String(data.constData(), data.size()));
     }
     else if(node.attribute("value"))
     {
@@ -400,6 +403,7 @@ Variant variantFromString(Variant::Type type, const String &str)
 
 Variant::Type Variant::type() const
 {
+    int idx = _value.index();
     return (Variant::Type)_value.index();
 }
 
@@ -430,6 +434,32 @@ String Variant::toString() const
         return ss.str();
     };
 
+    auto matrixString = [](auto matrix) {
+        std::stringstream ss;
+        ss << "{";
+        for(int i=0; i<matrix.rows(); i++)
+        {
+            ss << "{";
+            for(int o=0; o<matrix.cols(); o++)
+            {
+#if __GNUC__ >= 11 || __clang__
+                char str[128] = {0};
+                char *end = str + sizeof(str);
+                std::to_chars(str, end, matrix(i, o));
+                ss << str;
+#else
+                ss << std::to_string(matrix(i, o));
+#endif
+                if(o < matrix.cols() - 1)
+                    ss << ",";
+            }
+            ss << "}";
+            if(i < matrix.rows() - 1)
+                ss << ",";
+        }
+        return ss.str();
+    };
+
     switch(type())
     {
     case Variant::Type::Int8: toChars<Int8>(_value, str, end); break;
@@ -454,6 +484,8 @@ String Variant::toString() const
     case Variant::Type::UI64Vector: string = vectorString(std::get<UI64Vector>(_value)); break;
     case Variant::Type::F32Vector: string = vectorString(std::get<F32Vector>(_value)); break;
     case Variant::Type::F64Vector: string = vectorString(std::get<F64Vector>(_value)); break;
+    case Variant::Type::F32Matrix: string = matrixString(std::get<F32Matrix>(_value)); break;
+    case Variant::Type::F64Matrix: string = matrixString(std::get<F64Matrix>(_value)); break;
     case Variant::Type::String: string = std::get<String>(_value); break;
     case Variant::Type::TimePoint:
     {