From cf9c903a90c2a795ce1057c99a33669f40aaaabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Sat, 20 Jan 2024 08:02:10 +0100 Subject: [PATCH 01/10] Update version in CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb1580d..b15b13b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.14) -project(libXISF VERSION 0.2.10 LANGUAGES CXX C +project(libXISF VERSION 0.2.11 LANGUAGES CXX C HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.") From 033a34e2486223c358f60f7f7d532890e7afb9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Mon, 22 Jan 2024 21:23:30 +0100 Subject: [PATCH 02/10] Add F32Matrix and F64Matrix into toString --- variant.cpp | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/variant.cpp b/variant.cpp index 7476633..1cbf6fd 100644 --- a/variant.cpp +++ b/variant.cpp @@ -108,8 +108,8 @@ static std::map idToType = { {Variant::Type::UI32Matrix, "UI32Matrix"}, {Variant::Type::I64Matrix, "I64Matrix"}, {Variant::Type::UI64Matrix, "UI64Matrix"}, - {Variant::Type::F32Matrix, "I8Matrix"}, - {Variant::Type::F64Matrix, "UI8Matrix"}, + {Variant::Type::F32Matrix, "F32Matrix"}, + {Variant::Type::F64Matrix, "F64Matrix"}, }; template @@ -403,6 +403,7 @@ Variant variantFromString(Variant::Type type, const String &str) Variant::Type Variant::type() const { + int idx = _value.index(); return (Variant::Type)_value.index(); } @@ -433,6 +434,28 @@ String Variant::toString() const return ss.str(); }; + auto matrixString = [](auto matrix) { + std::stringstream ss; + ss << "{"; + for(int i=0; i(_value, str, end); break; @@ -457,6 +480,8 @@ String Variant::toString() const case Variant::Type::UI64Vector: string = vectorString(std::get(_value)); break; case Variant::Type::F32Vector: string = vectorString(std::get(_value)); break; case Variant::Type::F64Vector: string = vectorString(std::get(_value)); break; + case Variant::Type::F32Matrix: string = matrixString(std::get(_value)); break; + case Variant::Type::F64Matrix: string = matrixString(std::get(_value)); break; case Variant::Type::String: string = std::get(_value); break; case Variant::Type::TimePoint: { From bf690339b494fdd84254420170bd84fa4fefab51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Thu, 8 Feb 2024 22:51:08 +0100 Subject: [PATCH 03/10] Workaround for gcc 10 --- variant.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/variant.cpp b/variant.cpp index 1cbf6fd..bb8ba75 100644 --- a/variant.cpp +++ b/variant.cpp @@ -442,10 +442,14 @@ String Variant::toString() const ss << "{"; for(int o=0; o= 11 || __clang__ char str[128] = {0}; char *end = str + sizeof(str); std::to_chars(str, end, matrix(i, o)); ss << str; +#else + ss << std::to_string(matrix(i, o)); +#endif if(o < matrix.cols() - 1) ss << ","; } From e3ba8eedd883f68f6bdb07084d7866309ebf96f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Tue, 12 Mar 2024 23:03:54 +0100 Subject: [PATCH 04/10] Use std::stoull --- libxisf.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libxisf.cpp b/libxisf.cpp index 86fd1de..048fa63 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -711,7 +711,7 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock & else throw Error("Unknown compression codec"); - dataBlock.uncompressedSize = std::stoul(compression[1]); + dataBlock.uncompressedSize = std::stoull(compression[1]); if(compression[0].find("+sh") != std::string::npos) { @@ -741,8 +741,8 @@ DataBlock XISFReaderPrivate::parseDataBlock(const pugi::xml_node &node) } else if(location.size() >= 3 && location[0] == "attachment") { - dataBlock.attachmentPos = std::stoul(location[1]); - dataBlock.attachmentSize = std::stoul(location[2]); + dataBlock.attachmentPos = std::stoull(location[1]); + dataBlock.attachmentSize = std::stoull(location[2]); } else { @@ -818,9 +818,9 @@ Image XISFReaderPrivate::parseImage(const pugi::xml_node &node) std::vector geometry = splitString(node.attribute("geometry").as_string(), ':'); if(geometry.size() != 3)throw Error("We support only 2D images"); - image._width = std::stoul(geometry[0]); - image._height = std::stoul(geometry[1]); - image._channelCount = std::stoul(geometry[2]); + image._width = std::stoull(geometry[0]); + image._height = std::stoull(geometry[1]); + image._channelCount = std::stoull(geometry[2]); if(!image._width || !image._height || !image._channelCount)throw Error("Invalid image geometry"); std::vector bounds = splitString(node.attribute("bounds").as_string(), ':'); From 67887bd4c8becae4bfa18b9bfbf7c952f61ea1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Thu, 14 Mar 2024 22:34:16 +0100 Subject: [PATCH 05/10] Workaround for windows where it can't read more than 2 GiB --- libxisf.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/libxisf.cpp b/libxisf.cpp index 048fa63..feef5c8 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -49,6 +49,7 @@ static std::unordered_map colorSpaceToString; static DataBlock::CompressionCodec compressionCodecOverride = DataBlock::None; static bool byteShuffleOverride = false; static int compressionLevelOverride = -1; +const size_t GiB = 1073741824; static const std::unordered_map> fitsNameToPropertyIdTypeConvert = { {"OBSERVER", {"Observer:Name", Variant::Type::String}}, @@ -864,7 +865,15 @@ void XISFReaderPrivate::readAttachment(DataBlock &dataBlock) { ByteArray data(dataBlock.attachmentSize); _io->seekg(dataBlock.attachmentPos); - _io->read(data.data(), dataBlock.attachmentSize); + size_t size = dataBlock.attachmentSize; + char *ptr = data.data(); + while(size > 0) + { + size_t s = std::min(size, GiB); + _io->read(ptr, s); + size -= s; + ptr += s; + } dataBlock.decompress(data); } @@ -913,7 +922,15 @@ void XISFWriterPrivate::save(std::ostream &io) for(auto &image : _images) { - io.write(image._dataBlock.data.constData(), image._dataBlock.data.size()); + const char *ptr = image._dataBlock.data.constData(); + size_t size = image._dataBlock.data.size(); + while(size > 0) + { + size_t s = std::min(size, GiB); + io.write(ptr, s); + ptr += s; + size -= s; + } } } From 899fe982335f0ae33bbd0d5797480919423120ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Sun, 17 Mar 2024 14:14:03 +0100 Subject: [PATCH 06/10] Fix bug with writing ICCProfile --- libxisf.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libxisf.cpp b/libxisf.cpp index feef5c8..de6fd9e 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -1019,7 +1019,8 @@ void XISFWriterPrivate::writeImageElement(pugi::xml_node &node, const Image &ima if(image._iccProfile.size()) { ByteArray base64 = image._iccProfile; - base64.decodeBase64(); + base64.encodeBase64(); + base64.append('\0'); pugi::xml_node icc_node = image_node.append_child("ICCProfile"); icc_node.append_attribute("location").set_value("inline:base64"); icc_node.append_child(pugi::node_pcdata).set_value(base64.data()); From 4db1d865305405627d9db9579cb2f1dde1cad8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Sun, 17 Mar 2024 14:40:54 +0100 Subject: [PATCH 07/10] Add support for subblocks to handle +2GiB compressed images --- libxisf.cpp | 116 +++++++++++++++++++++++++++++++++++++++++++--------- libxisf.h | 1 + 2 files changed, 98 insertions(+), 19 deletions(-) diff --git a/libxisf.cpp b/libxisf.cpp index de6fd9e..004b256 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -128,6 +128,9 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding) else if(encoding == "base16") tmp.decodeHex(); + if(subblocks.size() == 0) + subblocks.push_back({tmp.size(), uncompressedSize}); + switch(codec) { case None: @@ -136,26 +139,54 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding) case Zlib: { data.resize(uncompressedSize); - uLongf size = uncompressedSize; - ::uncompress((Bytef*)data.data(), &size, (Bytef*)tmp.data(), tmp.size()); + const char *srcPtr = tmp.constData(); + char *dstPtr = data.data(); + for(auto &block : subblocks) + { + uLongf size = block.second; + ::uncompress((Bytef*)dstPtr, &size, (const Bytef*)srcPtr, block.first); + srcPtr += block.first; + dstPtr += block.second; + } break; } case LZ4: case LZ4HC: + { data.resize(uncompressedSize); - if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0) - throw Error("LZ4 decompression failed"); + const char *srcPtr = tmp.constData(); + char *dstPtr = data.data(); + for(auto &block : subblocks) + { + if(LZ4_decompress_safe(srcPtr, dstPtr, block.first, block.second) < 0) + throw Error("LZ4 decompression failed"); + srcPtr += block.first; + dstPtr += block.second; + } break; + } case ZSTD: #ifdef HAVE_ZSTD + { data.resize(uncompressedSize); - if(ZSTD_isError(ZSTD_decompress(data.data(), data.size(), tmp.constData(), tmp.size()))) - throw Error("ZSTD decompression failed"); + const char *srcPtr = tmp.constData(); + char *dstPtr = data.data(); + for(auto &block : subblocks) + { + if(ZSTD_isError(ZSTD_decompress(dstPtr, block.second, srcPtr, block.first))) + throw Error("ZSTD decompression failed"); + srcPtr += block.first; + dstPtr += block.second; + } + #else throw Error("ZSTD support not compiled"); #endif break; } + } + + subblocks.clear(); byteUnshuffle(data, byteShuffling); attachmentPos = 0; @@ -182,26 +213,48 @@ void DataBlock::compress(int sampleFormatSize) break; case Zlib: { - data.resize(compressBound(uncompressedSize)); - uLongf compressedSize = data.size(); - if(::compress2((Bytef*)data.data(), &compressedSize, (Bytef*)tmp.data(), tmp.size(), compressLevel) != Z_OK) - throw Error("Zlib compression failed"); - data.resize(compressedSize); + int64_t size = tmp.size(); + int64_t compSize = 0; + int64_t inPtr = 0; + while(inPtr < size) + { + int64_t inSize = UINT32_MAX < size - inPtr ? UINT32_MAX : size - inPtr; + data.resize(compSize + compressBound(inSize)); + uLongf outSize = data.size() - compSize; + if(::compress2((Bytef*)data.data() + compSize, &outSize, (const Bytef*)tmp.constData() + inPtr, inSize, compressLevel) != Z_OK) + throw Error("Zlib compression failed"); + + compSize += outSize; + inPtr += inSize; + subblocks.push_back({outSize, inSize}); + } + data.resize(compSize); break; } case LZ4: case LZ4HC: { - int compSize = 0; - data.resize(LZ4_compressBound(tmp.size())); - if(codec == LZ4) - compSize = LZ4_compress_default(tmp.constData(), data.data(), tmp.size(), data.size()); - else - compSize = LZ4_compress_HC(tmp.constData(), data.data(), tmp.size(), data.size(), compressLevel < 0 ? LZ4HC_CLEVEL_DEFAULT : compressLevel); + int64_t size = tmp.size(); + int64_t compSize = 0; + int64_t inPtr = 0; + while(inPtr < size) + { + int64_t inSize = LZ4_MAX_INPUT_SIZE < size - inPtr ? LZ4_MAX_INPUT_SIZE : size - inPtr; + data.resize(compSize + LZ4_compressBound(inSize)); + int outSize = 0; - if(compSize <= 0) - throw Error("LZ4 compression failed"); + if(codec == LZ4) + outSize = LZ4_compress_default(tmp.constData() + inPtr, data.data() + compSize, inSize, data.size() - compSize); + else + outSize = LZ4_compress_HC(tmp.constData() + inPtr, data.data() + compSize, inSize, data.size() - compSize, compressLevel < 0 ? LZ4HC_CLEVEL_DEFAULT : compressLevel); + if(outSize <= 0) + throw Error("LZ4 compression failed"); + + compSize += outSize; + inPtr += inSize; + subblocks.push_back({outSize, inSize}); + } data.resize(compSize); break; } @@ -721,6 +774,18 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock & else throw Error("Missing byte shuffling size"); } + + if(node.attribute("subblocks")) + { + std::vector subblocks = splitString(node.attribute("subblocks").as_string(), ':'); + for(auto &block : subblocks) + { + size_t pos = 0; + size_t comp = std::stoull(block, &pos); + size_t deco = std::stoull(block.substr(pos+1)); + dataBlock.subblocks.push_back({comp, deco}); + } + } } } @@ -1069,6 +1134,19 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con image_node.append_attribute("compression").set_value(codec.c_str()); } + + if(!dataBlock.subblocks.empty()) + { + std::string subblocks; + for(auto i = dataBlock.subblocks.begin(); i != dataBlock.subblocks.end(); i++) + { + if(i != dataBlock.subblocks.begin()) + subblocks += ":"; + + subblocks += std::to_string(i->first) + "," + std::to_string(i->second); + } + image_node.append_attribute("subblocks").set_value(subblocks.c_str()); + } } void XISFWriterPrivate::writePropertyElement(pugi::xml_node &node, const Property &property) diff --git a/libxisf.h b/libxisf.h index 1ea60c8..83cdde3 100644 --- a/libxisf.h +++ b/libxisf.h @@ -180,6 +180,7 @@ struct LIBXISF_EXPORT DataBlock uint64_t attachmentPos = 0; uint64_t attachmentSize = 0; uint64_t uncompressedSize = 0; + std::vector> subblocks; CompressionCodec codec = None; int compressLevel = -1; ByteArray data; From c1e986080caa5e8a6de02de2d6931c08ba592738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Thu, 21 Mar 2024 16:52:25 +0100 Subject: [PATCH 08/10] Don't use replace to update attachement position --- libxisf.cpp | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/libxisf.cpp b/libxisf.cpp index 004b256..97c687e 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -956,6 +956,7 @@ private: void writePropertyElement(pugi::xml_node &node, const Property &property); void writeFITSKeyword(pugi::xml_node &node, const FITSKeyword &keyword); void writeMetadata(pugi::xml_node &node); + void updateImageAttachmentPos(pugi::xml_node &root, size_t offset); ByteArray _xisfHeader; ByteArray _attachmentsData; std::vector _images; @@ -1026,25 +1027,26 @@ void XISFWriterPrivate::writeHeader() writeMetadata(root); - std::stringstream xml; - xml.write(signature, sizeof(signature)); - doc.save(xml, "", pugi::format_raw); - - std::string header = xml.str(); - uint32_t size = header.size(); - - uint32_t offset = 0; - std::string replace = "attachment:2147483648"; - for(auto &image : _images) + uint32_t size = 0; + std::string header; + while(true) { - std::string blockPos = std::string("attachment:") + std::to_string(size + offset); - size_t pos = header.find(replace, 32); - header.replace(pos, replace.size(), blockPos); - offset += image._dataBlock.data.size(); + std::stringstream xml; + xml.write(signature, sizeof(signature)); + doc.save(xml, "", pugi::format_raw); + header = xml.str(); + if(size != header.size()) + { + size = header.size(); + updateImageAttachmentPos(root, size); + } + else + { + break; + } } uint32_t headerSize = header.size() - sizeof(signature); - header.resize(size, 0); header.replace(8, sizeof(uint32_t), (const char*)&headerSize, sizeof(uint32_t)); _xisfHeader = ByteArray(header.c_str(), header.size()); @@ -1104,7 +1106,10 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con } else { - std::string attachment = "attachment:2147483648:" + std::to_string(dataBlock.data.size()); + std::string attachment = "attachment:"; + if(dataBlock.attachmentPos == 0) attachment += "99999"; + else attachment += std::to_string(dataBlock.attachmentPos); + attachment += ":" + std::to_string(dataBlock.data.size()); image_node.append_attribute("location").set_value(attachment.c_str()); } @@ -1177,6 +1182,19 @@ void XISFWriterPrivate::writeMetadata(pugi::xml_node &node) writePropertyElement(metadata, Property("XISF:CreatorApplication", "LibXISF")); } +void XISFWriterPrivate::updateImageAttachmentPos(pugi::xml_node &root, size_t offset) +{ + pugi::xpath_node_set imageNodes = root.select_nodes("//Image"); + int i = 0; + for(auto &image : _images) + { + pugi::xml_node node = imageNodes[i++].node(); + std::string location = "attachment:" + std::to_string(offset) + ":" + std::to_string(image._dataBlock.data.size()); + offset += image._dataBlock.data.size(); + node.attribute("location").set_value(location.c_str()); + } +} + XISFReader::XISFReader() { p = new XISFReaderPrivate; From f704b9f041a8e285f5f578d39053ec9c5de0ad9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Thu, 21 Mar 2024 18:12:25 +0100 Subject: [PATCH 09/10] Fix wrong bracket position --- libxisf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libxisf.cpp b/libxisf.cpp index 97c687e..46177b4 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -179,12 +179,12 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding) dstPtr += block.second; } + } #else throw Error("ZSTD support not compiled"); #endif break; } - } subblocks.clear(); From 87d65a31fc9571e132394f131c1b6defff5e5325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Sat, 23 Mar 2024 09:21:33 +0100 Subject: [PATCH 10/10] Release 0.2.12 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b15b13b..9e945e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.14) -project(libXISF VERSION 0.2.11 LANGUAGES CXX C +project(libXISF VERSION 0.2.12 LANGUAGES CXX C HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.")