From 0b0c865df02fccf1cb300553c2d1ce4d155fbaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Poizl?= Date: Mon, 3 Jul 2023 16:43:51 +0200 Subject: [PATCH] Support for ZSTD --- CMakeLists.txt | 6 ++++++ libxisf.cpp | 51 ++++++++++++++++++++++++++++++++++++++++++++++ libxisf.h | 5 ++++- test/benchmark.cpp | 22 ++++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fbd5163..01c157d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,12 @@ endif(USE_BUNDLED_LIBS) set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}) +pkg_check_modules(ZSTD libzstd IMPORTED_TARGET) +if(ZSTD_FOUND) + target_compile_definitions(XISF PRIVATE HAVE_ZSTD) + target_link_libraries(XISF PUBLIC PkgConfig::ZSTD) +endif(ZSTD_FOUND) + if(BUILD_SHARED_LIBS) target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY) else(BUILD_SHARED_LIBS) diff --git a/libxisf.cpp b/libxisf.cpp index 249fc36..a98090d 100644 --- a/libxisf.cpp +++ b/libxisf.cpp @@ -27,6 +27,9 @@ #include #include #include +#ifdef HAVE_ZSTD +#include +#endif #include "streambuffer.h" namespace LibXISF @@ -142,6 +145,15 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding) if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0) throw Error("LZ4 decompression failed"); break; + case ZSTD: +#ifdef HAVE_ZSTD + data.resize(uncompressedSize); + if(ZSTD_isError(ZSTD_decompress(data.data(), data.size(), tmp.constData(), tmp.size()))) + throw Error("ZSTD decompression failed"); +#else + throw Error("ZSTD support not compiled"); +#endif + break; } byteUnshuffle(data, byteShuffling); @@ -192,7 +204,32 @@ void DataBlock::compress(int sampleFormatSize) data.resize(compSize); break; } + case ZSTD: + { +#ifdef HAVE_ZSTD + size_t compSize = 0; + data.resize(ZSTD_compressBound(uncompressedSize)); + compSize = ZSTD_compress(data.data(), data.size(), tmp.data(), tmp.size(), compressLevel < 0 ? ZSTD_CLEVEL_DEFAULT : compressLevel); + if(ZSTD_isError(compSize)) + throw Error("ZSTD compression failed"); + + data.resize(compSize); +#else + throw Error("ZSTD support not compiled"); +#endif + break; } + } +} + +bool DataBlock::CompressionCodecSupported(CompressionCodec codec) +{ + (void)codec; +#ifndef HAVE_ZSTD + if(codec == ZSTD) + return false; +#endif + return true; } Property::Property(const String &_id, const char *_value) : @@ -663,6 +700,10 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock & dataBlock.codec = DataBlock::LZ4HC; else if(compression[0].find("lz4") == 0) dataBlock.codec = DataBlock::LZ4; +#ifdef HAVE_ZSTD + else if(compression[0].find("zstd") == 0) + dataBlock.codec = DataBlock::ZSTD; +#endif else throw Error("Unknown compression codec"); @@ -988,6 +1029,12 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con codec = "lz4"; else if(dataBlock.codec == DataBlock::LZ4HC) codec = "lz4hc"; + else if(dataBlock.codec == DataBlock::ZSTD) +#ifdef HAVE_ZSTD + codec = "zstd"; +#else + throw Error("ZSTD support not compiled"); +#endif if(dataBlock.byteShuffling > 1) codec += "+sh"; @@ -1149,6 +1196,10 @@ struct Init compressionCodecOverride = DataBlock::LZ4HC; else if(compression.find("lz4") == 0) compressionCodecOverride = DataBlock::LZ4; +#ifdef HAVE_ZSTD + else if(compression.find("zstd") == 0) + compressionCodecOverride = DataBlock::ZSTD; +#endif if(compression.find("+sh") != std::string::npos) byteShuffleOverride = true; diff --git a/libxisf.h b/libxisf.h index 89c5e14..1ea60c8 100644 --- a/libxisf.h +++ b/libxisf.h @@ -172,7 +172,8 @@ struct LIBXISF_EXPORT DataBlock None, Zlib, LZ4, - LZ4HC + LZ4HC, + ZSTD }; bool embedded = false; uint32_t byteShuffling = 0; @@ -184,6 +185,8 @@ struct LIBXISF_EXPORT DataBlock ByteArray data; void decompress(const ByteArray &input, const std::string &encoding = ""); void compress(int sampleFormatSize); + /// ZSTD compression can be disabled at compile time + static bool CompressionCodecSupported(CompressionCodec codec); }; struct LIBXISF_EXPORT Property diff --git a/test/benchmark.cpp b/test/benchmark.cpp index f53b3c1..82235cc 100644 --- a/test/benchmark.cpp +++ b/test/benchmark.cpp @@ -78,6 +78,17 @@ void benchmarkType(float avg, float stdDev) std::cout << "LZ4HC compression \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; } + if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD)) + { + image.setCompression(DataBlock::ZSTD); + timer.start(); + XISFWriter writer; + writer.writeImage(image); + ByteArray xisfImage; + writer.save(xisfImage); + std::cout << "ZSTD compression \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " + << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; + } image.setByteshuffling(true); { image.setCompression(DataBlock::Zlib); @@ -109,6 +120,17 @@ void benchmarkType(float avg, float stdDev) std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; } + if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD)) + { + image.setCompression(DataBlock::ZSTD); + timer.start(); + XISFWriter writer; + writer.writeImage(image); + ByteArray xisfImage; + writer.save(xisfImage); + std::cout << "ZSTD compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " + << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; + } } void benchmark()