9 Commits

Author SHA1 Message Date
nou aa356443d3 Convert aperture and focal length from mm to meters. 2024-01-05 16:15:30 +01:00
nou 16676540b4 Fix not loading String properties 2024-01-05 16:14:29 +01:00
nou bae3edd12a Resolve some erros for MSVC 2023-12-20 15:28:43 +01:00
nou c848ba75ff Fix typo in README 2023-12-02 14:35:01 +01:00
nou 8a1f305cc7 Fix incorrect header size 2023-10-16 19:17:04 +02:00
nou 8e05a58610 Version 0.2.9 2023-08-13 19:13:39 +02:00
nou 0b0c865df0 Support for ZSTD 2023-07-03 17:31:29 +02:00
nou a675e97e0b Release 0.2.8 2023-06-13 12:25:13 +02:00
nou 6028354c61 Remove linking against zstd 2023-06-12 23:41:48 +02:00
8 changed files with 101 additions and 14 deletions
+7 -3
View File
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.14) cmake_minimum_required(VERSION 3.14)
project(libXISF VERSION 0.2.7 LANGUAGES CXX C project(libXISF VERSION 0.2.10 LANGUAGES CXX C
HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF HOMEPAGE_URL https://gitea.nouspiro.space/nou/libXISF
DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.") DESCRIPTION "LibXISF is C++ library that can read and write XISF files produced by PixInsight.")
@@ -92,10 +92,14 @@ else(USE_BUNDLED_LIBS)
list(APPEND PC_LIBS_REQUIRE lz4 pugixml zlib) list(APPEND PC_LIBS_REQUIRE lz4 pugixml zlib)
endif(USE_BUNDLED_LIBS) endif(USE_BUNDLED_LIBS)
target_link_libraries(XISF PUBLIC zstd)
set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR}) set_target_properties(XISF PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
pkg_check_modules(ZSTD libzstd IMPORTED_TARGET)
if(ZSTD_FOUND)
target_compile_definitions(XISF PRIVATE HAVE_ZSTD)
target_link_libraries(XISF PUBLIC PkgConfig::ZSTD)
endif(ZSTD_FOUND)
if(BUILD_SHARED_LIBS) if(BUILD_SHARED_LIBS)
target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY) target_compile_definitions(XISF PRIVATE LIBXISF_LIBRARY)
else(BUILD_SHARED_LIBS) else(BUILD_SHARED_LIBS)
+1 -1
View File
@@ -9,7 +9,7 @@ To compile simply run these commands
``` ```
cmake -B build -S . cmake -B build -S .
cmake --build build --parallel cmake --build build --parallel
cmake --install . cmake --install build
``` ```
By default it use bundled libraries. If you wish to use external libraries you will may add By default it use bundled libraries. If you wish to use external libraries you will may add
+60 -5
View File
@@ -27,6 +27,9 @@
#include <lz4hc.h> #include <lz4hc.h>
#include <pugixml.hpp> #include <pugixml.hpp>
#include <zlib.h> #include <zlib.h>
#ifdef HAVE_ZSTD
#include <zstd.h>
#endif
#include "streambuffer.h" #include "streambuffer.h"
namespace LibXISF namespace LibXISF
@@ -83,12 +86,12 @@ static void byteShuffle(ByteArray &data, int itemSize)
{ {
ByteArray &input = data; ByteArray &input = data;
ByteArray output(input.size()); ByteArray output(input.size());
int num = input.size() / itemSize; size_t num = input.size() / itemSize;
char *s = output.data(); char *s = output.data();
for(int i=0; i<itemSize; i++) for(int i=0; i<itemSize; i++)
{ {
const char *u = input.constData() + i; const char *u = input.constData() + i;
for(int o=0; o<num; o++, s++, u += itemSize) for(size_t o=0; o<num; o++, s++, u += itemSize)
*s = *u; *s = *u;
} }
memcpy(s, input.constData() + num * itemSize, input.size() % itemSize); memcpy(s, input.constData() + num * itemSize, input.size() % itemSize);
@@ -102,12 +105,12 @@ static void byteUnshuffle(ByteArray &data, int itemSize)
{ {
ByteArray &input = data; ByteArray &input = data;
ByteArray output(input.size()); ByteArray output(input.size());
int num = input.size() / itemSize; size_t num = input.size() / itemSize;
const char *s = input.constData(); const char *s = input.constData();
for(int i=0; i<itemSize; i++) for(int i=0; i<itemSize; i++)
{ {
char *u = output.data() + i; char *u = output.data() + i;
for(int o=0; o<num; o++, s++, u += itemSize) for(size_t o=0; o<num; o++, s++, u += itemSize)
*u = *s; *u = *s;
} }
memcpy(output.data() + num * itemSize, s, input.size() % itemSize); memcpy(output.data() + num * itemSize, s, input.size() % itemSize);
@@ -142,6 +145,15 @@ void DataBlock::decompress(const ByteArray &input, const String &encoding)
if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0) if(LZ4_decompress_safe(tmp.constData(), data.data(), tmp.size(), data.size()) < 0)
throw Error("LZ4 decompression failed"); throw Error("LZ4 decompression failed");
break; break;
case ZSTD:
#ifdef HAVE_ZSTD
data.resize(uncompressedSize);
if(ZSTD_isError(ZSTD_decompress(data.data(), data.size(), tmp.constData(), tmp.size())))
throw Error("ZSTD decompression failed");
#else
throw Error("ZSTD support not compiled");
#endif
break;
} }
byteUnshuffle(data, byteShuffling); byteUnshuffle(data, byteShuffling);
@@ -192,7 +204,32 @@ void DataBlock::compress(int sampleFormatSize)
data.resize(compSize); data.resize(compSize);
break; break;
} }
case ZSTD:
{
#ifdef HAVE_ZSTD
size_t compSize = 0;
data.resize(ZSTD_compressBound(uncompressedSize));
compSize = ZSTD_compress(data.data(), data.size(), tmp.data(), tmp.size(), compressLevel < 0 ? ZSTD_CLEVEL_DEFAULT : compressLevel);
if(ZSTD_isError(compSize))
throw Error("ZSTD compression failed");
data.resize(compSize);
#else
throw Error("ZSTD support not compiled");
#endif
break;
} }
}
}
bool DataBlock::CompressionCodecSupported(CompressionCodec codec)
{
(void)codec;
#ifndef HAVE_ZSTD
if(codec == ZSTD)
return false;
#endif
return true;
} }
Property::Property(const String &_id, const char *_value) : Property::Property(const String &_id, const char *_value) :
@@ -352,6 +389,10 @@ bool Image::addFITSKeywordAsProperty(const String &name, const String &value)
{ {
auto &c = fitsNameToPropertyIdTypeConvert.at(name); auto &c = fitsNameToPropertyIdTypeConvert.at(name);
Property prop(c.first, variantFromString(c.second, value)); Property prop(c.first, variantFromString(c.second, value));
if(name == "APTDIA" || name == "FOCALLEN")
prop.value.value<LibXISF::Float32>() /= 1000.0f;
updateProperty(prop); updateProperty(prop);
return true; return true;
} }
@@ -663,6 +704,10 @@ void XISFReaderPrivate::parseCompression(const pugi::xml_node &node, DataBlock &
dataBlock.codec = DataBlock::LZ4HC; dataBlock.codec = DataBlock::LZ4HC;
else if(compression[0].find("lz4") == 0) else if(compression[0].find("lz4") == 0)
dataBlock.codec = DataBlock::LZ4; dataBlock.codec = DataBlock::LZ4;
#ifdef HAVE_ZSTD
else if(compression[0].find("zstd") == 0)
dataBlock.codec = DataBlock::ZSTD;
#endif
else else
throw Error("Unknown compression codec"); throw Error("Unknown compression codec");
@@ -916,7 +961,7 @@ void XISFWriterPrivate::writeHeader()
offset += image._dataBlock.data.size(); offset += image._dataBlock.data.size();
} }
uint32_t headerSize = size - sizeof(signature); uint32_t headerSize = header.size() - sizeof(signature);
header.resize(size, 0); header.resize(size, 0);
header.replace(8, sizeof(uint32_t), (const char*)&headerSize, sizeof(uint32_t)); header.replace(8, sizeof(uint32_t), (const char*)&headerSize, sizeof(uint32_t));
@@ -988,6 +1033,12 @@ void XISFWriterPrivate::writeDataBlockAttributes(pugi::xml_node &image_node, con
codec = "lz4"; codec = "lz4";
else if(dataBlock.codec == DataBlock::LZ4HC) else if(dataBlock.codec == DataBlock::LZ4HC)
codec = "lz4hc"; codec = "lz4hc";
else if(dataBlock.codec == DataBlock::ZSTD)
#ifdef HAVE_ZSTD
codec = "zstd";
#else
throw Error("ZSTD support not compiled");
#endif
if(dataBlock.byteShuffling > 1) if(dataBlock.byteShuffling > 1)
codec += "+sh"; codec += "+sh";
@@ -1149,6 +1200,10 @@ struct Init
compressionCodecOverride = DataBlock::LZ4HC; compressionCodecOverride = DataBlock::LZ4HC;
else if(compression.find("lz4") == 0) else if(compression.find("lz4") == 0)
compressionCodecOverride = DataBlock::LZ4; compressionCodecOverride = DataBlock::LZ4;
#ifdef HAVE_ZSTD
else if(compression.find("zstd") == 0)
compressionCodecOverride = DataBlock::ZSTD;
#endif
if(compression.find("+sh") != std::string::npos) if(compression.find("+sh") != std::string::npos)
byteShuffleOverride = true; byteShuffleOverride = true;
+4 -1
View File
@@ -172,7 +172,8 @@ struct LIBXISF_EXPORT DataBlock
None, None,
Zlib, Zlib,
LZ4, LZ4,
LZ4HC LZ4HC,
ZSTD
}; };
bool embedded = false; bool embedded = false;
uint32_t byteShuffling = 0; uint32_t byteShuffling = 0;
@@ -184,6 +185,8 @@ struct LIBXISF_EXPORT DataBlock
ByteArray data; ByteArray data;
void decompress(const ByteArray &input, const std::string &encoding = ""); void decompress(const ByteArray &input, const std::string &encoding = "");
void compress(int sampleFormatSize); void compress(int sampleFormatSize);
/// ZSTD compression can be disabled at compile time
static bool CompressionCodecSupported(CompressionCodec codec);
}; };
struct LIBXISF_EXPORT Property struct LIBXISF_EXPORT Property
+1 -1
View File
@@ -89,7 +89,7 @@ std::streamsize StreamBuffer::xsgetn(char_type *s, std::streamsize n)
std::streamsize len = egptr() - gptr(); std::streamsize len = egptr() - gptr();
if(len > 0) if(len > 0)
{ {
std::streamsize c = std::min(n, len); std::streamsize c = n < len ? n : len;
std::memcpy(s, gptr(), c); std::memcpy(s, gptr(), c);
gbump(c); gbump(c);
ret = c; ret = c;
+22
View File
@@ -78,6 +78,17 @@ void benchmarkType(float avg, float stdDev)
std::cout << "LZ4HC compression \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " std::cout << "LZ4HC compression \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
<< size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
} }
if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
{
image.setCompression(DataBlock::ZSTD);
timer.start();
XISFWriter writer;
writer.writeImage(image);
ByteArray xisfImage;
writer.save(xisfImage);
std::cout << "ZSTD compression \tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
<< size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
}
image.setByteshuffling(true); image.setByteshuffling(true);
{ {
image.setCompression(DataBlock::Zlib); image.setCompression(DataBlock::Zlib);
@@ -109,6 +120,17 @@ void benchmarkType(float avg, float stdDev)
std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: " std::cout << "LZ4HC compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
<< size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl; << size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
} }
if(DataBlock::CompressionCodecSupported(DataBlock::ZSTD))
{
image.setCompression(DataBlock::ZSTD);
timer.start();
XISFWriter writer;
writer.writeImage(image);
ByteArray xisfImage;
writer.save(xisfImage);
std::cout << "ZSTD compression SH\tElapsed time: " << timer.elapsed() << " " << "ms\tSpeed: "
<< size/1024.0/1.024/timer.elapsed() << "MiB/s\tRatio: " << baseSize/xisfImage.size() << std::endl;
}
} }
void benchmark() void benchmark()
+1 -1
View File
@@ -52,7 +52,7 @@ void sha1(uint8_t *data, size_t len, uint8_t *hash)
nlen += 64 - nlen % 64; nlen += 64 - nlen % 64;
tmp.resize(nlen, 0); tmp.resize(nlen, 0);
size_t ml = len * 8; uint64_t ml = len * 8;
tmp[nlen - 1] = ml & 0xff; tmp[nlen - 1] = ml & 0xff;
tmp[nlen - 2] = ml >> 8 & 0xff; tmp[nlen - 2] = ml >> 8 & 0xff;
tmp[nlen - 3] = ml >> 16 & 0xff; tmp[nlen - 3] = ml >> 16 & 0xff;
+4 -1
View File
@@ -196,9 +196,12 @@ void deserializeVariant(const pugi::xml_node &node, Variant &variant, const Byte
std::string type = node.attribute("type").as_string(); std::string type = node.attribute("type").as_string();
Variant::Type typeId = typeToId[type]; Variant::Type typeId = typeToId[type];
if(typeId == Variant::Type::String && !node.attribute("location")) if(typeId == Variant::Type::String)
{ {
if(!node.attribute("location"))
variant.setValue(node.text().as_string()); variant.setValue(node.text().as_string());
else
variant.setValue(String(data.constData(), data.size()));
} }
else if(node.attribute("value")) else if(node.attribute("value"))
{ {