nou/libXISF
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 11 Wiki Activity

Update pugi, lz4 and zlib

Browse Source
This commit is contained in:
Dušan Poizl
2025-01-25 14:55:30 +01:00
parent 9a32138f6a
commit c7e931db24
101 changed files with 5467 additions and 5157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lz4/README.md
+8 -2
View File
@@ -108,6 +108,12 @@ The following build macro can be selected to adjust source code behavior at comp
Remove support of dynamic memory allocation.
For more details, see description of this macro in `lib/lz4.c`.
- `LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT` : experimental feature aimed at producing the same
compressed output on platforms of different endianness (i.e. little-endian and big-endian).
Output on little-endian platforms shall remain unchanged, while big-endian platforms will start producing
the same output as little-endian ones. This isn't expected to impact backward- and forward-compatibility
in any way.
- `LZ4_FREESTANDING` : by setting this build macro to 1,
LZ4/HC removes dependencies on the C standard library,
including allocation functions and `memmove()`, `memcpy()`, and `memset()`.
@@ -131,8 +137,8 @@ The following build macro can be selected to adjust source code behavior at comp
#### Makefile variables
The following `Makefile` variables can be selected to alter the profile of produced binaries :
- `BUILD_SHARED` : generate `libzstd` dynamic library (enabled by default)
- `BUILD_STATIC` : generate `libzstd` static library (enabled by default)
- `BUILD_SHARED` : generate `liblz4` dynamic library (enabled by default)
- `BUILD_STATIC` : generate `liblz4` static library (enabled by default)
#### Amalgamation
lz4/lz4.c
+129 -51
View File
@@ -1,6 +1,6 @@
/*
LZ4 - Fast LZ compression algorithm
Copyright (C) 2011-2020, Yann Collet.
Copyright (C) 2011-2023, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -79,7 +79,7 @@
( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) \
|| defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define LZ4_FORCE_MEMORY_ACCESS 2
# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__)
# elif (defined(__INTEL_COMPILER) && !defined(_WIN32)) || defined(__GNUC__) || defined(_MSC_VER)
# define LZ4_FORCE_MEMORY_ACCESS 1
# endif
#endif
@@ -106,15 +106,13 @@
# define LZ4_SRC_INCLUDED 1
#endif
#ifndef LZ4_STATIC_LINKING_ONLY
#define LZ4_STATIC_LINKING_ONLY
#endif
#ifndef LZ4_DISABLE_DEPRECATE_WARNINGS
#define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
# define LZ4_DISABLE_DEPRECATE_WARNINGS /* due to LZ4_decompress_safe_withPrefix64k */
#endif
#define LZ4_STATIC_LINKING_ONLY /* LZ4_DISTANCE_MAX */
#ifndef LZ4_STATIC_LINKING_ONLY
# define LZ4_STATIC_LINKING_ONLY
#endif
#include "lz4.h"
/* see also "memory routines" below */
@@ -126,14 +124,17 @@
# include <intrin.h> /* only present in VS2005+ */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 6237) /* disable: C6237: conditional expression is always 0 */
# pragma warning(disable : 6239) /* disable: C6239: (<non-zero constant> && <expression>) always evaluates to the result of <expression> */
# pragma warning(disable : 6240) /* disable: C6240: (<expression> && <non-zero constant>) always evaluates to the result of <expression> */
# pragma warning(disable : 6326) /* disable: C6326: Potential comparison of a constant with another constant */
#endif /* _MSC_VER */
#ifndef LZ4_FORCE_INLINE
# ifdef _MSC_VER /* Visual Studio */
# if defined (_MSC_VER) && !defined (__clang__) /* MSVC */
# define LZ4_FORCE_INLINE static __forceinline
# else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# if defined (__GNUC__) || defined (__clang__)
# define LZ4_FORCE_INLINE static inline __attribute__((always_inline))
# else
# define LZ4_FORCE_INLINE static inline
@@ -365,6 +366,11 @@ static unsigned LZ4_isLittleEndian(void)
return one.c[0];
}
#if defined(__GNUC__) || defined(__INTEL_COMPILER)
#define LZ4_PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__))
#elif defined(_MSC_VER)
#define LZ4_PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop))
#endif
#if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2)
/* lie to the compiler about data alignment; use with caution */
@@ -380,9 +386,9 @@ static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
typedef struct { U16 u16; } __attribute__((packed)) LZ4_unalign16;
typedef struct { U32 u32; } __attribute__((packed)) LZ4_unalign32;
typedef struct { reg_t uArch; } __attribute__((packed)) LZ4_unalignST;
LZ4_PACK(typedef struct { U16 u16; }) LZ4_unalign16;
LZ4_PACK(typedef struct { U32 u32; }) LZ4_unalign32;
LZ4_PACK(typedef struct { reg_t uArch; }) LZ4_unalignST;
static U16 LZ4_read16(const void* ptr) { return ((const LZ4_unalign16*)ptr)->u16; }
static U32 LZ4_read32(const void* ptr) { return ((const LZ4_unalign32*)ptr)->u32; }
@@ -427,10 +433,22 @@ static U16 LZ4_readLE16(const void* memPtr)
return LZ4_read16(memPtr);
} else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)((U16)p[0] + (p[1]<<8));
return (U16)((U16)p[0] | (p[1]<<8));
}
}
#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT
static U32 LZ4_readLE32(const void* memPtr)
{
if (LZ4_isLittleEndian()) {
return LZ4_read32(memPtr);
} else {
const BYTE* p = (const BYTE*)memPtr;
return (U32)p[0] | (p[1]<<8) | (p[2]<<16) | (p[3]<<24);
}
}
#endif
static void LZ4_writeLE16(void* memPtr, U16 value)
{
if (LZ4_isLittleEndian()) {
@@ -512,7 +530,7 @@ LZ4_wildCopy32(void* dstPtr, const void* srcPtr, void* dstEnd)
/* LZ4_memcpy_using_offset() presumes :
* - dstEnd >= dstPtr + MINMATCH
* - there is at least 8 bytes available to write after dstEnd */
* - there is at least 12 bytes available to write after dstEnd */
LZ4_FORCE_INLINE void
LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const size_t offset)
{
@@ -527,12 +545,12 @@ LZ4_memcpy_using_offset(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd, const si
case 2:
LZ4_memcpy(v, srcPtr, 2);
LZ4_memcpy(&v[2], srcPtr, 2);
#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */
# pragma warning(push)
# pragma warning(disable : 6385) /* warning C6385: Reading invalid data from 'v'. */
#endif
LZ4_memcpy(&v[4], v, 4);
#if defined(_MSC_VER) && (_MSC_VER <= 1933) /* MSVC 2022 ver 17.3 or earlier */
#if defined(_MSC_VER) && (_MSC_VER <= 1937) /* MSVC 2022 ver 17.7 or earlier */
# pragma warning(pop)
#endif
break;
@@ -779,7 +797,12 @@ LZ4_FORCE_INLINE U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
LZ4_FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType)
{
if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType);
#ifdef LZ4_STATIC_LINKING_ONLY_ENDIANNESS_INDEPENDENT_OUTPUT
return LZ4_hash4(LZ4_readLE32(p), tableType);
#else
return LZ4_hash4(LZ4_read32(p), tableType);
#endif
}
LZ4_FORCE_INLINE void LZ4_clearHash(U32 h, void* tableBase, tableType_t const tableType)
@@ -898,7 +921,7 @@ LZ4_prepareTable(LZ4_stream_t_internal* const cctx,
cctx->dictSize = 0;
}
/** LZ4_compress_generic() :
/** LZ4_compress_generic_validated() :
* inlined, to ensure branches are decided at compilation time.
* The following conditions are presumed already validated:
* - source != NULL
@@ -1080,7 +1103,10 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
/* Catch up */
filledIp = ip;
while (((ip>anchor) & (match > lowLimit)) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; }
assert(ip > anchor); /* this is always true as ip has been advanced before entering the main loop */
if ((match > lowLimit) && unlikely(ip[-1] == match[-1])) {
do { ip--; match--; } while (((ip > anchor) & (match > lowLimit)) && (unlikely(ip[-1] == match[-1])));
}
/* Encode Literals */
{ unsigned const litLength = (unsigned)(ip - anchor);
@@ -1095,7 +1121,7 @@ LZ4_FORCE_INLINE int LZ4_compress_generic_validated(
goto _last_literals;
}
if (litLength >= RUN_MASK) {
int len = (int)(litLength - RUN_MASK);
unsigned len = litLength - RUN_MASK;
*token = (RUN_MASK<<ML_BITS);
for(; len >= 255 ; len-=255) *op++ = 255;
*op++ = (BYTE)len;
@@ -1452,22 +1478,30 @@ int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacit
/* Note!: This function leaves the stream in an unclean/broken state!
* It is not safe to subsequently use the same state with a _fastReset() or
* _continue() call without resetting it. */
static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration)
{
void* const s = LZ4_initStream(state, sizeof (*state));
assert(s != NULL); (void)s;
if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration);
} else {
if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration);
} else {
tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1);
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration);
} }
}
int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration)
{
int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration);
/* clean the state on exit */
LZ4_initStream(state, sizeof (LZ4_stream_t));
return r;
}
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
@@ -1479,7 +1513,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
LZ4_stream_t* const ctx = &ctxBody;
#endif
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1);
#if (LZ4_HEAPMODE)
FREEMEM(ctx);
@@ -1548,8 +1582,11 @@ int LZ4_freeStream (LZ4_stream_t* LZ4_stream)
#endif
typedef enum { _ld_fast, _ld_slow } LoadDict_mode_e;
#define HASH_UNIT sizeof(reg_t)
int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
int LZ4_loadDict_internal(LZ4_stream_t* LZ4_dict,
const char* dictionary, int dictSize,
LoadDict_mode_e _ld)
{
LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse;
const tableType_t tableType = byU32;
@@ -1585,13 +1622,39 @@ int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
while (p <= dictEnd-HASH_UNIT) {
U32 const h = LZ4_hashPosition(p, tableType);
/* Note: overwriting => favors positions end of dictionary */
LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType);
p+=3; idx32+=3;
}
if (_ld == _ld_slow) {
/* Fill hash table with additional references, to improve compression capability */
p = dict->dictionary;
idx32 = dict->currentOffset - dict->dictSize;
while (p <= dictEnd-HASH_UNIT) {
U32 const h = LZ4_hashPosition(p, tableType);
U32 const limit = dict->currentOffset - 64 KB;
if (LZ4_getIndexOnHash(h, dict->hashTable, tableType) <= limit) {
/* Note: not overwriting => favors positions beginning of dictionary */
LZ4_putIndexOnHash(idx32, h, dict->hashTable, tableType);
}
p++; idx32++;
}
}
return (int)dict->dictSize;
}
int LZ4_loadDict(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_fast);
}
int LZ4_loadDictSlow(LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize)
{
return LZ4_loadDict_internal(LZ4_dict, dictionary, dictSize, _ld_slow);
}
void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream)
{
const LZ4_stream_t_internal* dictCtx = (dictionaryStream == NULL) ? NULL :
@@ -1923,6 +1986,17 @@ read_variable_length(const BYTE** ip, const BYTE* ilimit,
if (initial_check && unlikely((*ip) >= ilimit)) { /* read limit reached */
return rvl_error;
}
s = **ip;
(*ip)++;
length += s;
if (unlikely((*ip) > ilimit)) { /* read limit reached */
return rvl_error;
}
/* accumulator overflow detection (32-bit mode only) */
if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) {
return rvl_error;
}
if (likely(s != 255)) return length;
do {
s = **ip;
(*ip)++;
@@ -1931,10 +2005,10 @@ read_variable_length(const BYTE** ip, const BYTE* ilimit,
return rvl_error;
}
/* accumulator overflow detection (32-bit mode only) */
if ((sizeof(length)<8) && unlikely(length > ((Rvl_t)(-1)/2)) ) {
if ((sizeof(length) < 8) && unlikely(length > ((Rvl_t)(-1)/2)) ) {
return rvl_error;
}
} while (s==255);
} while (s == 255);
return length;
}
@@ -2000,7 +2074,7 @@ LZ4_decompress_generic(
* note : fast loop may show a regression for some client arm chips. */
#if LZ4_FAST_DEC_LOOP
if ((oend - op) < FASTLOOP_SAFE_DISTANCE) {
DEBUGLOG(6, "skip fast decode loop");
DEBUGLOG(6, "move to safe decode loop");
goto safe_decode;
}
@@ -2012,6 +2086,7 @@ LZ4_decompress_generic(
assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length);
/* decode literal length */
if (length == RUN_MASK) {
@@ -2025,49 +2100,47 @@ LZ4_decompress_generic(
if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
/* copy literals */
cpy = op+length;
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
if ((cpy>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
LZ4_wildCopy32(op, ip, cpy);
ip += length; op = cpy;
} else {
cpy = op+length;
DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
if ((op+length>oend-32) || (ip+length>iend-32)) { goto safe_literal_copy; }
LZ4_wildCopy32(op, ip, op+length);
ip += length; op += length;
} else if (ip <= iend-(16 + 1/*max lit + offset + nextToken*/)) {
/* We don't need to check oend, since we check it once for each loop below */
if (ip > iend-(16 + 1/*max lit + offset + nextToken*/)) { goto safe_literal_copy; }
DEBUGLOG(7, "copy %u bytes in a 16-bytes stripe", (unsigned)length);
/* Literals can only be <= 14, but hope compilers optimize better when copy by a register size */
LZ4_memcpy(op, ip, 16);
ip += length; op = cpy;
ip += length; op += length;
} else {
goto safe_literal_copy;
}
/* get offset */
offset = LZ4_readLE16(ip); ip+=2;
DEBUGLOG(6, " offset = %zu", offset);
DEBUGLOG(6, "blockPos%6u: offset = %u", (unsigned)(op-(BYTE*)dst), (unsigned)offset);
match = op - offset;
assert(match <= op); /* overflow check */
/* get matchlength */
length = token & ML_MASK;
DEBUGLOG(7, " match length token = %u (len==%u)", (unsigned)length, (unsigned)length+MINMATCH);
if (length == ML_MASK) {
size_t const addl = read_variable_length(&ip, iend - LASTLITERALS + 1, 0);
if (addl == rvl_error) {
DEBUGLOG(6, "error reading long match length");
DEBUGLOG(5, "error reading long match length");
goto _output_error;
}
length += addl;
length += MINMATCH;
DEBUGLOG(7, " long match length == %u", (unsigned)length);
if (unlikely((uptrval)(op)+length<(uptrval)op)) { goto _output_error; } /* overflow detection */
if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) {
DEBUGLOG(6, "Error : offset outside buffers");
goto _output_error;
}
if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
goto safe_match_copy;
}
} else {
length += MINMATCH;
if (op + length >= oend - FASTLOOP_SAFE_DISTANCE) {
DEBUGLOG(7, "moving to safe_match_copy (ml==%u)", (unsigned)length);
goto safe_match_copy;
}
@@ -2086,7 +2159,7 @@ LZ4_decompress_generic(
} } }
if ( checkOffset && (unlikely(match + dictSize < lowPrefix)) ) {
DEBUGLOG(6, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match);
DEBUGLOG(5, "Error : pos=%zi, offset=%zi => outside buffers", op-lowPrefix, op-match);
goto _output_error;
}
/* match starting within external dictionary */
@@ -2143,6 +2216,7 @@ LZ4_decompress_generic(
assert(ip < iend);
token = *ip++;
length = token >> ML_BITS; /* literal length */
DEBUGLOG(7, "blockPos%6u: litLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length);
/* A two-stage shortcut for the most common case:
* 1) If the literal length is 0..14, and there is enough space,
@@ -2163,6 +2237,7 @@ LZ4_decompress_generic(
/* The second stage: prepare for match copying, decode full info.
* If it doesn't work out, the info won't be wasted. */
length = token & ML_MASK; /* match length */
DEBUGLOG(7, "blockPos%6u: matchLength token = %u (len=%u)", (unsigned)(op-(BYTE*)dst), (unsigned)length, (unsigned)length + 4);
offset = LZ4_readLE16(ip); ip += 2;
match = op - offset;
assert(match <= op); /* check overflow */
@@ -2194,11 +2269,12 @@ LZ4_decompress_generic(
if (unlikely((uptrval)(ip)+length<(uptrval)(ip))) { goto _output_error; } /* overflow detection */
}
/* copy literals */
cpy = op+length;
#if LZ4_FAST_DEC_LOOP
safe_literal_copy:
#endif
/* copy literals */
cpy = op+length;
LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH);
if ((cpy>oend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) {
/* We've either hit the input parsing restriction or the output parsing restriction.
@@ -2234,9 +2310,10 @@ LZ4_decompress_generic(
* so check that we exactly consume the input and don't overrun the output buffer.
*/
if ((ip+length != iend) || (cpy > oend)) {
DEBUGLOG(6, "should have been last run of literals")
DEBUGLOG(6, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend);
DEBUGLOG(6, "or cpy(%p) > oend(%p)", cpy, oend);
DEBUGLOG(5, "should have been last run of literals")
DEBUGLOG(5, "ip(%p) + length(%i) = %p != iend (%p)", ip, (int)length, ip+length, iend);
DEBUGLOG(5, "or cpy(%p) > (oend-MFLIMIT)(%p)", cpy, oend-MFLIMIT);
DEBUGLOG(5, "after writing %u bytes / %i bytes available", (unsigned)(op-(BYTE*)dst), outputSize);
goto _output_error;
}
}
@@ -2262,6 +2339,7 @@ LZ4_decompress_generic(
/* get matchlength */
length = token & ML_MASK;
DEBUGLOG(7, "blockPos%6u: matchLength token = %u", (unsigned)(op-(BYTE*)dst), (unsigned)length);
_copy_match:
if (length == ML_MASK) {
@@ -2351,7 +2429,7 @@ LZ4_decompress_generic(
while (op < cpy) { *op++ = *match++; }
} else {
LZ4_memcpy(op, match, 8);
if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }
if (length > 16) { LZ4_wildCopy8(op+8, match+8, cpy); }
}
op = cpy; /* wildcopy correction */
}
lz4/lz4.h
+81 -59
View File
@@ -1,7 +1,7 @@
/*
* LZ4 - Fast LZ compression algorithm
* Header File
* Copyright (C) 2011-2020, Yann Collet.
* Copyright (C) 2011-2023, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
@@ -129,8 +129,8 @@ extern "C" {
/*------ Version ------*/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_MINOR 10 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
@@ -144,23 +144,25 @@ LZ4LIB_API const char* LZ4_versionString (void); /**< library version string;
/*-************************************
* Tuning parameter
* Tuning memory usage
**************************************/
#define LZ4_MEMORY_USAGE_MIN 10
#define LZ4_MEMORY_USAGE_DEFAULT 14
#define LZ4_MEMORY_USAGE_MAX 20
/*!
* LZ4_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; )
* Increasing memory usage improves compression ratio, at the cost of speed.
* Can be selected at compile time, by setting LZ4_MEMORY_USAGE.
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB)
* Increasing memory usage improves compression ratio, generally at the cost of speed.
* Reduced memory usage may improve speed at the cost of ratio, thanks to better cache locality.
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
* Default value is 14, for 16KB, which nicely fits into most L1 caches.
*/
#ifndef LZ4_MEMORY_USAGE
# define LZ4_MEMORY_USAGE LZ4_MEMORY_USAGE_DEFAULT
#endif
/* These are absolute limits, they should not be changed by users */
#define LZ4_MEMORY_USAGE_MIN 10
#define LZ4_MEMORY_USAGE_DEFAULT 14
#define LZ4_MEMORY_USAGE_MAX 20
#if (LZ4_MEMORY_USAGE < LZ4_MEMORY_USAGE_MIN)
# error "LZ4_MEMORY_USAGE is too small !"
#endif
@@ -191,7 +193,7 @@ LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int
/*! LZ4_decompress_safe() :
* @compressedSize : is the exact complete size of the compressed block.
* @dstCapacity : is the size of destination buffer (which must be already allocated),
* is an upper bound of decompressed size.
* presumed an upper bound of decompressed size.
* @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
@@ -243,20 +245,20 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_compress_destSize() :
* Reverse the logic : compresses as much data as possible from 'src' buffer
* into already allocated buffer 'dst', of size >= 'targetDestSize'.
* into already allocated buffer 'dst', of size >= 'dstCapacity'.
* This function either compresses the entire 'src' content into 'dst' if it's large enough,
* or fill 'dst' buffer completely with as much data as possible from 'src'.
* note: acceleration parameter is fixed to "default".
*
* *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
* *srcSizePtr : in+out parameter. Initially contains size of input.
* Will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
* New value is necessarily <= input value.
* @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
* @return : Nb bytes written into 'dst' (necessarily <= dstCapacity)
* or 0 if compression fails.
*
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed un v1.9.2+):
* Note : from v1.8.2 to v1.9.1, this function had a bug (fixed in v1.9.2+):
* the produced compressed content could, in specific circumstances,
* require to be decompressed into a destination buffer larger
* by at least 1 byte than the content to decompress.
@@ -267,8 +269,7 @@ LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* d
* a dstCapacity which is > decompressedSize, by at least 1 byte.
* See https://github.com/lz4/lz4/issues/859 for details
*/
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize);
/*! LZ4_decompress_safe_partial() :
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
@@ -312,7 +313,7 @@ LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcS
***********************************************/
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
/**
/*!
Note about RC_INVOKED
- RC_INVOKED is predefined symbol of rc.exe (the resource compiler which is part of MSVC/Visual Studio).
@@ -362,13 +363,58 @@ LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
* LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
* The same dictionary will have to be loaded on decompression side for successful decoding.
* Dictionary are useful for better compression of small data (KB range).
* While LZ4 accept any input as dictionary,
* results are generally better when using Zstandard's Dictionary Builder.
* While LZ4 itself accepts any input as dictionary, dictionary efficiency is also a topic.
* When in doubt, employ the Zstandard's Dictionary Builder.
* Loading a size of 0 is allowed, and is the same as reset.
* @return : loaded dictionary size, in bytes (necessarily <= 64 KB)
* @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded)
*/
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ4_loadDictSlow() : v1.10.0+
* Same as LZ4_loadDict(),
* but uses a bit more cpu to reference the dictionary content more thoroughly.
* This is expected to slightly improve compression ratio.
* The extra-cpu cost is likely worth it if the dictionary is re-used across multiple sessions.
* @return : loaded dictionary size, in bytes (note: only the last 64 KB are loaded)
*/
LZ4LIB_API int LZ4_loadDictSlow(LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ4_attach_dictionary() : stable since v1.10.0
*
* This allows efficient re-use of a static dictionary multiple times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
* in which the working stream references @dictionaryStream in-place.
*
* Several assumptions are made about the state of @dictionaryStream.
* Currently, only states which have been prepared by LZ4_loadDict() or
* LZ4_loadDictSlow() should be expected to work.
*
* Alternatively, the provided @dictionaryStream may be NULL,
* in which case any existing dictionary stream is unset.
*
* If a dictionary is provided, it replaces any pre-existing stream history.
* The dictionary contents are the only history that can be referenced and
* logically immediately precede the data compressed in the first subsequent
* compression call.
*
* The dictionary will only remain attached to the working stream through the
* first compression call, at the end of which it is cleared.
* @dictionaryStream stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the compression session.
*
* Note: there is no equivalent LZ4_attach_*() method on the decompression side
* because there is no initialization cost, hence no need to share the cost across multiple sessions.
* To decompress LZ4 blocks using dictionary, attached or not,
* just employ the regular LZ4_setStreamDecode() for streaming,
* or the stateless LZ4_decompress_safe_usingDict() for one-shot decompression.
*/
LZ4LIB_API void
LZ4_attach_dictionary(LZ4_stream_t* workingStream,
const LZ4_stream_t* dictionaryStream);
/*! LZ4_compress_fast_continue() :
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
* 'dst' buffer must be already allocated.
@@ -546,9 +592,9 @@ LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
#define LZ4_STATIC_3504398509
#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
#define LZ4LIB_STATIC_API LZ4LIB_API
# define LZ4LIB_STATIC_API LZ4LIB_API
#else
#define LZ4LIB_STATIC_API
# define LZ4LIB_STATIC_API
#endif
@@ -564,36 +610,11 @@ LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
*/
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_attach_dictionary() :
* This is an experimental API that allows
* efficient use of a static dictionary many times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
* in which the working stream references the dictionary stream in-place.
*
* Several assumptions are made about the state of the dictionary stream.
* Currently, only streams which have been prepared by LZ4_loadDict() should
* be expected to work.
*
* Alternatively, the provided dictionaryStream may be NULL,
* in which case any existing dictionary stream is unset.
*
* If a dictionary is provided, it replaces any pre-existing stream history.
* The dictionary contents are the only history that can be referenced and
* logically immediately precede the data compressed in the first subsequent
* compression call.
*
* The dictionary will only remain attached to the working stream through the
* first compression call, at the end of which it is cleared. The dictionary
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the completion of the first compression call on the stream.
/*! LZ4_compress_destSize_extState() : introduced in v1.10.0
* Same as LZ4_compress_destSize(), but using an externally allocated state.
* Also: exposes @acceleration
*/
LZ4LIB_STATIC_API void
LZ4_attach_dictionary(LZ4_stream_t* workingStream,
const LZ4_stream_t* dictionaryStream);
int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration);
/*! In-place compression and decompression
*
@@ -705,7 +726,7 @@ struct LZ4_stream_t_internal {
/* Implicit padding to ensure structure is aligned */
};
#define LZ4_STREAM_MINSIZE ((1UL << LZ4_MEMORY_USAGE) + 32) /* static size, for inter-version compatibility */
#define LZ4_STREAM_MINSIZE ((1UL << (LZ4_MEMORY_USAGE)) + 32) /* static size, for inter-version compatibility */
union LZ4_stream_u {
char minStateSize[LZ4_STREAM_MINSIZE];
LZ4_stream_t_internal internal_donotuse;
@@ -726,7 +747,7 @@ union LZ4_stream_u {
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
* Note3: Before v1.9.0, use LZ4_resetStream() instead
**/
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* stateBuffer, size_t size);
/*! LZ4_streamDecode_t :
@@ -838,11 +859,12 @@ LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4
* But they may happen if input data is invalid (error or intentional tampering).
* As a consequence, use these functions in trusted environments with trusted data **only**.
*/
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial() instead")
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider migrating towards LZ4_decompress_safe_continue() instead. "
"Note that the contract will change (requires block's compressed size, instead of decompressed size)")
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_partial_usingDict() instead")
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
/*! LZ4_resetStream() :
lz4/lz4hc.c
+817 -262
View File
File diff suppressed because it is too large Load Diff
lz4/lz4hc.h
+43 -42
View File
@@ -44,7 +44,7 @@ extern "C" {
/* --- Useful constants --- */
#define LZ4HC_CLEVEL_MIN 3
#define LZ4HC_CLEVEL_MIN 2
#define LZ4HC_CLEVEL_DEFAULT 9
#define LZ4HC_CLEVEL_OPT_MIN 10
#define LZ4HC_CLEVEL_MAX 12
@@ -126,6 +126,8 @@ LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
After reset, a first "fictional block" can be designated as initial dictionary,
using LZ4_loadDictHC() (Optional).
Note: In order for LZ4_loadDictHC() to create the correct data structure,
it is essential to set the compression level _before_ loading the dictionary.
Invoke LZ4_compress_HC_continue() to compress each successive block.
The number of blocks is unlimited.
@@ -135,12 +137,12 @@ LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
It's allowed to update compression level anytime between blocks,
using LZ4_setCompressionLevel() (experimental).
'dst' buffer should be sized to handle worst case scenarios
@dst buffer should be sized to handle worst case scenarios
(see LZ4_compressBound(), it ensures compression success).
In case of failure, the API does not guarantee recovery,
so the state _must_ be reset.
To ensure compression success
whenever `dst` buffer size cannot be made >= LZ4_compressBound(),
whenever @dst buffer size cannot be made >= LZ4_compressBound(),
consider using LZ4_compress_HC_continue_destSize().
Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks,
@@ -176,6 +178,34 @@ LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr
LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
/*! LZ4_attach_HC_dictionary() : stable since v1.10.0
* This API allows for the efficient re-use of a static dictionary many times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
* working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
* in which the working stream references the dictionary stream in-place.
*
* Several assumptions are made about the state of the dictionary stream.
* Currently, only streams which have been prepared by LZ4_loadDictHC() should
* be expected to work.
*
* Alternatively, the provided dictionary stream pointer may be NULL, in which
* case any existing dictionary stream is unset.
*
* A dictionary should only be attached to a stream without any history (i.e.,
* a stream that has just been reset).
*
* The dictionary will remain attached to the working stream only for the
* current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
* dictionary context association from the working stream. The dictionary
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the lifetime of the stream session.
*/
LZ4LIB_API void
LZ4_attach_HC_dictionary(LZ4_streamHC_t* working_stream,
const LZ4_streamHC_t* dictionary_stream);
/*^**********************************************
* !!!!!! STATIC LINKING ONLY !!!!!!
@@ -204,18 +234,18 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
struct LZ4HC_CCtx_internal
{
LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE];
LZ4_u16 chainTable[LZ4HC_MAXD];
const LZ4_byte* end; /* next block here to continue on current prefix */
LZ4_u32 hashTable[LZ4HC_HASHTABLESIZE];
LZ4_u16 chainTable[LZ4HC_MAXD];
const LZ4_byte* end; /* next block here to continue on current prefix */
const LZ4_byte* prefixStart; /* Indexes relative to this position */
const LZ4_byte* dictStart; /* alternate reference for extDict */
LZ4_u32 dictLimit; /* below that point, need extDict */
LZ4_u32 lowLimit; /* below that point, no more dict */
LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set,
otherwise, favor compression ratio */
LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */
LZ4_u32 dictLimit; /* below that point, need extDict */
LZ4_u32 lowLimit; /* below that point, no more history */
LZ4_u32 nextToUpdate; /* index from which to continue dictionary update */
short compressionLevel;
LZ4_i8 favorDecSpeed; /* favor decompression speed if this flag set,
otherwise, favor compression ratio */
LZ4_i8 dirty; /* stream has to be fully reset if this flag is set */
const LZ4HC_CCtx_internal* dictCtx;
};
@@ -376,35 +406,6 @@ LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset (
int srcSize, int dstCapacity,
int compressionLevel);
/*! LZ4_attach_HC_dictionary() :
* This is an experimental API that allows for the efficient use of a
* static dictionary many times.
*
* Rather than re-loading the dictionary buffer into a working context before
* each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
* working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
* in which the working stream references the dictionary stream in-place.
*
* Several assumptions are made about the state of the dictionary stream.
* Currently, only streams which have been prepared by LZ4_loadDictHC() should
* be expected to work.
*
* Alternatively, the provided dictionary stream pointer may be NULL, in which
* case any existing dictionary stream is unset.
*
* A dictionary should only be attached to a stream without any history (i.e.,
* a stream that has just been reset).
*
* The dictionary will remain attached to the working stream only for the
* current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
* dictionary context association from the working stream. The dictionary
* stream (and source buffer) must remain in-place / accessible / unchanged
* through the lifetime of the stream session.
*/
LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
LZ4_streamHC_t *working_stream,
const LZ4_streamHC_t *dictionary_stream);
#if defined (__cplusplus)
}
#endif