mirror of
https://github.com/RfidResearchGroup/proxmark3.git
synced 2024-09-20 15:26:13 +08:00
commit
fc1f137f0c
|
@ -3,6 +3,8 @@ All notable changes to this project will be documented in this file.
|
||||||
This project uses the changelog in accordance with [keepchangelog](http://keepachangelog.com/). Please use this to write notable changes, which is not the same as git commit log...
|
This project uses the changelog in accordance with [keepchangelog](http://keepachangelog.com/). Please use this to write notable changes, which is not the same as git commit log...
|
||||||
|
|
||||||
## [unreleased][unreleased]
|
## [unreleased][unreleased]
|
||||||
|
- Change pushed some ARM source to compile with THUMB. Saves a lot of space of ARM flash memory @slurdge
|
||||||
|
- Change from using ZLIB compression of FPGA images in ARM to LZ4. 15%+ speedups when swapping between HF & LF now @slurdge
|
||||||
- Fix `emv scan -w` - hash mismatch, solution similar to below (@iceman1001)
|
- Fix `emv scan -w` - hash mismatch, solution similar to below (@iceman1001)
|
||||||
- Fix 'emv roca' - hash mismatch, merged from offical repo (@pwpiwi)
|
- Fix 'emv roca' - hash mismatch, merged from offical repo (@pwpiwi)
|
||||||
- Fix 'lf ti demod' - now calculates correct crc (@iceman1001)
|
- Fix 'lf ti demod' - now calculates correct crc (@iceman1001)
|
||||||
|
|
|
@ -75,13 +75,13 @@ include Standalone/Makefile.inc
|
||||||
#the FPGA bitstream files. Note: order matters!
|
#the FPGA bitstream files. Note: order matters!
|
||||||
FPGA_BITSTREAMS = fpga_lf.bit fpga_hf.bit
|
FPGA_BITSTREAMS = fpga_lf.bit fpga_hf.bit
|
||||||
|
|
||||||
#the zlib source files required for decompressing the fpga config at run time
|
#the lz4 source files required for decompressing the fpga config at run time
|
||||||
SRC_ZLIB = inflate.c inffast.c inftrees.c adler32.c zutil.c
|
SRC_LZ4 = lz4.c
|
||||||
#additional defines required to compile zlib
|
#additional defines required to compile lz4
|
||||||
ZLIB_CFLAGS = -DZ_SOLO -DZ_PREFIX -DNO_GZIP -DZLIB_PM3_TUNED
|
LZ4_CFLAGS = -DLZ4_MEMORY_USAGE=8
|
||||||
APP_CFLAGS += $(ZLIB_CFLAGS)
|
APP_CFLAGS += $(LZ4_CFLAGS)
|
||||||
# zlib includes:
|
# lz4 includes:
|
||||||
APP_CFLAGS += -I../common/zlib
|
APP_CFLAGS += -I../common/lz4
|
||||||
|
|
||||||
# stdint.h provided locally until GCC 4.5 becomes C99 compliant,
|
# stdint.h provided locally until GCC 4.5 becomes C99 compliant,
|
||||||
# stack-protect , no-pie reduces size on Gentoo Hardened 8.2 gcc
|
# stack-protect , no-pie reduces size on Gentoo Hardened 8.2 gcc
|
||||||
|
@ -93,7 +93,7 @@ THUMBSRC = start.c \
|
||||||
$(SRC_ISO15693) \
|
$(SRC_ISO15693) \
|
||||||
$(SRC_NFCBARCODE) \
|
$(SRC_NFCBARCODE) \
|
||||||
$(SRC_LF) \
|
$(SRC_LF) \
|
||||||
$(SRC_ZLIB) \
|
$(SRC_LZ4) \
|
||||||
$(SRC_LEGIC) \
|
$(SRC_LEGIC) \
|
||||||
$(SRC_FLASH) \
|
$(SRC_FLASH) \
|
||||||
$(SRC_SMARTCARD) \
|
$(SRC_SMARTCARD) \
|
||||||
|
|
|
@ -17,10 +17,19 @@
|
||||||
#include "ticks.h"
|
#include "ticks.h"
|
||||||
#include "dbprint.h"
|
#include "dbprint.h"
|
||||||
#include "util.h"
|
#include "util.h"
|
||||||
#include "zlib.h"
|
|
||||||
#include "fpga.h"
|
#include "fpga.h"
|
||||||
#include "string.h"
|
#include "string.h"
|
||||||
|
|
||||||
|
#include "lz4.h" // uncompress
|
||||||
|
|
||||||
|
typedef struct lz4_stream_s {
|
||||||
|
LZ4_streamDecode_t* lz4StreamDecode;
|
||||||
|
char * next_in;
|
||||||
|
int avail_in;
|
||||||
|
} lz4_stream;
|
||||||
|
|
||||||
|
typedef lz4_stream* lz4_streamp;
|
||||||
|
|
||||||
// remember which version of the bitstream we have already downloaded to the FPGA
|
// remember which version of the bitstream we have already downloaded to the FPGA
|
||||||
static int downloaded_bitstream = 0;
|
static int downloaded_bitstream = 0;
|
||||||
|
|
||||||
|
@ -30,8 +39,6 @@ extern uint8_t _binary_obj_fpga_all_bit_z_start, _binary_obj_fpga_all_bit_z_end;
|
||||||
static uint8_t *fpga_image_ptr = NULL;
|
static uint8_t *fpga_image_ptr = NULL;
|
||||||
static uint32_t uncompressed_bytes_cnt;
|
static uint32_t uncompressed_bytes_cnt;
|
||||||
|
|
||||||
#define OUTPUT_BUFFER_LEN 80
|
|
||||||
|
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
// Set up the Serial Peripheral Interface as master
|
// Set up the Serial Peripheral Interface as master
|
||||||
// Used to write the FPGA config word
|
// Used to write the FPGA config word
|
||||||
|
@ -180,18 +187,23 @@ bool FpgaSetupSscDma(uint8_t *buf, int len) {
|
||||||
// Uncompress (inflate) the FPGA data. Returns one decompressed byte with
|
// Uncompress (inflate) the FPGA data. Returns one decompressed byte with
|
||||||
// each call.
|
// each call.
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
static int get_from_fpga_combined_stream(z_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
static int get_from_fpga_combined_stream(lz4_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
||||||
if (fpga_image_ptr == compressed_fpga_stream->next_out) { // need more data
|
if (fpga_image_ptr == output_buffer + FPGA_RING_BUFFER_BYTES) { // need more data
|
||||||
compressed_fpga_stream->next_out = output_buffer;
|
|
||||||
compressed_fpga_stream->avail_out = OUTPUT_BUFFER_LEN;
|
|
||||||
fpga_image_ptr = output_buffer;
|
fpga_image_ptr = output_buffer;
|
||||||
int res = inflate(compressed_fpga_stream, Z_SYNC_FLUSH);
|
int cmp_bytes;
|
||||||
|
memcpy(&cmp_bytes, compressed_fpga_stream->next_in, sizeof(int));
|
||||||
if (res != Z_OK)
|
compressed_fpga_stream->next_in += 4;
|
||||||
Dbprintf("inflate returned: %d, %s", res, compressed_fpga_stream->msg);
|
compressed_fpga_stream->avail_in -= cmp_bytes + 4;
|
||||||
|
int res = LZ4_decompress_safe_continue(compressed_fpga_stream->lz4StreamDecode,
|
||||||
if (res < 0)
|
compressed_fpga_stream->next_in,
|
||||||
|
(char*)output_buffer,
|
||||||
|
cmp_bytes,
|
||||||
|
FPGA_RING_BUFFER_BYTES);
|
||||||
|
if (res <= 0) {
|
||||||
|
Dbprintf("inflate returned: %d", res);
|
||||||
return res;
|
return res;
|
||||||
|
}
|
||||||
|
compressed_fpga_stream->next_in += cmp_bytes;
|
||||||
}
|
}
|
||||||
uncompressed_bytes_cnt++;
|
uncompressed_bytes_cnt++;
|
||||||
return *fpga_image_ptr++;
|
return *fpga_image_ptr++;
|
||||||
|
@ -202,7 +214,7 @@ static int get_from_fpga_combined_stream(z_streamp compressed_fpga_stream, uint8
|
||||||
// are combined into one big file:
|
// are combined into one big file:
|
||||||
// 288 bytes from FPGA file 1, followed by 288 bytes from FGPA file 2, etc.
|
// 288 bytes from FPGA file 1, followed by 288 bytes from FGPA file 2, etc.
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
static int get_from_fpga_stream(int bitstream_version, z_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
static int get_from_fpga_stream(int bitstream_version, lz4_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
||||||
while ((uncompressed_bytes_cnt / FPGA_INTERLEAVE_SIZE) % g_fpga_bitstream_num != (bitstream_version - 1)) {
|
while ((uncompressed_bytes_cnt / FPGA_INTERLEAVE_SIZE) % g_fpga_bitstream_num != (bitstream_version - 1)) {
|
||||||
// skip undesired data belonging to other bitstream_versions
|
// skip undesired data belonging to other bitstream_versions
|
||||||
get_from_fpga_combined_stream(compressed_fpga_stream, output_buffer);
|
get_from_fpga_combined_stream(compressed_fpga_stream, output_buffer);
|
||||||
|
@ -211,37 +223,23 @@ static int get_from_fpga_stream(int bitstream_version, z_streamp compressed_fpga
|
||||||
return get_from_fpga_combined_stream(compressed_fpga_stream, output_buffer);
|
return get_from_fpga_combined_stream(compressed_fpga_stream, output_buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
static voidpf fpga_inflate_malloc(voidpf opaque, uInt items, uInt size) {
|
|
||||||
return BigBuf_malloc(items * size);
|
|
||||||
}
|
|
||||||
|
|
||||||
// free eventually allocated BigBuf memory
|
|
||||||
static void fpga_inflate_free(voidpf opaque, voidpf address) {
|
|
||||||
BigBuf_free();
|
|
||||||
BigBuf_Clear_ext(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
// Initialize decompression of the respective (HF or LF) FPGA stream
|
// Initialize decompression of the respective (HF or LF) FPGA stream
|
||||||
//----------------------------------------------------------------------------
|
//----------------------------------------------------------------------------
|
||||||
static bool reset_fpga_stream(int bitstream_version, z_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
static bool reset_fpga_stream(int bitstream_version, lz4_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
||||||
uint8_t header[FPGA_BITSTREAM_FIXED_HEADER_SIZE];
|
uint8_t header[FPGA_BITSTREAM_FIXED_HEADER_SIZE];
|
||||||
|
|
||||||
uncompressed_bytes_cnt = 0;
|
uncompressed_bytes_cnt = 0;
|
||||||
|
|
||||||
// initialize z_stream structure for inflate:
|
// initialize z_stream structure for inflate:
|
||||||
compressed_fpga_stream->next_in = &_binary_obj_fpga_all_bit_z_start;
|
compressed_fpga_stream->next_in = (char*)&_binary_obj_fpga_all_bit_z_start;
|
||||||
compressed_fpga_stream->avail_in = &_binary_obj_fpga_all_bit_z_end - &_binary_obj_fpga_all_bit_z_start;
|
compressed_fpga_stream->avail_in = &_binary_obj_fpga_all_bit_z_end - &_binary_obj_fpga_all_bit_z_start;
|
||||||
compressed_fpga_stream->next_out = output_buffer;
|
|
||||||
compressed_fpga_stream->avail_out = OUTPUT_BUFFER_LEN;
|
int res = LZ4_setStreamDecode(compressed_fpga_stream->lz4StreamDecode, NULL, 0);
|
||||||
compressed_fpga_stream->zalloc = &fpga_inflate_malloc;
|
if (res == 0)
|
||||||
compressed_fpga_stream->zfree = &fpga_inflate_free;
|
|
||||||
|
|
||||||
int res = inflateInit2(compressed_fpga_stream, 0);
|
|
||||||
if (res < 0)
|
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
fpga_image_ptr = output_buffer;
|
fpga_image_ptr = output_buffer + FPGA_RING_BUFFER_BYTES;
|
||||||
|
|
||||||
for (uint16_t i = 0; i < FPGA_BITSTREAM_FIXED_HEADER_SIZE; i++)
|
for (uint16_t i = 0; i < FPGA_BITSTREAM_FIXED_HEADER_SIZE; i++)
|
||||||
header[i] = get_from_fpga_stream(bitstream_version, compressed_fpga_stream, output_buffer);
|
header[i] = get_from_fpga_stream(bitstream_version, compressed_fpga_stream, output_buffer);
|
||||||
|
@ -266,7 +264,7 @@ static void DownloadFPGA_byte(uint8_t w) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download the fpga image starting at current stream position with length FpgaImageLen bytes
|
// Download the fpga image starting at current stream position with length FpgaImageLen bytes
|
||||||
static void DownloadFPGA(int bitstream_version, int FpgaImageLen, z_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
static void DownloadFPGA(int bitstream_version, int FpgaImageLen, lz4_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
||||||
int i = 0;
|
int i = 0;
|
||||||
|
|
||||||
AT91C_BASE_PIOA->PIO_OER = GPIO_FPGA_ON;
|
AT91C_BASE_PIOA->PIO_OER = GPIO_FPGA_ON;
|
||||||
|
@ -348,7 +346,7 @@ static void DownloadFPGA(int bitstream_version, int FpgaImageLen, z_streamp comp
|
||||||
* (big endian), <length> bytes content. Except for section 'e' which has 4 bytes
|
* (big endian), <length> bytes content. Except for section 'e' which has 4 bytes
|
||||||
* length.
|
* length.
|
||||||
*/
|
*/
|
||||||
static int bitparse_find_section(int bitstream_version, char section_name, uint32_t *section_length, z_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
static int bitparse_find_section(int bitstream_version, char section_name, uint32_t *section_length, lz4_streamp compressed_fpga_stream, uint8_t *output_buffer) {
|
||||||
int result = 0;
|
int result = 0;
|
||||||
#define MAX_FPGA_BIT_STREAM_HEADER_SEARCH 100 // maximum number of bytes to search for the requested section
|
#define MAX_FPGA_BIT_STREAM_HEADER_SEARCH 100 // maximum number of bytes to search for the requested section
|
||||||
uint16_t numbytes = 0;
|
uint16_t numbytes = 0;
|
||||||
|
@ -407,8 +405,6 @@ void FpgaDownloadAndGo(int bitstream_version) {
|
||||||
|
|
||||||
// Send waiting time extension request as this will take a while
|
// Send waiting time extension request as this will take a while
|
||||||
send_wtx(1500);
|
send_wtx(1500);
|
||||||
z_stream compressed_fpga_stream;
|
|
||||||
uint8_t output_buffer[OUTPUT_BUFFER_LEN] = {0x00};
|
|
||||||
|
|
||||||
bool verbose = (DBGLEVEL > 3);
|
bool verbose = (DBGLEVEL > 3);
|
||||||
|
|
||||||
|
@ -416,6 +412,11 @@ void FpgaDownloadAndGo(int bitstream_version) {
|
||||||
BigBuf_free();
|
BigBuf_free();
|
||||||
BigBuf_Clear_ext(verbose);
|
BigBuf_Clear_ext(verbose);
|
||||||
|
|
||||||
|
lz4_stream compressed_fpga_stream;
|
||||||
|
LZ4_streamDecode_t lz4StreamDecode_body = {{ 0 }};
|
||||||
|
compressed_fpga_stream.lz4StreamDecode = &lz4StreamDecode_body;
|
||||||
|
uint8_t * output_buffer = BigBuf_malloc(FPGA_RING_BUFFER_BYTES);
|
||||||
|
|
||||||
if (!reset_fpga_stream(bitstream_version, &compressed_fpga_stream, output_buffer))
|
if (!reset_fpga_stream(bitstream_version, &compressed_fpga_stream, output_buffer))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -425,8 +426,6 @@ void FpgaDownloadAndGo(int bitstream_version) {
|
||||||
downloaded_bitstream = bitstream_version;
|
downloaded_bitstream = bitstream_version;
|
||||||
}
|
}
|
||||||
|
|
||||||
inflateEnd(&compressed_fpga_stream);
|
|
||||||
|
|
||||||
// turn off antenna
|
// turn off antenna
|
||||||
FpgaWriteConfWord(FPGA_MAJOR_MODE_OFF);
|
FpgaWriteConfWord(FPGA_MAJOR_MODE_OFF);
|
||||||
|
|
||||||
|
|
|
@ -14,70 +14,40 @@
|
||||||
|
|
||||||
#include "proxmark3_arm.h"
|
#include "proxmark3_arm.h"
|
||||||
#include "appmain.h"
|
#include "appmain.h"
|
||||||
#include "zlib.h"
|
#include "lz4.h"
|
||||||
#include "BigBuf.h"
|
#include "BigBuf.h"
|
||||||
|
#include "string.h"
|
||||||
|
|
||||||
static uint8_t *next_free_memory;
|
static uint8_t *next_free_memory;
|
||||||
extern struct common_area common_area;
|
extern struct common_area common_area;
|
||||||
extern char __data_src_start__, __data_start__, __data_end__, __bss_start__, __bss_end__;
|
extern char __data_src_start__, __data_start__, __data_end__, __bss_start__, __bss_end__;
|
||||||
|
|
||||||
static voidpf inflate_malloc(voidpf opaque, uInt items, uInt size) {
|
|
||||||
uint8_t *allocated_memory;
|
|
||||||
|
|
||||||
allocated_memory = next_free_memory;
|
|
||||||
next_free_memory += items * size;
|
|
||||||
return allocated_memory;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void inflate_free(voidpf opaque, voidpf address) {
|
|
||||||
// nothing to do
|
|
||||||
}
|
|
||||||
|
|
||||||
static void uncompress_data_section(void) {
|
static void uncompress_data_section(void) {
|
||||||
z_stream data_section;
|
|
||||||
|
|
||||||
next_free_memory = BigBuf_get_addr();
|
next_free_memory = BigBuf_get_addr();
|
||||||
|
int avail_in;
|
||||||
|
memcpy(&avail_in, &__data_start__, sizeof(int));
|
||||||
|
int avail_out = &__data_end__ - &__data_start__; // uncompressed size. Correct.
|
||||||
|
// uncompress data segment to RAM
|
||||||
|
uintptr_t p = (uintptr_t)&__data_src_start__;
|
||||||
|
int res = LZ4_decompress_safe((char*)p + 4, &__data_start__, avail_in, avail_out);
|
||||||
|
|
||||||
// initialize zstream structure
|
|
||||||
data_section.next_in = (uint8_t *) &__data_src_start__;
|
|
||||||
data_section.avail_in = &__data_end__ - &__data_start__; // uncompressed size. Wrong but doesn't matter.
|
|
||||||
data_section.next_out = (uint8_t *) &__data_start__;
|
|
||||||
data_section.avail_out = &__data_end__ - &__data_start__; // uncompressed size. Correct.
|
|
||||||
data_section.zalloc = &inflate_malloc;
|
|
||||||
data_section.zfree = &inflate_free;
|
|
||||||
data_section.opaque = NULL;
|
|
||||||
|
|
||||||
// initialize zlib for inflate
|
|
||||||
int res = inflateInit2(&data_section, 15);
|
|
||||||
if (res < 0)
|
if (res < 0)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// uncompress data segment to RAM
|
|
||||||
inflate(&data_section, Z_FINISH);
|
|
||||||
|
|
||||||
// save the size of the compressed data section
|
// save the size of the compressed data section
|
||||||
common_area.arg1 = data_section.total_in;
|
common_area.arg1 = res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void __attribute__((section(".startos"))) Vector(void);
|
void __attribute__((section(".startos"))) Vector(void);
|
||||||
void Vector(void) {
|
void Vector(void) {
|
||||||
/* Stack should have been set up by the bootloader */
|
/* Stack should have been set up by the bootloader */
|
||||||
// char *src;
|
|
||||||
char *dst, *end;
|
|
||||||
|
|
||||||
uncompress_data_section();
|
uncompress_data_section();
|
||||||
|
|
||||||
/* Set up (that is: clear) BSS. */
|
/* Set up (that is: clear) BSS. */
|
||||||
dst = &__bss_start__;
|
char *dst = &__bss_start__;
|
||||||
end = &__bss_end__;
|
char *end = &__bss_end__;
|
||||||
while (dst < end) *dst++ = 0;
|
while (dst < end) *dst++ = 0;
|
||||||
|
|
||||||
// Set up data segment: Copy from flash to ram
|
|
||||||
// src = &__data_src_start__;
|
|
||||||
// dst = &__data_start__;
|
|
||||||
// end = &__data_end__;
|
|
||||||
// while(dst < end) *dst++ = *src++;
|
|
||||||
|
|
||||||
AppMain();
|
AppMain();
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -20,6 +20,23 @@ void *memcpy(void *dest, const void *src, int len) {
|
||||||
return dest;
|
return dest;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void *memmove (void *dest, const void *src, size_t len)
|
||||||
|
{
|
||||||
|
char *d = dest;
|
||||||
|
const char *s = src;
|
||||||
|
if (d < s)
|
||||||
|
while (len--)
|
||||||
|
*d++ = *s++;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
char *lasts = (char*)s + (len-1);
|
||||||
|
char *lastd = d + (len-1);
|
||||||
|
while (len--)
|
||||||
|
*lastd-- = *lasts--;
|
||||||
|
}
|
||||||
|
return dest;
|
||||||
|
}
|
||||||
|
|
||||||
void *memset(void *dest, int c, int len) {
|
void *memset(void *dest, int c, int len) {
|
||||||
uint8_t *d = dest;
|
uint8_t *d = dest;
|
||||||
while ((len--) > 0) {
|
while ((len--) > 0) {
|
||||||
|
|
|
@ -16,6 +16,7 @@
|
||||||
|
|
||||||
int strlen(const char *str);
|
int strlen(const char *str);
|
||||||
void *memcpy(void *dest, const void *src, int len);
|
void *memcpy(void *dest, const void *src, int len);
|
||||||
|
void *memmove (void *dest, const void *src, size_t len);
|
||||||
void *memset(void *dest, int c, int len);
|
void *memset(void *dest, int c, int len);
|
||||||
int memcmp(const void *av, const void *bv, int len);
|
int memcmp(const void *av, const void *bv, int len);
|
||||||
void memxor(uint8_t *dest, uint8_t *src, size_t len);
|
void memxor(uint8_t *dest, uint8_t *src, size_t len);
|
||||||
|
|
19
common/get_lz4.sh
Executable file
19
common/get_lz4.sh
Executable file
|
@ -0,0 +1,19 @@
|
||||||
|
version=1.9.2
|
||||||
|
mkdir -p lz4
|
||||||
|
wget https://github.com/lz4/lz4/archive/v$version.zip
|
||||||
|
unzip -o -j v$version "lz4-$version/LICENSE" "lz4-$version/lib/lz4.h" "lz4-$version/lib/lz4.c" "lz4-$version/lib/lz4hc.h" "lz4-$version/lib/lz4hc.c" -d lz4
|
||||||
|
rm v$version.zip
|
||||||
|
patch -p0 << EOF
|
||||||
|
diff -Naur lz4/lz4.c lz4/lz4.c
|
||||||
|
--- lz4/lz4.c 2019-08-15 13:59:59.000000000 +0200
|
||||||
|
+++ lz4/lz4.c 2020-06-07 12:50:11.788924953 +0200
|
||||||
|
@@ -1650,7 +1650,7 @@
|
||||||
|
* Note that it is important for performance that this function really get inlined,
|
||||||
|
* in order to remove useless branches during compilation optimization.
|
||||||
|
*/
|
||||||
|
-LZ4_FORCE_INLINE int
|
||||||
|
+int
|
||||||
|
LZ4_decompress_generic(
|
||||||
|
const char* const src,
|
||||||
|
char* const dst,
|
||||||
|
EOF
|
11
common/lz4/LICENSE
Normal file
11
common/lz4/LICENSE
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
This repository uses 2 different licenses :
|
||||||
|
- all files in the `lib` directory use a BSD 2-Clause license
|
||||||
|
- all other files use a GPLv2 license, unless explicitly stated otherwise
|
||||||
|
|
||||||
|
Relevant license is reminded at the top of each source file,
|
||||||
|
and with presence of COPYING or LICENSE file in associated directories.
|
||||||
|
|
||||||
|
This model is selected to emphasize that
|
||||||
|
files in the `lib` directory are designed to be included into 3rd party applications,
|
||||||
|
while all other files, in `programs`, `tests` or `examples`,
|
||||||
|
receive more limited attention and support for such scenario.
|
2398
common/lz4/lz4.c
Normal file
2398
common/lz4/lz4.c
Normal file
File diff suppressed because it is too large
Load diff
764
common/lz4/lz4.h
Normal file
764
common/lz4/lz4.h
Normal file
|
@ -0,0 +1,764 @@
|
||||||
|
/*
|
||||||
|
* LZ4 - Fast LZ compression algorithm
|
||||||
|
* Header File
|
||||||
|
* Copyright (C) 2011-present, Yann Collet.
|
||||||
|
|
||||||
|
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
You can contact the author at :
|
||||||
|
- LZ4 homepage : http://www.lz4.org
|
||||||
|
- LZ4 source repository : https://github.com/lz4/lz4
|
||||||
|
*/
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef LZ4_H_2983827168210
|
||||||
|
#define LZ4_H_2983827168210
|
||||||
|
|
||||||
|
/* --- Dependency --- */
|
||||||
|
#include <stddef.h> /* size_t */
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
Introduction
|
||||||
|
|
||||||
|
LZ4 is lossless compression algorithm, providing compression speed >500 MB/s per core,
|
||||||
|
scalable with multi-cores CPU. It features an extremely fast decoder, with speed in
|
||||||
|
multiple GB/s per core, typically reaching RAM speed limits on multi-core systems.
|
||||||
|
|
||||||
|
The LZ4 compression library provides in-memory compression and decompression functions.
|
||||||
|
It gives full buffer control to user.
|
||||||
|
Compression can be done in:
|
||||||
|
- a single step (described as Simple Functions)
|
||||||
|
- a single step, reusing a context (described in Advanced Functions)
|
||||||
|
- unbounded multiple steps (described as Streaming compression)
|
||||||
|
|
||||||
|
lz4.h generates and decodes LZ4-compressed blocks (doc/lz4_Block_format.md).
|
||||||
|
Decompressing such a compressed block requires additional metadata.
|
||||||
|
Exact metadata depends on exact decompression function.
|
||||||
|
For the typical case of LZ4_decompress_safe(),
|
||||||
|
metadata includes block's compressed size, and maximum bound of decompressed size.
|
||||||
|
Each application is free to encode and pass such metadata in whichever way it wants.
|
||||||
|
|
||||||
|
lz4.h only handle blocks, it can not generate Frames.
|
||||||
|
|
||||||
|
Blocks are different from Frames (doc/lz4_Frame_format.md).
|
||||||
|
Frames bundle both blocks and metadata in a specified manner.
|
||||||
|
Embedding metadata is required for compressed data to be self-contained and portable.
|
||||||
|
Frame format is delivered through a companion API, declared in lz4frame.h.
|
||||||
|
The `lz4` CLI can only manage frames.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*^***************************************************************
|
||||||
|
* Export parameters
|
||||||
|
*****************************************************************/
|
||||||
|
/*
|
||||||
|
* LZ4_DLL_EXPORT :
|
||||||
|
* Enable exporting of functions when building a Windows DLL
|
||||||
|
* LZ4LIB_VISIBILITY :
|
||||||
|
* Control library symbols visibility.
|
||||||
|
*/
|
||||||
|
#ifndef LZ4LIB_VISIBILITY
|
||||||
|
# if defined(__GNUC__) && (__GNUC__ >= 4)
|
||||||
|
# define LZ4LIB_VISIBILITY __attribute__ ((visibility ("default")))
|
||||||
|
# else
|
||||||
|
# define LZ4LIB_VISIBILITY
|
||||||
|
# endif
|
||||||
|
#endif
|
||||||
|
#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1)
|
||||||
|
# define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY
|
||||||
|
#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1)
|
||||||
|
# define LZ4LIB_API __declspec(dllimport) LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
|
||||||
|
#else
|
||||||
|
# define LZ4LIB_API LZ4LIB_VISIBILITY
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*------ Version ------*/
|
||||||
|
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
|
||||||
|
#define LZ4_VERSION_MINOR 9 /* for new (non-breaking) interface capabilities */
|
||||||
|
#define LZ4_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */
|
||||||
|
|
||||||
|
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
|
||||||
|
|
||||||
|
#define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE
|
||||||
|
#define LZ4_QUOTE(str) #str
|
||||||
|
#define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str)
|
||||||
|
#define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION)
|
||||||
|
|
||||||
|
LZ4LIB_API int LZ4_versionNumber (void); /**< library version number; useful to check dll version */
|
||||||
|
LZ4LIB_API const char* LZ4_versionString (void); /**< library version string; useful to check dll version */
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Tuning parameter
|
||||||
|
**************************************/
|
||||||
|
/*!
|
||||||
|
* LZ4_MEMORY_USAGE :
|
||||||
|
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
|
||||||
|
* Increasing memory usage improves compression ratio.
|
||||||
|
* Reduced memory usage may improve speed, thanks to better cache locality.
|
||||||
|
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
|
||||||
|
*/
|
||||||
|
#ifndef LZ4_MEMORY_USAGE
|
||||||
|
# define LZ4_MEMORY_USAGE 14
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Simple Functions
|
||||||
|
**************************************/
|
||||||
|
/*! LZ4_compress_default() :
|
||||||
|
* Compresses 'srcSize' bytes from buffer 'src'
|
||||||
|
* into already allocated 'dst' buffer of size 'dstCapacity'.
|
||||||
|
* Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize).
|
||||||
|
* It also runs faster, so it's a recommended setting.
|
||||||
|
* If the function cannot compress 'src' into a more limited 'dst' budget,
|
||||||
|
* compression stops *immediately*, and the function result is zero.
|
||||||
|
* In which case, 'dst' content is undefined (invalid).
|
||||||
|
* srcSize : max supported value is LZ4_MAX_INPUT_SIZE.
|
||||||
|
* dstCapacity : size of buffer 'dst' (which must be already allocated)
|
||||||
|
* @return : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity)
|
||||||
|
* or 0 if compression fails
|
||||||
|
* Note : This function is protected against buffer overflow scenarios (never writes outside 'dst' buffer, nor read outside 'source' buffer).
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacity);
|
||||||
|
|
||||||
|
/*! LZ4_decompress_safe() :
|
||||||
|
* compressedSize : is the exact complete size of the compressed block.
|
||||||
|
* dstCapacity : is the size of destination buffer (which must be already allocated), presumed an upper bound of decompressed size.
|
||||||
|
* @return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity)
|
||||||
|
* If destination buffer is not large enough, decoding will stop and output an error code (negative value).
|
||||||
|
* If the source stream is detected malformed, the function will stop decoding and return a negative result.
|
||||||
|
* Note 1 : This function is protected against malicious data packets :
|
||||||
|
* it will never writes outside 'dst' buffer, nor read outside 'source' buffer,
|
||||||
|
* even if the compressed block is maliciously modified to order the decoder to do these actions.
|
||||||
|
* In such case, the decoder stops immediately, and considers the compressed block malformed.
|
||||||
|
* Note 2 : compressedSize and dstCapacity must be provided to the function, the compressed block does not contain them.
|
||||||
|
* The implementation is free to send / store / derive this information in whichever way is most beneficial.
|
||||||
|
* If there is a need for a different format which bundles together both compressed data and its metadata, consider looking at lz4frame.h instead.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_decompress_safe (const char* src, char* dst, int compressedSize, int dstCapacity);
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Advanced Functions
|
||||||
|
**************************************/
|
||||||
|
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
|
||||||
|
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
|
||||||
|
|
||||||
|
/*! LZ4_compressBound() :
|
||||||
|
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
|
||||||
|
This function is primarily useful for memory allocation purposes (destination buffer size).
|
||||||
|
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
|
||||||
|
Note that LZ4_compress_default() compresses faster when dstCapacity is >= LZ4_compressBound(srcSize)
|
||||||
|
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
|
||||||
|
return : maximum output size in a "worst case" scenario
|
||||||
|
or 0, if input size is incorrect (too large or negative)
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compressBound(int inputSize);
|
||||||
|
|
||||||
|
/*! LZ4_compress_fast() :
|
||||||
|
Same as LZ4_compress_default(), but allows selection of "acceleration" factor.
|
||||||
|
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
|
||||||
|
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
|
||||||
|
An acceleration value of "1" is the same as regular LZ4_compress_default()
|
||||||
|
Values <= 0 will be replaced by ACCELERATION_DEFAULT (currently == 1, see lz4.c).
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_compress_fast_extState() :
|
||||||
|
* Same as LZ4_compress_fast(), using an externally allocated memory space for its state.
|
||||||
|
* Use LZ4_sizeofState() to know how much memory must be allocated,
|
||||||
|
* and allocate it on 8-bytes boundaries (using `malloc()` typically).
|
||||||
|
* Then, provide this buffer as `void* state` to compression function.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_sizeofState(void);
|
||||||
|
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_compress_destSize() :
|
||||||
|
* Reverse the logic : compresses as much data as possible from 'src' buffer
|
||||||
|
* into already allocated buffer 'dst', of size >= 'targetDestSize'.
|
||||||
|
* This function either compresses the entire 'src' content into 'dst' if it's large enough,
|
||||||
|
* or fill 'dst' buffer completely with as much data as possible from 'src'.
|
||||||
|
* note: acceleration parameter is fixed to "default".
|
||||||
|
*
|
||||||
|
* *srcSizePtr : will be modified to indicate how many bytes where read from 'src' to fill 'dst'.
|
||||||
|
* New value is necessarily <= input value.
|
||||||
|
* @return : Nb bytes written into 'dst' (necessarily <= targetDestSize)
|
||||||
|
* or 0 if compression fails.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_decompress_safe_partial() :
|
||||||
|
* Decompress an LZ4 compressed block, of size 'srcSize' at position 'src',
|
||||||
|
* into destination buffer 'dst' of size 'dstCapacity'.
|
||||||
|
* Up to 'targetOutputSize' bytes will be decoded.
|
||||||
|
* The function stops decoding on reaching this objective,
|
||||||
|
* which can boost performance when only the beginning of a block is required.
|
||||||
|
*
|
||||||
|
* @return : the number of bytes decoded in `dst` (necessarily <= dstCapacity)
|
||||||
|
* If source stream is detected malformed, function returns a negative result.
|
||||||
|
*
|
||||||
|
* Note : @return can be < targetOutputSize, if compressed block contains less data.
|
||||||
|
*
|
||||||
|
* Note 2 : this function features 2 parameters, targetOutputSize and dstCapacity,
|
||||||
|
* and expects targetOutputSize <= dstCapacity.
|
||||||
|
* It effectively stops decoding on reaching targetOutputSize,
|
||||||
|
* so dstCapacity is kind of redundant.
|
||||||
|
* This is because in a previous version of this function,
|
||||||
|
* decoding operation would not "break" a sequence in the middle.
|
||||||
|
* As a consequence, there was no guarantee that decoding would stop at exactly targetOutputSize,
|
||||||
|
* it could write more bytes, though only up to dstCapacity.
|
||||||
|
* Some "margin" used to be required for this operation to work properly.
|
||||||
|
* This is no longer necessary.
|
||||||
|
* The function nonetheless keeps its signature, in an effort to not break API.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_decompress_safe_partial (const char* src, char* dst, int srcSize, int targetOutputSize, int dstCapacity);
|
||||||
|
|
||||||
|
|
||||||
|
/*-*********************************************
|
||||||
|
* Streaming Compression Functions
|
||||||
|
***********************************************/
|
||||||
|
typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */
|
||||||
|
|
||||||
|
LZ4LIB_API LZ4_stream_t* LZ4_createStream(void);
|
||||||
|
LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr);
|
||||||
|
|
||||||
|
/*! LZ4_resetStream_fast() : v1.9.0+
|
||||||
|
* Use this to prepare an LZ4_stream_t for a new chain of dependent blocks
|
||||||
|
* (e.g., LZ4_compress_fast_continue()).
|
||||||
|
*
|
||||||
|
* An LZ4_stream_t must be initialized once before usage.
|
||||||
|
* This is automatically done when created by LZ4_createStream().
|
||||||
|
* However, should the LZ4_stream_t be simply declared on stack (for example),
|
||||||
|
* it's necessary to initialize it first, using LZ4_initStream().
|
||||||
|
*
|
||||||
|
* After init, start any new stream with LZ4_resetStream_fast().
|
||||||
|
* A same LZ4_stream_t can be re-used multiple times consecutively
|
||||||
|
* and compress multiple streams,
|
||||||
|
* provided that it starts each new stream with LZ4_resetStream_fast().
|
||||||
|
*
|
||||||
|
* LZ4_resetStream_fast() is much faster than LZ4_initStream(),
|
||||||
|
* but is not compatible with memory regions containing garbage data.
|
||||||
|
*
|
||||||
|
* Note: it's only useful to call LZ4_resetStream_fast()
|
||||||
|
* in the context of streaming compression.
|
||||||
|
* The *extState* functions perform their own resets.
|
||||||
|
* Invoking LZ4_resetStream_fast() before is redundant, and even counterproductive.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API void LZ4_resetStream_fast (LZ4_stream_t* streamPtr);
|
||||||
|
|
||||||
|
/*! LZ4_loadDict() :
|
||||||
|
* Use this function to reference a static dictionary into LZ4_stream_t.
|
||||||
|
* The dictionary must remain available during compression.
|
||||||
|
* LZ4_loadDict() triggers a reset, so any previous data will be forgotten.
|
||||||
|
* The same dictionary will have to be loaded on decompression side for successful decoding.
|
||||||
|
* Dictionary are useful for better compression of small data (KB range).
|
||||||
|
* While LZ4 accept any input as dictionary,
|
||||||
|
* results are generally better when using Zstandard's Dictionary Builder.
|
||||||
|
* Loading a size of 0 is allowed, and is the same as reset.
|
||||||
|
* @return : loaded dictionary size, in bytes (necessarily <= 64 KB)
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
|
||||||
|
|
||||||
|
/*! LZ4_compress_fast_continue() :
|
||||||
|
* Compress 'src' content using data from previously compressed blocks, for better compression ratio.
|
||||||
|
* 'dst' buffer must be already allocated.
|
||||||
|
* If dstCapacity >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
|
||||||
|
*
|
||||||
|
* @return : size of compressed block
|
||||||
|
* or 0 if there is an error (typically, cannot fit into 'dst').
|
||||||
|
*
|
||||||
|
* Note 1 : Each invocation to LZ4_compress_fast_continue() generates a new block.
|
||||||
|
* Each block has precise boundaries.
|
||||||
|
* Each block must be decompressed separately, calling LZ4_decompress_*() with relevant metadata.
|
||||||
|
* It's not possible to append blocks together and expect a single invocation of LZ4_decompress_*() to decompress them together.
|
||||||
|
*
|
||||||
|
* Note 2 : The previous 64KB of source data is __assumed__ to remain present, unmodified, at same address in memory !
|
||||||
|
*
|
||||||
|
* Note 3 : When input is structured as a double-buffer, each buffer can have any size, including < 64 KB.
|
||||||
|
* Make sure that buffers are separated, by at least one byte.
|
||||||
|
* This construction ensures that each block only depends on previous block.
|
||||||
|
*
|
||||||
|
* Note 4 : If input buffer is a ring-buffer, it can have any size, including < 64 KB.
|
||||||
|
*
|
||||||
|
* Note 5 : After an error, the stream status is undefined (invalid), it can only be reset or freed.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||||
|
|
||||||
|
/*! LZ4_saveDict() :
|
||||||
|
* If last 64KB data cannot be guaranteed to remain available at its current memory location,
|
||||||
|
* save it into a safer place (char* safeBuffer).
|
||||||
|
* This is schematically equivalent to a memcpy() followed by LZ4_loadDict(),
|
||||||
|
* but is much faster, because LZ4_saveDict() doesn't need to rebuild tables.
|
||||||
|
* @return : saved dictionary size in bytes (necessarily <= maxDictSize), or 0 if error.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int maxDictSize);
|
||||||
|
|
||||||
|
|
||||||
|
/*-**********************************************
|
||||||
|
* Streaming Decompression Functions
|
||||||
|
* Bufferless synchronous API
|
||||||
|
************************************************/
|
||||||
|
typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* tracking context */
|
||||||
|
|
||||||
|
/*! LZ4_createStreamDecode() and LZ4_freeStreamDecode() :
|
||||||
|
* creation / destruction of streaming decompression tracking context.
|
||||||
|
* A tracking context can be re-used multiple times.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void);
|
||||||
|
LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
|
||||||
|
|
||||||
|
/*! LZ4_setStreamDecode() :
|
||||||
|
* An LZ4_streamDecode_t context can be allocated once and re-used multiple times.
|
||||||
|
* Use this function to start decompression of a new stream of blocks.
|
||||||
|
* A dictionary can optionally be set. Use NULL or size 0 for a reset order.
|
||||||
|
* Dictionary is presumed stable : it must remain accessible and unmodified during next decompression.
|
||||||
|
* @return : 1 if OK, 0 if error
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
|
||||||
|
|
||||||
|
/*! LZ4_decoderRingBufferSize() : v1.8.2+
|
||||||
|
* Note : in a ring buffer scenario (optional),
|
||||||
|
* blocks are presumed decompressed next to each other
|
||||||
|
* up to the moment there is not enough remaining space for next block (remainingSize < maxBlockSize),
|
||||||
|
* at which stage it resumes from beginning of ring buffer.
|
||||||
|
* When setting such a ring buffer for streaming decompression,
|
||||||
|
* provides the minimum size of this ring buffer
|
||||||
|
* to be compatible with any source respecting maxBlockSize condition.
|
||||||
|
* @return : minimum ring buffer size,
|
||||||
|
* or 0 if there is an error (invalid maxBlockSize).
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_decoderRingBufferSize(int maxBlockSize);
|
||||||
|
#define LZ4_DECODER_RING_BUFFER_SIZE(maxBlockSize) (65536 + 14 + (maxBlockSize)) /* for static allocation; maxBlockSize presumed valid */
|
||||||
|
|
||||||
|
/*! LZ4_decompress_*_continue() :
|
||||||
|
* These decoding functions allow decompression of consecutive blocks in "streaming" mode.
|
||||||
|
* A block is an unsplittable entity, it must be presented entirely to a decompression function.
|
||||||
|
* Decompression functions only accepts one block at a time.
|
||||||
|
* The last 64KB of previously decoded data *must* remain available and unmodified at the memory position where they were decoded.
|
||||||
|
* If less than 64KB of data has been decoded, all the data must be present.
|
||||||
|
*
|
||||||
|
* Special : if decompression side sets a ring buffer, it must respect one of the following conditions :
|
||||||
|
* - Decompression buffer size is _at least_ LZ4_decoderRingBufferSize(maxBlockSize).
|
||||||
|
* maxBlockSize is the maximum size of any single block. It can have any value > 16 bytes.
|
||||||
|
* In which case, encoding and decoding buffers do not need to be synchronized.
|
||||||
|
* Actually, data can be produced by any source compliant with LZ4 format specification, and respecting maxBlockSize.
|
||||||
|
* - Synchronized mode :
|
||||||
|
* Decompression buffer size is _exactly_ the same as compression buffer size,
|
||||||
|
* and follows exactly same update rule (block boundaries at same positions),
|
||||||
|
* and decoding function is provided with exact decompressed size of each block (exception for last block of the stream),
|
||||||
|
* _then_ decoding & encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||||
|
* - Decompression buffer is larger than encoding buffer, by a minimum of maxBlockSize more bytes.
|
||||||
|
* In which case, encoding and decoding buffers do not need to be synchronized,
|
||||||
|
* and encoding ring buffer can have any size, including small ones ( < 64 KB).
|
||||||
|
*
|
||||||
|
* Whenever these conditions are not possible,
|
||||||
|
* save the last 64KB of decoded data into a safe buffer where it can't be modified during decompression,
|
||||||
|
* then indicate where this data is saved using LZ4_setStreamDecode(), before decompressing next block.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int srcSize, int dstCapacity);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_decompress_*_usingDict() :
|
||||||
|
* These decoding functions work the same as
|
||||||
|
* a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue()
|
||||||
|
* They are stand-alone, and don't need an LZ4_streamDecode_t structure.
|
||||||
|
* Dictionary is presumed stable : it must remain accessible and unmodified during decompression.
|
||||||
|
* Performance tip : Decompression speed can be substantially increased
|
||||||
|
* when dst == dictStart + dictSize.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* src, char* dst, int srcSize, int dstCapcity, const char* dictStart, int dictSize);
|
||||||
|
|
||||||
|
#endif /* LZ4_H_2983827168210 */
|
||||||
|
|
||||||
|
|
||||||
|
/*^*************************************
|
||||||
|
* !!!!!! STATIC LINKING ONLY !!!!!!
|
||||||
|
***************************************/
|
||||||
|
|
||||||
|
/*-****************************************************************************
|
||||||
|
* Experimental section
|
||||||
|
*
|
||||||
|
* Symbols declared in this section must be considered unstable. Their
|
||||||
|
* signatures or semantics may change, or they may be removed altogether in the
|
||||||
|
* future. They are therefore only safe to depend on when the caller is
|
||||||
|
* statically linked against the library.
|
||||||
|
*
|
||||||
|
* To protect against unsafe usage, not only are the declarations guarded,
|
||||||
|
* the definitions are hidden by default
|
||||||
|
* when building LZ4 as a shared/dynamic library.
|
||||||
|
*
|
||||||
|
* In order to access these declarations,
|
||||||
|
* define LZ4_STATIC_LINKING_ONLY in your application
|
||||||
|
* before including LZ4's headers.
|
||||||
|
*
|
||||||
|
* In order to make their implementations accessible dynamically, you must
|
||||||
|
* define LZ4_PUBLISH_STATIC_FUNCTIONS when building the LZ4 library.
|
||||||
|
******************************************************************************/
|
||||||
|
|
||||||
|
#ifdef LZ4_STATIC_LINKING_ONLY
|
||||||
|
|
||||||
|
#ifndef LZ4_STATIC_3504398509
|
||||||
|
#define LZ4_STATIC_3504398509
|
||||||
|
|
||||||
|
#ifdef LZ4_PUBLISH_STATIC_FUNCTIONS
|
||||||
|
#define LZ4LIB_STATIC_API LZ4LIB_API
|
||||||
|
#else
|
||||||
|
#define LZ4LIB_STATIC_API
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_compress_fast_extState_fastReset() :
|
||||||
|
* A variant of LZ4_compress_fast_extState().
|
||||||
|
*
|
||||||
|
* Using this variant avoids an expensive initialization step.
|
||||||
|
* It is only safe to call if the state buffer is known to be correctly initialized already
|
||||||
|
* (see above comment on LZ4_resetStream_fast() for a definition of "correctly initialized").
|
||||||
|
* From a high level, the difference is that
|
||||||
|
* this function initializes the provided state with a call to something like LZ4_resetStream_fast()
|
||||||
|
* while LZ4_compress_fast_extState() starts with a call to LZ4_resetStream().
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
|
||||||
|
|
||||||
|
/*! LZ4_attach_dictionary() :
|
||||||
|
* This is an experimental API that allows
|
||||||
|
* efficient use of a static dictionary many times.
|
||||||
|
*
|
||||||
|
* Rather than re-loading the dictionary buffer into a working context before
|
||||||
|
* each compression, or copying a pre-loaded dictionary's LZ4_stream_t into a
|
||||||
|
* working LZ4_stream_t, this function introduces a no-copy setup mechanism,
|
||||||
|
* in which the working stream references the dictionary stream in-place.
|
||||||
|
*
|
||||||
|
* Several assumptions are made about the state of the dictionary stream.
|
||||||
|
* Currently, only streams which have been prepared by LZ4_loadDict() should
|
||||||
|
* be expected to work.
|
||||||
|
*
|
||||||
|
* Alternatively, the provided dictionaryStream may be NULL,
|
||||||
|
* in which case any existing dictionary stream is unset.
|
||||||
|
*
|
||||||
|
* If a dictionary is provided, it replaces any pre-existing stream history.
|
||||||
|
* The dictionary contents are the only history that can be referenced and
|
||||||
|
* logically immediately precede the data compressed in the first subsequent
|
||||||
|
* compression call.
|
||||||
|
*
|
||||||
|
* The dictionary will only remain attached to the working stream through the
|
||||||
|
* first compression call, at the end of which it is cleared. The dictionary
|
||||||
|
* stream (and source buffer) must remain in-place / accessible / unchanged
|
||||||
|
* through the completion of the first compression call on the stream.
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API void LZ4_attach_dictionary(LZ4_stream_t* workingStream, const LZ4_stream_t* dictionaryStream);
|
||||||
|
|
||||||
|
|
||||||
|
/*! In-place compression and decompression
|
||||||
|
*
|
||||||
|
* It's possible to have input and output sharing the same buffer,
|
||||||
|
* for highly contrained memory environments.
|
||||||
|
* In both cases, it requires input to lay at the end of the buffer,
|
||||||
|
* and decompression to start at beginning of the buffer.
|
||||||
|
* Buffer size must feature some margin, hence be larger than final size.
|
||||||
|
*
|
||||||
|
* |<------------------------buffer--------------------------------->|
|
||||||
|
* |<-----------compressed data--------->|
|
||||||
|
* |<-----------decompressed size------------------>|
|
||||||
|
* |<----margin---->|
|
||||||
|
*
|
||||||
|
* This technique is more useful for decompression,
|
||||||
|
* since decompressed size is typically larger,
|
||||||
|
* and margin is short.
|
||||||
|
*
|
||||||
|
* In-place decompression will work inside any buffer
|
||||||
|
* which size is >= LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize).
|
||||||
|
* This presumes that decompressedSize > compressedSize.
|
||||||
|
* Otherwise, it means compression actually expanded data,
|
||||||
|
* and it would be more efficient to store such data with a flag indicating it's not compressed.
|
||||||
|
* This can happen when data is not compressible (already compressed, or encrypted).
|
||||||
|
*
|
||||||
|
* For in-place compression, margin is larger, as it must be able to cope with both
|
||||||
|
* history preservation, requiring input data to remain unmodified up to LZ4_DISTANCE_MAX,
|
||||||
|
* and data expansion, which can happen when input is not compressible.
|
||||||
|
* As a consequence, buffer size requirements are much higher,
|
||||||
|
* and memory savings offered by in-place compression are more limited.
|
||||||
|
*
|
||||||
|
* There are ways to limit this cost for compression :
|
||||||
|
* - Reduce history size, by modifying LZ4_DISTANCE_MAX.
|
||||||
|
* Note that it is a compile-time constant, so all compressions will apply this limit.
|
||||||
|
* Lower values will reduce compression ratio, except when input_size < LZ4_DISTANCE_MAX,
|
||||||
|
* so it's a reasonable trick when inputs are known to be small.
|
||||||
|
* - Require the compressor to deliver a "maximum compressed size".
|
||||||
|
* This is the `dstCapacity` parameter in `LZ4_compress*()`.
|
||||||
|
* When this size is < LZ4_COMPRESSBOUND(inputSize), then compression can fail,
|
||||||
|
* in which case, the return code will be 0 (zero).
|
||||||
|
* The caller must be ready for these cases to happen,
|
||||||
|
* and typically design a backup scheme to send data uncompressed.
|
||||||
|
* The combination of both techniques can significantly reduce
|
||||||
|
* the amount of margin required for in-place compression.
|
||||||
|
*
|
||||||
|
* In-place compression can work in any buffer
|
||||||
|
* which size is >= (maxCompressedSize)
|
||||||
|
* with maxCompressedSize == LZ4_COMPRESSBOUND(srcSize) for guaranteed compression success.
|
||||||
|
* LZ4_COMPRESS_INPLACE_BUFFER_SIZE() depends on both maxCompressedSize and LZ4_DISTANCE_MAX,
|
||||||
|
* so it's possible to reduce memory requirements by playing with them.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#define LZ4_DECOMPRESS_INPLACE_MARGIN(compressedSize) (((compressedSize) >> 8) + 32)
|
||||||
|
#define LZ4_DECOMPRESS_INPLACE_BUFFER_SIZE(decompressedSize) ((decompressedSize) + LZ4_DECOMPRESS_INPLACE_MARGIN(decompressedSize)) /**< note: presumes that compressedSize < decompressedSize. note2: margin is overestimated a bit, since it could use compressedSize instead */
|
||||||
|
|
||||||
|
#ifndef LZ4_DISTANCE_MAX /* history window size; can be user-defined at compile time */
|
||||||
|
# define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#define LZ4_COMPRESS_INPLACE_MARGIN (LZ4_DISTANCE_MAX + 32) /* LZ4_DISTANCE_MAX can be safely replaced by srcSize when it's smaller */
|
||||||
|
#define LZ4_COMPRESS_INPLACE_BUFFER_SIZE(maxCompressedSize) ((maxCompressedSize) + LZ4_COMPRESS_INPLACE_MARGIN) /**< maxCompressedSize is generally LZ4_COMPRESSBOUND(inputSize), but can be set to any lower value, with the risk that compression can fail (return code 0(zero)) */
|
||||||
|
|
||||||
|
#endif /* LZ4_STATIC_3504398509 */
|
||||||
|
#endif /* LZ4_STATIC_LINKING_ONLY */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#ifndef LZ4_H_98237428734687
|
||||||
|
#define LZ4_H_98237428734687
|
||||||
|
|
||||||
|
/*-************************************************************
|
||||||
|
* PRIVATE DEFINITIONS
|
||||||
|
**************************************************************
|
||||||
|
* Do not use these definitions directly.
|
||||||
|
* They are only exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`.
|
||||||
|
* Accessing members will expose code to API and/or ABI break in future versions of the library.
|
||||||
|
**************************************************************/
|
||||||
|
#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
|
||||||
|
#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
|
||||||
|
#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */
|
||||||
|
|
||||||
|
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
|
||||||
|
struct LZ4_stream_t_internal {
|
||||||
|
uint32_t hashTable[LZ4_HASH_SIZE_U32];
|
||||||
|
uint32_t currentOffset;
|
||||||
|
uint16_t dirty;
|
||||||
|
uint16_t tableType;
|
||||||
|
const uint8_t* dictionary;
|
||||||
|
const LZ4_stream_t_internal* dictCtx;
|
||||||
|
uint32_t dictSize;
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
const uint8_t* externalDict;
|
||||||
|
size_t extDictSize;
|
||||||
|
const uint8_t* prefixEnd;
|
||||||
|
size_t prefixSize;
|
||||||
|
} LZ4_streamDecode_t_internal;
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
typedef struct LZ4_stream_t_internal LZ4_stream_t_internal;
|
||||||
|
struct LZ4_stream_t_internal {
|
||||||
|
unsigned int hashTable[LZ4_HASH_SIZE_U32];
|
||||||
|
unsigned int currentOffset;
|
||||||
|
unsigned short dirty;
|
||||||
|
unsigned short tableType;
|
||||||
|
const unsigned char* dictionary;
|
||||||
|
const LZ4_stream_t_internal* dictCtx;
|
||||||
|
unsigned int dictSize;
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
const unsigned char* externalDict;
|
||||||
|
const unsigned char* prefixEnd;
|
||||||
|
size_t extDictSize;
|
||||||
|
size_t prefixSize;
|
||||||
|
} LZ4_streamDecode_t_internal;
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*! LZ4_stream_t :
|
||||||
|
* information structure to track an LZ4 stream.
|
||||||
|
* LZ4_stream_t can also be created using LZ4_createStream(), which is recommended.
|
||||||
|
* The structure definition can be convenient for static allocation
|
||||||
|
* (on stack, or as part of larger structure).
|
||||||
|
* Init this structure with LZ4_initStream() before first use.
|
||||||
|
* note : only use this definition in association with static linking !
|
||||||
|
* this definition is not API/ABI safe, and may change in a future version.
|
||||||
|
*/
|
||||||
|
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4 + ((sizeof(void*)==16) ? 4 : 0) /*AS-400*/ )
|
||||||
|
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
|
||||||
|
union LZ4_stream_u {
|
||||||
|
unsigned long long table[LZ4_STREAMSIZE_U64];
|
||||||
|
LZ4_stream_t_internal internal_donotuse;
|
||||||
|
} ; /* previously typedef'd to LZ4_stream_t */
|
||||||
|
|
||||||
|
/*! LZ4_initStream() : v1.9.0+
|
||||||
|
* An LZ4_stream_t structure must be initialized at least once.
|
||||||
|
* This is automatically done when invoking LZ4_createStream(),
|
||||||
|
* but it's not when the structure is simply declared on stack (for example).
|
||||||
|
*
|
||||||
|
* Use LZ4_initStream() to properly initialize a newly declared LZ4_stream_t.
|
||||||
|
* It can also initialize any arbitrary buffer of sufficient size,
|
||||||
|
* and will @return a pointer of proper type upon initialization.
|
||||||
|
*
|
||||||
|
* Note : initialization fails if size and alignment conditions are not respected.
|
||||||
|
* In which case, the function will @return NULL.
|
||||||
|
* Note2: An LZ4_stream_t structure guarantees correct alignment and size.
|
||||||
|
* Note3: Before v1.9.0, use LZ4_resetStream() instead
|
||||||
|
*/
|
||||||
|
LZ4LIB_API LZ4_stream_t* LZ4_initStream (void* buffer, size_t size);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_streamDecode_t :
|
||||||
|
* information structure to track an LZ4 stream during decompression.
|
||||||
|
* init this structure using LZ4_setStreamDecode() before first use.
|
||||||
|
* note : only use in association with static linking !
|
||||||
|
* this definition is not API/ABI safe,
|
||||||
|
* and may change in a future version !
|
||||||
|
*/
|
||||||
|
#define LZ4_STREAMDECODESIZE_U64 (4 + ((sizeof(void*)==16) ? 2 : 0) /*AS-400*/ )
|
||||||
|
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
|
||||||
|
union LZ4_streamDecode_u {
|
||||||
|
unsigned long long table[LZ4_STREAMDECODESIZE_U64];
|
||||||
|
LZ4_streamDecode_t_internal internal_donotuse;
|
||||||
|
} ; /* previously typedef'd to LZ4_streamDecode_t */
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Obsolete Functions
|
||||||
|
**************************************/
|
||||||
|
|
||||||
|
/*! Deprecation warnings
|
||||||
|
*
|
||||||
|
* Deprecated functions make the compiler generate a warning when invoked.
|
||||||
|
* This is meant to invite users to update their source code.
|
||||||
|
* Should deprecation warnings be a problem, it is generally possible to disable them,
|
||||||
|
* typically with -Wno-deprecated-declarations for gcc
|
||||||
|
* or _CRT_SECURE_NO_WARNINGS in Visual.
|
||||||
|
*
|
||||||
|
* Another method is to define LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||||
|
* before including the header file.
|
||||||
|
*/
|
||||||
|
#ifdef LZ4_DISABLE_DEPRECATE_WARNINGS
|
||||||
|
# define LZ4_DEPRECATED(message) /* disable deprecation warnings */
|
||||||
|
#else
|
||||||
|
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
|
||||||
|
# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */
|
||||||
|
# define LZ4_DEPRECATED(message) [[deprecated(message)]]
|
||||||
|
# elif (LZ4_GCC_VERSION >= 405) || defined(__clang__)
|
||||||
|
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
|
||||||
|
# elif (LZ4_GCC_VERSION >= 301)
|
||||||
|
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
|
||||||
|
# elif defined(_MSC_VER)
|
||||||
|
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
|
||||||
|
# else
|
||||||
|
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
|
||||||
|
# define LZ4_DEPRECATED(message)
|
||||||
|
# endif
|
||||||
|
#endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */
|
||||||
|
|
||||||
|
/* Obsolete compression functions */
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress (const char* src, char* dest, int srcSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_default() instead") LZ4LIB_API int LZ4_compress_limitedOutput (const char* src, char* dest, int srcSize, int maxOutputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") LZ4LIB_API int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") LZ4LIB_API int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||||
|
|
||||||
|
/* Obsolete decompression functions */
|
||||||
|
LZ4_DEPRECATED("use LZ4_decompress_fast() instead") LZ4LIB_API int LZ4_uncompress (const char* source, char* dest, int outputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_decompress_safe() instead") LZ4LIB_API int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize);
|
||||||
|
|
||||||
|
/* Obsolete streaming functions; degraded functionality; do not use!
|
||||||
|
*
|
||||||
|
* In order to perform streaming compression, these functions depended on data
|
||||||
|
* that is no longer tracked in the state. They have been preserved as well as
|
||||||
|
* possible: using them will still produce a correct output. However, they don't
|
||||||
|
* actually retain any history between compression calls. The compression ratio
|
||||||
|
* achieved will therefore be no better than compressing each chunk
|
||||||
|
* independently.
|
||||||
|
*/
|
||||||
|
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API void* LZ4_create (char* inputBuffer);
|
||||||
|
LZ4_DEPRECATED("Use LZ4_createStream() instead") LZ4LIB_API int LZ4_sizeofStreamState(void);
|
||||||
|
LZ4_DEPRECATED("Use LZ4_resetStream() instead") LZ4LIB_API int LZ4_resetStreamState(void* state, char* inputBuffer);
|
||||||
|
LZ4_DEPRECATED("Use LZ4_saveDict() instead") LZ4LIB_API char* LZ4_slideInputBuffer (void* state);
|
||||||
|
|
||||||
|
/* Obsolete streaming decoding functions */
|
||||||
|
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") LZ4LIB_API int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") LZ4LIB_API int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
|
||||||
|
|
||||||
|
/*! LZ4_decompress_fast() : **unsafe!**
|
||||||
|
* These functions used to be faster than LZ4_decompress_safe(),
|
||||||
|
* but it has changed, and they are now slower than LZ4_decompress_safe().
|
||||||
|
* This is because LZ4_decompress_fast() doesn't know the input size,
|
||||||
|
* and therefore must progress more cautiously in the input buffer to not read beyond the end of block.
|
||||||
|
* On top of that `LZ4_decompress_fast()` is not protected vs malformed or malicious inputs, making it a security liability.
|
||||||
|
* As a consequence, LZ4_decompress_fast() is strongly discouraged, and deprecated.
|
||||||
|
*
|
||||||
|
* The last remaining LZ4_decompress_fast() specificity is that
|
||||||
|
* it can decompress a block without knowing its compressed size.
|
||||||
|
* Such functionality could be achieved in a more secure manner,
|
||||||
|
* by also providing the maximum size of input buffer,
|
||||||
|
* but it would require new prototypes, and adaptation of the implementation to this new use case.
|
||||||
|
*
|
||||||
|
* Parameters:
|
||||||
|
* originalSize : is the uncompressed size to regenerate.
|
||||||
|
* `dst` must be already allocated, its size must be >= 'originalSize' bytes.
|
||||||
|
* @return : number of bytes read from source buffer (== compressed size).
|
||||||
|
* The function expects to finish at block's end exactly.
|
||||||
|
* If the source stream is detected malformed, the function stops decoding and returns a negative result.
|
||||||
|
* note : LZ4_decompress_fast*() requires originalSize. Thanks to this information, it never writes past the output buffer.
|
||||||
|
* However, since it doesn't know its 'src' size, it may read an unknown amount of input, past input buffer bounds.
|
||||||
|
* Also, since match offsets are not validated, match reads from 'src' may underflow too.
|
||||||
|
* These issues never happen if input (compressed) data is correct.
|
||||||
|
* But they may happen if input data is invalid (error or intentional tampering).
|
||||||
|
* As a consequence, use these functions in trusted environments with trusted data **only**.
|
||||||
|
*/
|
||||||
|
|
||||||
|
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe() instead")
|
||||||
|
LZ4LIB_API int LZ4_decompress_fast (const char* src, char* dst, int originalSize);
|
||||||
|
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_continue() instead")
|
||||||
|
LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* src, char* dst, int originalSize);
|
||||||
|
LZ4_DEPRECATED("This function is deprecated and unsafe. Consider using LZ4_decompress_safe_usingDict() instead")
|
||||||
|
LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* src, char* dst, int originalSize, const char* dictStart, int dictSize);
|
||||||
|
|
||||||
|
/*! LZ4_resetStream() :
|
||||||
|
* An LZ4_stream_t structure must be initialized at least once.
|
||||||
|
* This is done with LZ4_initStream(), or LZ4_resetStream().
|
||||||
|
* Consider switching to LZ4_initStream(),
|
||||||
|
* invoking LZ4_resetStream() will trigger deprecation warnings in the future.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr);
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* LZ4_H_98237428734687 */
|
||||||
|
|
||||||
|
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
}
|
||||||
|
#endif
|
1538
common/lz4/lz4hc.c
Normal file
1538
common/lz4/lz4hc.c
Normal file
File diff suppressed because it is too large
Load diff
438
common/lz4/lz4hc.h
Normal file
438
common/lz4/lz4hc.h
Normal file
|
@ -0,0 +1,438 @@
|
||||||
|
/*
|
||||||
|
LZ4 HC - High Compression Mode of LZ4
|
||||||
|
Header File
|
||||||
|
Copyright (C) 2011-2017, Yann Collet.
|
||||||
|
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
You can contact the author at :
|
||||||
|
- LZ4 source repository : https://github.com/lz4/lz4
|
||||||
|
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
|
||||||
|
*/
|
||||||
|
#ifndef LZ4_HC_H_19834876238432
|
||||||
|
#define LZ4_HC_H_19834876238432
|
||||||
|
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* --- Dependency --- */
|
||||||
|
/* note : lz4hc requires lz4.h/lz4.c for compilation */
|
||||||
|
#include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */
|
||||||
|
|
||||||
|
|
||||||
|
/* --- Useful constants --- */
|
||||||
|
#define LZ4HC_CLEVEL_MIN 3
|
||||||
|
#define LZ4HC_CLEVEL_DEFAULT 9
|
||||||
|
#define LZ4HC_CLEVEL_OPT_MIN 10
|
||||||
|
#define LZ4HC_CLEVEL_MAX 12
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Block Compression
|
||||||
|
**************************************/
|
||||||
|
/*! LZ4_compress_HC() :
|
||||||
|
* Compress data from `src` into `dst`, using the powerful but slower "HC" algorithm.
|
||||||
|
* `dst` must be already allocated.
|
||||||
|
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
|
||||||
|
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
|
||||||
|
* `compressionLevel` : any value between 1 and LZ4HC_CLEVEL_MAX will work.
|
||||||
|
* Values > LZ4HC_CLEVEL_MAX behave the same as LZ4HC_CLEVEL_MAX.
|
||||||
|
* @return : the number of bytes written into 'dst'
|
||||||
|
* or 0 if compression fails.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel);
|
||||||
|
|
||||||
|
|
||||||
|
/* Note :
|
||||||
|
* Decompression functions are provided within "lz4.h" (BSD license)
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_compress_HC_extStateHC() :
|
||||||
|
* Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`.
|
||||||
|
* `state` size is provided by LZ4_sizeofStateHC().
|
||||||
|
* Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() should do properly).
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_sizeofStateHC(void);
|
||||||
|
LZ4LIB_API int LZ4_compress_HC_extStateHC(void* stateHC, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
|
||||||
|
|
||||||
|
|
||||||
|
/*! LZ4_compress_HC_destSize() : v1.9.0+
|
||||||
|
* Will compress as much data as possible from `src`
|
||||||
|
* to fit into `targetDstSize` budget.
|
||||||
|
* Result is provided in 2 parts :
|
||||||
|
* @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)
|
||||||
|
* or 0 if compression fails.
|
||||||
|
* `srcSizePtr` : on success, *srcSizePtr is updated to indicate how much bytes were read from `src`
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_HC_destSize(void* stateHC,
|
||||||
|
const char* src, char* dst,
|
||||||
|
int* srcSizePtr, int targetDstSize,
|
||||||
|
int compressionLevel);
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Streaming Compression
|
||||||
|
* Bufferless synchronous API
|
||||||
|
**************************************/
|
||||||
|
typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */
|
||||||
|
|
||||||
|
/*! LZ4_createStreamHC() and LZ4_freeStreamHC() :
|
||||||
|
* These functions create and release memory for LZ4 HC streaming state.
|
||||||
|
* Newly created states are automatically initialized.
|
||||||
|
* A same state can be used multiple times consecutively,
|
||||||
|
* starting with LZ4_resetStreamHC_fast() to start a new stream of blocks.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void);
|
||||||
|
LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
|
||||||
|
|
||||||
|
/*
|
||||||
|
These functions compress data in successive blocks of any size,
|
||||||
|
using previous blocks as dictionary, to improve compression ratio.
|
||||||
|
One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
|
||||||
|
There is an exception for ring buffers, which can be smaller than 64 KB.
|
||||||
|
Ring-buffer scenario is automatically detected and handled within LZ4_compress_HC_continue().
|
||||||
|
|
||||||
|
Before starting compression, state must be allocated and properly initialized.
|
||||||
|
LZ4_createStreamHC() does both, though compression level is set to LZ4HC_CLEVEL_DEFAULT.
|
||||||
|
|
||||||
|
Selecting the compression level can be done with LZ4_resetStreamHC_fast() (starts a new stream)
|
||||||
|
or LZ4_setCompressionLevel() (anytime, between blocks in the same stream) (experimental).
|
||||||
|
LZ4_resetStreamHC_fast() only works on states which have been properly initialized at least once,
|
||||||
|
which is automatically the case when state is created using LZ4_createStreamHC().
|
||||||
|
|
||||||
|
After reset, a first "fictional block" can be designated as initial dictionary,
|
||||||
|
using LZ4_loadDictHC() (Optional).
|
||||||
|
|
||||||
|
Invoke LZ4_compress_HC_continue() to compress each successive block.
|
||||||
|
The number of blocks is unlimited.
|
||||||
|
Previous input blocks, including initial dictionary when present,
|
||||||
|
must remain accessible and unmodified during compression.
|
||||||
|
|
||||||
|
It's allowed to update compression level anytime between blocks,
|
||||||
|
using LZ4_setCompressionLevel() (experimental).
|
||||||
|
|
||||||
|
'dst' buffer should be sized to handle worst case scenarios
|
||||||
|
(see LZ4_compressBound(), it ensures compression success).
|
||||||
|
In case of failure, the API does not guarantee recovery,
|
||||||
|
so the state _must_ be reset.
|
||||||
|
To ensure compression success
|
||||||
|
whenever `dst` buffer size cannot be made >= LZ4_compressBound(),
|
||||||
|
consider using LZ4_compress_HC_continue_destSize().
|
||||||
|
|
||||||
|
Whenever previous input blocks can't be preserved unmodified in-place during compression of next blocks,
|
||||||
|
it's possible to copy the last blocks into a more stable memory space, using LZ4_saveDictHC().
|
||||||
|
Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer' (<= 64 KB)
|
||||||
|
|
||||||
|
After completing a streaming compression,
|
||||||
|
it's possible to start a new stream of blocks, using the same LZ4_streamHC_t state,
|
||||||
|
just by resetting it, using LZ4_resetStreamHC_fast().
|
||||||
|
*/
|
||||||
|
|
||||||
|
LZ4LIB_API void LZ4_resetStreamHC_fast(LZ4_streamHC_t* streamHCPtr, int compressionLevel); /* v1.9.0+ */
|
||||||
|
LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
|
||||||
|
|
||||||
|
LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr,
|
||||||
|
const char* src, char* dst,
|
||||||
|
int srcSize, int maxDstSize);
|
||||||
|
|
||||||
|
/*! LZ4_compress_HC_continue_destSize() : v1.9.0+
|
||||||
|
* Similar to LZ4_compress_HC_continue(),
|
||||||
|
* but will read as much data as possible from `src`
|
||||||
|
* to fit into `targetDstSize` budget.
|
||||||
|
* Result is provided into 2 parts :
|
||||||
|
* @return : the number of bytes written into 'dst' (necessarily <= targetDstSize)
|
||||||
|
* or 0 if compression fails.
|
||||||
|
* `srcSizePtr` : on success, *srcSizePtr will be updated to indicate how much bytes were read from `src`.
|
||||||
|
* Note that this function may not consume the entire input.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API int LZ4_compress_HC_continue_destSize(LZ4_streamHC_t* LZ4_streamHCPtr,
|
||||||
|
const char* src, char* dst,
|
||||||
|
int* srcSizePtr, int targetDstSize);
|
||||||
|
|
||||||
|
LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*^**********************************************
|
||||||
|
* !!!!!! STATIC LINKING ONLY !!!!!!
|
||||||
|
***********************************************/
|
||||||
|
|
||||||
|
/*-******************************************************************
|
||||||
|
* PRIVATE DEFINITIONS :
|
||||||
|
* Do not use these definitions directly.
|
||||||
|
* They are merely exposed to allow static allocation of `LZ4_streamHC_t`.
|
||||||
|
* Declare an `LZ4_streamHC_t` directly, rather than any type below.
|
||||||
|
* Even then, only do so in the context of static linking, as definitions may change between versions.
|
||||||
|
********************************************************************/
|
||||||
|
|
||||||
|
#define LZ4HC_DICTIONARY_LOGSIZE 16
|
||||||
|
#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
|
||||||
|
#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
|
||||||
|
|
||||||
|
#define LZ4HC_HASH_LOG 15
|
||||||
|
#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)
|
||||||
|
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
|
||||||
|
|
||||||
|
|
||||||
|
#if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
|
||||||
|
struct LZ4HC_CCtx_internal
|
||||||
|
{
|
||||||
|
uint32_t hashTable[LZ4HC_HASHTABLESIZE];
|
||||||
|
uint16_t chainTable[LZ4HC_MAXD];
|
||||||
|
const uint8_t* end; /* next block here to continue on current prefix */
|
||||||
|
const uint8_t* base; /* All index relative to this position */
|
||||||
|
const uint8_t* dictBase; /* alternate base for extDict */
|
||||||
|
uint32_t dictLimit; /* below that point, need extDict */
|
||||||
|
uint32_t lowLimit; /* below that point, no more dict */
|
||||||
|
uint32_t nextToUpdate; /* index from which to continue dictionary update */
|
||||||
|
short compressionLevel;
|
||||||
|
int8_t favorDecSpeed; /* favor decompression speed if this flag set,
|
||||||
|
otherwise, favor compression ratio */
|
||||||
|
int8_t dirty; /* stream has to be fully reset if this flag is set */
|
||||||
|
const LZ4HC_CCtx_internal* dictCtx;
|
||||||
|
};
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
typedef struct LZ4HC_CCtx_internal LZ4HC_CCtx_internal;
|
||||||
|
struct LZ4HC_CCtx_internal
|
||||||
|
{
|
||||||
|
unsigned int hashTable[LZ4HC_HASHTABLESIZE];
|
||||||
|
unsigned short chainTable[LZ4HC_MAXD];
|
||||||
|
const unsigned char* end; /* next block here to continue on current prefix */
|
||||||
|
const unsigned char* base; /* All index relative to this position */
|
||||||
|
const unsigned char* dictBase; /* alternate base for extDict */
|
||||||
|
unsigned int dictLimit; /* below that point, need extDict */
|
||||||
|
unsigned int lowLimit; /* below that point, no more dict */
|
||||||
|
unsigned int nextToUpdate; /* index from which to continue dictionary update */
|
||||||
|
short compressionLevel;
|
||||||
|
char favorDecSpeed; /* favor decompression speed if this flag set,
|
||||||
|
otherwise, favor compression ratio */
|
||||||
|
char dirty; /* stream has to be fully reset if this flag is set */
|
||||||
|
const LZ4HC_CCtx_internal* dictCtx;
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
/* Do not use these definitions directly !
|
||||||
|
* Declare or allocate an LZ4_streamHC_t instead.
|
||||||
|
*/
|
||||||
|
#define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56 + ((sizeof(void*)==16) ? 56 : 0) /* AS400*/ ) /* 262200 or 262256*/
|
||||||
|
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
|
||||||
|
union LZ4_streamHC_u {
|
||||||
|
size_t table[LZ4_STREAMHCSIZE_SIZET];
|
||||||
|
LZ4HC_CCtx_internal internal_donotuse;
|
||||||
|
}; /* previously typedef'd to LZ4_streamHC_t */
|
||||||
|
|
||||||
|
/* LZ4_streamHC_t :
|
||||||
|
* This structure allows static allocation of LZ4 HC streaming state.
|
||||||
|
* This can be used to allocate statically, on state, or as part of a larger structure.
|
||||||
|
*
|
||||||
|
* Such state **must** be initialized using LZ4_initStreamHC() before first use.
|
||||||
|
*
|
||||||
|
* Note that invoking LZ4_initStreamHC() is not required when
|
||||||
|
* the state was created using LZ4_createStreamHC() (which is recommended).
|
||||||
|
* Using the normal builder, a newly created state is automatically initialized.
|
||||||
|
*
|
||||||
|
* Static allocation shall only be used in combination with static linking.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* LZ4_initStreamHC() : v1.9.0+
|
||||||
|
* Required before first use of a statically allocated LZ4_streamHC_t.
|
||||||
|
* Before v1.9.0 : use LZ4_resetStreamHC() instead
|
||||||
|
*/
|
||||||
|
LZ4LIB_API LZ4_streamHC_t* LZ4_initStreamHC (void* buffer, size_t size);
|
||||||
|
|
||||||
|
|
||||||
|
/*-************************************
|
||||||
|
* Deprecated Functions
|
||||||
|
**************************************/
|
||||||
|
/* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */
|
||||||
|
|
||||||
|
/* deprecated compression functions */
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC (const char* source, char* dest, int inputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
|
||||||
|
|
||||||
|
/* Obsolete streaming functions; degraded functionality; do not use!
|
||||||
|
*
|
||||||
|
* In order to perform streaming compression, these functions depended on data
|
||||||
|
* that is no longer tracked in the state. They have been preserved as well as
|
||||||
|
* possible: using them will still produce a correct output. However, use of
|
||||||
|
* LZ4_slideInputBufferHC() will truncate the history of the stream, rather
|
||||||
|
* than preserve a window-sized chunk of history.
|
||||||
|
*/
|
||||||
|
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API void* LZ4_createHC (const char* inputBuffer);
|
||||||
|
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") LZ4LIB_API char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
|
||||||
|
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") LZ4LIB_API int LZ4_freeHC (void* LZ4HC_Data);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") LZ4LIB_API int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
|
||||||
|
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") LZ4LIB_API int LZ4_sizeofStreamStateHC(void);
|
||||||
|
LZ4_DEPRECATED("use LZ4_initStreamHC() instead") LZ4LIB_API int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
|
||||||
|
|
||||||
|
|
||||||
|
/* LZ4_resetStreamHC() is now replaced by LZ4_initStreamHC().
|
||||||
|
* The intention is to emphasize the difference with LZ4_resetStreamHC_fast(),
|
||||||
|
* which is now the recommended function to start a new stream of blocks,
|
||||||
|
* but cannot be used to initialize a memory segment containing arbitrary garbage data.
|
||||||
|
*
|
||||||
|
* It is recommended to switch to LZ4_initStreamHC().
|
||||||
|
* LZ4_resetStreamHC() will generate deprecation warnings in a future version.
|
||||||
|
*/
|
||||||
|
LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
|
||||||
|
|
||||||
|
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#endif /* LZ4_HC_H_19834876238432 */
|
||||||
|
|
||||||
|
|
||||||
|
/*-**************************************************
|
||||||
|
* !!!!! STATIC LINKING ONLY !!!!!
|
||||||
|
* Following definitions are considered experimental.
|
||||||
|
* They should not be linked from DLL,
|
||||||
|
* as there is no guarantee of API stability yet.
|
||||||
|
* Prototypes will be promoted to "stable" status
|
||||||
|
* after successfull usage in real-life scenarios.
|
||||||
|
***************************************************/
|
||||||
|
#ifdef LZ4_HC_STATIC_LINKING_ONLY /* protection macro */
|
||||||
|
#ifndef LZ4_HC_SLO_098092834
|
||||||
|
#define LZ4_HC_SLO_098092834
|
||||||
|
|
||||||
|
#define LZ4_STATIC_LINKING_ONLY /* LZ4LIB_STATIC_API */
|
||||||
|
#include "lz4.h"
|
||||||
|
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/*! LZ4_setCompressionLevel() : v1.8.0+ (experimental)
|
||||||
|
* It's possible to change compression level
|
||||||
|
* between successive invocations of LZ4_compress_HC_continue*()
|
||||||
|
* for dynamic adaptation.
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API void LZ4_setCompressionLevel(
|
||||||
|
LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
|
||||||
|
|
||||||
|
/*! LZ4_favorDecompressionSpeed() : v1.8.2+ (experimental)
|
||||||
|
* Opt. Parser will favor decompression speed over compression ratio.
|
||||||
|
* Only applicable to levels >= LZ4HC_CLEVEL_OPT_MIN.
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API void LZ4_favorDecompressionSpeed(
|
||||||
|
LZ4_streamHC_t* LZ4_streamHCPtr, int favor);
|
||||||
|
|
||||||
|
/*! LZ4_resetStreamHC_fast() : v1.9.0+
|
||||||
|
* When an LZ4_streamHC_t is known to be in a internally coherent state,
|
||||||
|
* it can often be prepared for a new compression with almost no work, only
|
||||||
|
* sometimes falling back to the full, expensive reset that is always required
|
||||||
|
* when the stream is in an indeterminate state (i.e., the reset performed by
|
||||||
|
* LZ4_resetStreamHC()).
|
||||||
|
*
|
||||||
|
* LZ4_streamHCs are guaranteed to be in a valid state when:
|
||||||
|
* - returned from LZ4_createStreamHC()
|
||||||
|
* - reset by LZ4_resetStreamHC()
|
||||||
|
* - memset(stream, 0, sizeof(LZ4_streamHC_t))
|
||||||
|
* - the stream was in a valid state and was reset by LZ4_resetStreamHC_fast()
|
||||||
|
* - the stream was in a valid state and was then used in any compression call
|
||||||
|
* that returned success
|
||||||
|
* - the stream was in an indeterminate state and was used in a compression
|
||||||
|
* call that fully reset the state (LZ4_compress_HC_extStateHC()) and that
|
||||||
|
* returned success
|
||||||
|
*
|
||||||
|
* Note:
|
||||||
|
* A stream that was last used in a compression call that returned an error
|
||||||
|
* may be passed to this function. However, it will be fully reset, which will
|
||||||
|
* clear any existing history and settings from the context.
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API void LZ4_resetStreamHC_fast(
|
||||||
|
LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel);
|
||||||
|
|
||||||
|
/*! LZ4_compress_HC_extStateHC_fastReset() :
|
||||||
|
* A variant of LZ4_compress_HC_extStateHC().
|
||||||
|
*
|
||||||
|
* Using this variant avoids an expensive initialization step. It is only safe
|
||||||
|
* to call if the state buffer is known to be correctly initialized already
|
||||||
|
* (see above comment on LZ4_resetStreamHC_fast() for a definition of
|
||||||
|
* "correctly initialized"). From a high level, the difference is that this
|
||||||
|
* function initializes the provided state with a call to
|
||||||
|
* LZ4_resetStreamHC_fast() while LZ4_compress_HC_extStateHC() starts with a
|
||||||
|
* call to LZ4_resetStreamHC().
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API int LZ4_compress_HC_extStateHC_fastReset (
|
||||||
|
void* state,
|
||||||
|
const char* src, char* dst,
|
||||||
|
int srcSize, int dstCapacity,
|
||||||
|
int compressionLevel);
|
||||||
|
|
||||||
|
/*! LZ4_attach_HC_dictionary() :
|
||||||
|
* This is an experimental API that allows for the efficient use of a
|
||||||
|
* static dictionary many times.
|
||||||
|
*
|
||||||
|
* Rather than re-loading the dictionary buffer into a working context before
|
||||||
|
* each compression, or copying a pre-loaded dictionary's LZ4_streamHC_t into a
|
||||||
|
* working LZ4_streamHC_t, this function introduces a no-copy setup mechanism,
|
||||||
|
* in which the working stream references the dictionary stream in-place.
|
||||||
|
*
|
||||||
|
* Several assumptions are made about the state of the dictionary stream.
|
||||||
|
* Currently, only streams which have been prepared by LZ4_loadDictHC() should
|
||||||
|
* be expected to work.
|
||||||
|
*
|
||||||
|
* Alternatively, the provided dictionary stream pointer may be NULL, in which
|
||||||
|
* case any existing dictionary stream is unset.
|
||||||
|
*
|
||||||
|
* A dictionary should only be attached to a stream without any history (i.e.,
|
||||||
|
* a stream that has just been reset).
|
||||||
|
*
|
||||||
|
* The dictionary will remain attached to the working stream only for the
|
||||||
|
* current stream session. Calls to LZ4_resetStreamHC(_fast) will remove the
|
||||||
|
* dictionary context association from the working stream. The dictionary
|
||||||
|
* stream (and source buffer) must remain in-place / accessible / unchanged
|
||||||
|
* through the lifetime of the stream session.
|
||||||
|
*/
|
||||||
|
LZ4LIB_STATIC_API void LZ4_attach_HC_dictionary(
|
||||||
|
LZ4_streamHC_t *working_stream,
|
||||||
|
const LZ4_streamHC_t *dictionary_stream);
|
||||||
|
|
||||||
|
#if defined (__cplusplus)
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#endif /* LZ4_HC_SLO_098092834 */
|
||||||
|
#endif /* LZ4_HC_STATIC_LINKING_ONLY */
|
|
@ -35,7 +35,7 @@ OBJDIR = obj
|
||||||
INCLUDE = -I../include -I../common_arm -I../common_fpga -I../common -I.
|
INCLUDE = -I../include -I../common_arm -I../common_fpga -I../common -I.
|
||||||
|
|
||||||
# Also search prerequisites in the common directory (for usb.c), the fpga directory (for fpga.bit), and the zlib directory
|
# Also search prerequisites in the common directory (for usb.c), the fpga directory (for fpga.bit), and the zlib directory
|
||||||
VPATH = . ../common_arm ../common ../common/crapto1 ../common/mbedtls ../common/zlib ../fpga ../armsrc/Standalone
|
VPATH = . ../common_arm ../common ../common/crapto1 ../common/mbedtls ../common/lz4 ../fpga ../armsrc/Standalone
|
||||||
|
|
||||||
INCLUDES = ../include/proxmark3_arm.h ../include/at91sam7s512.h ../include/config_gpio.h ../include/pm3_cmd.h
|
INCLUDES = ../include/proxmark3_arm.h ../include/at91sam7s512.h ../include/config_gpio.h ../include/pm3_cmd.h
|
||||||
|
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
#define FPGA_BITSTREAM_FIXED_HEADER_SIZE sizeof(bitparse_fixed_header)
|
#define FPGA_BITSTREAM_FIXED_HEADER_SIZE sizeof(bitparse_fixed_header)
|
||||||
#define FPGA_INTERLEAVE_SIZE 288
|
#define FPGA_INTERLEAVE_SIZE 288
|
||||||
#define FPGA_CONFIG_SIZE 42336L // our current fpga_[lh]f.bit files are 42175 bytes. Rounded up to next multiple of FPGA_INTERLEAVE_SIZE
|
#define FPGA_CONFIG_SIZE 42336L // our current fpga_[lh]f.bit files are 42175 bytes. Rounded up to next multiple of FPGA_INTERLEAVE_SIZE
|
||||||
|
#define FPGA_RING_BUFFER_BYTES (1024 * 39)
|
||||||
#define FPGA_TRACE_SIZE 3072
|
#define FPGA_TRACE_SIZE 3072
|
||||||
|
|
||||||
static const uint8_t bitparse_fixed_header[] = {0x00, 0x09, 0x0f, 0xf0, 0x0f, 0xf0, 0x0f, 0xf0, 0x0f, 0xf0, 0x00, 0x00, 0x01};
|
static const uint8_t bitparse_fixed_header[] = {0x00, 0x09, 0x0f, 0xf0, 0x0f, 0xf0, 0x0f, 0xf0, 0x0f, 0xf0, 0x00, 0x00, 0x01};
|
||||||
|
|
|
@ -1,13 +1,14 @@
|
||||||
MYSRCPATHS =
|
MYSRCPATHS =
|
||||||
MYINCLUDES = -I../../common_fpga
|
MYINCLUDES = -I../../common_fpga
|
||||||
MYCFLAGS = -std=c99 -D_ISOC99_SOURCE
|
MYCFLAGS = -std=c99 -D_ISOC99_SOURCE
|
||||||
MYDEFS =
|
MYDEFS =
|
||||||
MYSRCS =
|
MYSRCS =
|
||||||
MYLIBS =
|
MYLIBS =
|
||||||
|
|
||||||
ZLIBPATH = ../../common/zlib
|
MYSRCPATHS += ../../common/lz4
|
||||||
MYINCLUDES += -I$(ZLIBPATH)
|
MYINCLUDES += -I../../common/lz4
|
||||||
MYLIBS += $(OBJDIR)/libz.a
|
MYSRCS += lz4hc.c lz4.c
|
||||||
|
MYCFLAGS += -DLZ4_MEMORY_USAGE=20 -Wno-redundant-decls -Wno-old-style-definition -Wno-missing-prototypes -Wno-missing-declarations
|
||||||
|
|
||||||
BINS = fpga_compress
|
BINS = fpga_compress
|
||||||
|
|
||||||
|
@ -16,9 +17,6 @@ ROOT_DIR:=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||||
include ../../Makefile.host
|
include ../../Makefile.host
|
||||||
|
|
||||||
fpga_compress: $(OBJDIR)/fpga_compress.o $(MYOBJS)
|
fpga_compress: $(OBJDIR)/fpga_compress.o $(MYOBJS)
|
||||||
$(OBJDIR)/libz.a:
|
|
||||||
$(info [*] MAKE zlib)
|
|
||||||
$(Q)$(MAKE) --no-print-directory -C $(ZLIBPATH) OBJDIR=$(ROOT_DIR)$(OBJDIR) BINDIR=$(ROOT_DIR)$(OBJDIR) all
|
|
||||||
|
|
||||||
tarbin: $(BINS)
|
tarbin: $(BINS)
|
||||||
$(info [=] TAR ../proxmark3-$(platform)-bin.tar)
|
$(info [=] TAR ../proxmark3-$(platform)-bin.tar)
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
// Compression tool for FPGA config files. Compress several *.bit files at
|
// Compression tool for FPGA config files. Compress several *.bit files at
|
||||||
// compile time. Decompression is done at run time (see fpgaloader.c).
|
// compile time. Decompression is done at run time (see fpgaloader.c).
|
||||||
// This uses the zlib library tuned to this specific case. The small file sizes
|
// This uses the lz4 library tuned to this specific case. The small file sizes
|
||||||
// allow to use "insane" parameters for optimum compression ratio.
|
// allow to use "insane" parameters for optimum compression ratio.
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@ -15,28 +15,11 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <libgen.h>
|
#include <libgen.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <inttypes.h>
|
||||||
#include "fpga.h"
|
#include "fpga.h"
|
||||||
#include "zlib.h"
|
#include "lz4hc.h"
|
||||||
|
|
||||||
// zlib configuration
|
|
||||||
#define COMPRESS_LEVEL 9 // use best possible compression
|
|
||||||
#define COMPRESS_WINDOW_BITS 15 // default = max = 15 for a window of 2^15 = 32KBytes
|
|
||||||
#define COMPRESS_MEM_LEVEL 9 // determines the amount of memory allocated during compression. Default = 8.
|
|
||||||
|
|
||||||
/* COMPRESS_STRATEGY can be
|
|
||||||
Z_DEFAULT_STRATEGY (the default),
|
|
||||||
Z_FILTERED (more huffmann, less string matching),
|
|
||||||
Z_HUFFMAN_ONLY (huffman only, no string matching)
|
|
||||||
Z_RLE (distances limited to one)
|
|
||||||
Z_FIXED (prevents the use of dynamic Huffman codes)
|
|
||||||
*/
|
|
||||||
|
|
||||||
#define COMPRESS_STRATEGY Z_DEFAULT_STRATEGY
|
|
||||||
// zlib tuning parameters:
|
|
||||||
#define COMPRESS_GOOD_LENGTH 258
|
|
||||||
#define COMPRESS_MAX_LAZY 258
|
|
||||||
#define COMPRESS_MAX_NICE_LENGTH 258
|
|
||||||
#define COMPRESS_MAX_CHAIN 8192
|
|
||||||
|
|
||||||
#define HARDNESTED_TABLE_SIZE (uint32_t)(sizeof(uint32_t) * ((1L<<19)+1))
|
#define HARDNESTED_TABLE_SIZE (uint32_t)(sizeof(uint32_t) * ((1L<<19)+1))
|
||||||
|
|
||||||
|
@ -52,18 +35,6 @@ static void usage(void) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static voidpf fpga_deflate_malloc(voidpf opaque, uInt items, uInt size) {
|
|
||||||
(void) opaque;
|
|
||||||
return calloc(items * size, sizeof(uint8_t));
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static void fpga_deflate_free(voidpf opaque, voidpf address) {
|
|
||||||
(void) opaque;
|
|
||||||
free(address);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static bool all_feof(FILE *infile[], uint8_t num_infiles) {
|
static bool all_feof(FILE *infile[], uint8_t num_infiles) {
|
||||||
for (uint16_t i = 0; i < num_infiles; i++) {
|
for (uint16_t i = 0; i < num_infiles; i++) {
|
||||||
if (!feof(infile[i])) {
|
if (!feof(infile[i])) {
|
||||||
|
@ -76,10 +47,8 @@ static bool all_feof(FILE *infile[], uint8_t num_infiles) {
|
||||||
|
|
||||||
static int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, bool hardnested_mode) {
|
static int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, bool hardnested_mode) {
|
||||||
uint8_t *fpga_config;
|
uint8_t *fpga_config;
|
||||||
uint32_t i;
|
//int32_t ret;
|
||||||
int32_t ret;
|
|
||||||
uint8_t c;
|
uint8_t c;
|
||||||
z_stream compressed_fpga_stream;
|
|
||||||
|
|
||||||
if (hardnested_mode) {
|
if (hardnested_mode) {
|
||||||
fpga_config = calloc(num_infiles * HARDNESTED_TABLE_SIZE, sizeof(uint8_t));
|
fpga_config = calloc(num_infiles * HARDNESTED_TABLE_SIZE, sizeof(uint8_t));
|
||||||
|
@ -87,10 +56,10 @@ static int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, boo
|
||||||
fpga_config = calloc(num_infiles * FPGA_CONFIG_SIZE, sizeof(uint8_t));
|
fpga_config = calloc(num_infiles * FPGA_CONFIG_SIZE, sizeof(uint8_t));
|
||||||
}
|
}
|
||||||
// read the input files. Interleave them into fpga_config[]
|
// read the input files. Interleave them into fpga_config[]
|
||||||
i = 0;
|
uint32_t total_size = 0;
|
||||||
do {
|
do {
|
||||||
|
|
||||||
if (i >= num_infiles * (hardnested_mode ? HARDNESTED_TABLE_SIZE : FPGA_CONFIG_SIZE)) {
|
if (total_size >= num_infiles * (hardnested_mode ? HARDNESTED_TABLE_SIZE : FPGA_CONFIG_SIZE)) {
|
||||||
if (hardnested_mode) {
|
if (hardnested_mode) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
"Input file too big (> %" PRIu32 " bytes). This is probably not a hardnested bitflip state table.\n"
|
"Input file too big (> %" PRIu32 " bytes). This is probably not a hardnested bitflip state table.\n"
|
||||||
|
@ -112,53 +81,47 @@ static int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, boo
|
||||||
for (uint16_t k = 0; k < FPGA_INTERLEAVE_SIZE; k++) {
|
for (uint16_t k = 0; k < FPGA_INTERLEAVE_SIZE; k++) {
|
||||||
c = (uint8_t)fgetc(infile[j]);
|
c = (uint8_t)fgetc(infile[j]);
|
||||||
if (!feof(infile[j])) {
|
if (!feof(infile[j])) {
|
||||||
fpga_config[i++] = c;
|
fpga_config[total_size++] = c;
|
||||||
} else if (num_infiles > 1) {
|
} else if (num_infiles > 1) {
|
||||||
fpga_config[i++] = '\0';
|
fpga_config[total_size++] = '\0';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} while (!all_feof(infile, num_infiles));
|
} while (!all_feof(infile, num_infiles));
|
||||||
|
|
||||||
// initialize zlib structures
|
uint32_t buffer_size = FPGA_RING_BUFFER_BYTES;
|
||||||
compressed_fpga_stream.next_in = fpga_config;
|
if (num_infiles == 1)
|
||||||
compressed_fpga_stream.avail_in = i;
|
buffer_size = 1024*1024; //1M for now
|
||||||
compressed_fpga_stream.zalloc = fpga_deflate_malloc;
|
uint32_t outsize_max = LZ4_compressBound(buffer_size);
|
||||||
compressed_fpga_stream.zfree = fpga_deflate_free;
|
|
||||||
compressed_fpga_stream.opaque = Z_NULL;
|
|
||||||
|
|
||||||
ret = deflateInit2(&compressed_fpga_stream,
|
char *outbuf = calloc(outsize_max, sizeof(char));
|
||||||
COMPRESS_LEVEL,
|
|
||||||
Z_DEFLATED,
|
|
||||||
COMPRESS_WINDOW_BITS,
|
|
||||||
COMPRESS_MEM_LEVEL,
|
|
||||||
COMPRESS_STRATEGY);
|
|
||||||
|
|
||||||
// estimate the size of the compressed output
|
LZ4_streamHC_t* lz4_streamhc = LZ4_createStreamHC();
|
||||||
uint32_t outsize_max = deflateBound(&compressed_fpga_stream, compressed_fpga_stream.avail_in);
|
LZ4_resetStreamHC_fast(lz4_streamhc, LZ4HC_CLEVEL_MAX);
|
||||||
uint8_t *outbuf = calloc(outsize_max, sizeof(uint8_t));
|
|
||||||
compressed_fpga_stream.next_out = outbuf;
|
|
||||||
compressed_fpga_stream.avail_out = outsize_max;
|
|
||||||
|
|
||||||
if (ret == Z_OK) {
|
int current_in = 0;
|
||||||
ret = deflateTune(&compressed_fpga_stream,
|
int current_out = 0;
|
||||||
COMPRESS_GOOD_LENGTH,
|
char * ring_buffer = calloc(buffer_size, sizeof(char));
|
||||||
COMPRESS_MAX_LAZY,
|
while (current_in < total_size) {
|
||||||
COMPRESS_MAX_NICE_LENGTH,
|
int bytes_to_copy = FPGA_RING_BUFFER_BYTES;
|
||||||
COMPRESS_MAX_CHAIN);
|
if (total_size - current_in < FPGA_RING_BUFFER_BYTES)
|
||||||
|
bytes_to_copy = total_size - current_in;
|
||||||
|
memcpy(ring_buffer, fpga_config + current_in, bytes_to_copy);
|
||||||
|
int cmp_bytes = LZ4_compress_HC_continue(lz4_streamhc, ring_buffer, outbuf, bytes_to_copy, outsize_max);
|
||||||
|
fwrite(&cmp_bytes, sizeof(int), 1, outfile);
|
||||||
|
fwrite(outbuf, sizeof(char), cmp_bytes, outfile);
|
||||||
|
current_in += bytes_to_copy;
|
||||||
|
current_out += cmp_bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ret == Z_OK) {
|
free(ring_buffer);
|
||||||
ret = deflate(&compressed_fpga_stream, Z_FINISH);
|
|
||||||
}
|
|
||||||
|
|
||||||
fprintf(stdout, "compressed %u input bytes to %lu output bytes\n", i, compressed_fpga_stream.total_out);
|
fprintf(stdout, "compressed %u input bytes to %u output bytes\n", total_size, current_out);
|
||||||
|
|
||||||
if (ret != Z_STREAM_END) {
|
if (current_out == 0) {
|
||||||
fprintf(stderr, "Error in deflate(): %d %s\n", ret, compressed_fpga_stream.msg);
|
fprintf(stderr, "Error in lz4");
|
||||||
free(outbuf);
|
free(outbuf);
|
||||||
deflateEnd(&compressed_fpga_stream);
|
|
||||||
for (uint16_t j = 0; j < num_infiles; j++) {
|
for (uint16_t j = 0; j < num_infiles; j++) {
|
||||||
fclose(infile[j]);
|
fclose(infile[j]);
|
||||||
}
|
}
|
||||||
|
@ -167,90 +130,65 @@ static int zlib_compress(FILE *infile[], uint8_t num_infiles, FILE *outfile, boo
|
||||||
return (EXIT_FAILURE);
|
return (EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (i = 0; i < compressed_fpga_stream.total_out; i++) {
|
|
||||||
fputc(outbuf[i], outfile);
|
|
||||||
}
|
|
||||||
|
|
||||||
free(outbuf);
|
free(outbuf);
|
||||||
deflateEnd(&compressed_fpga_stream);
|
|
||||||
for (uint16_t j = 0; j < num_infiles; j++) {
|
for (uint16_t j = 0; j < num_infiles; j++) {
|
||||||
fclose(infile[j]);
|
fclose(infile[j]);
|
||||||
}
|
}
|
||||||
fclose(outfile);
|
fclose(outfile);
|
||||||
|
LZ4_freeStreamHC(lz4_streamhc);
|
||||||
free(fpga_config);
|
free(fpga_config);
|
||||||
|
|
||||||
return (EXIT_SUCCESS);
|
return (EXIT_SUCCESS);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
typedef struct lz4_stream_s {
|
||||||
|
LZ4_streamDecode_t* lz4StreamDecode;
|
||||||
|
char * next_in;
|
||||||
|
int avail_in;
|
||||||
|
} lz4_stream;
|
||||||
|
|
||||||
static int zlib_decompress(FILE *infile, FILE *outfile) {
|
static int zlib_decompress(FILE *infile, FILE *outfile) {
|
||||||
#define DECOMPRESS_BUF_SIZE 1024
|
|
||||||
uint8_t outbuf[DECOMPRESS_BUF_SIZE];
|
|
||||||
uint8_t inbuf[DECOMPRESS_BUF_SIZE];
|
|
||||||
int32_t ret;
|
|
||||||
|
|
||||||
z_stream compressed_fpga_stream;
|
LZ4_streamDecode_t lz4StreamDecode_body = {{ 0 }};
|
||||||
|
char outbuf[FPGA_RING_BUFFER_BYTES];
|
||||||
|
|
||||||
// initialize zlib structures
|
fseek(infile, 0L, SEEK_END);
|
||||||
compressed_fpga_stream.next_in = inbuf;
|
long infile_size = ftell(infile);
|
||||||
compressed_fpga_stream.avail_in = 0;
|
fseek(infile, 0L, SEEK_SET);
|
||||||
compressed_fpga_stream.next_out = outbuf;
|
|
||||||
compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
|
|
||||||
compressed_fpga_stream.zalloc = fpga_deflate_malloc;
|
|
||||||
compressed_fpga_stream.zfree = fpga_deflate_free;
|
|
||||||
compressed_fpga_stream.opaque = Z_NULL;
|
|
||||||
|
|
||||||
ret = inflateInit2(&compressed_fpga_stream, 0);
|
char * inbuf = calloc(infile_size, sizeof(char));
|
||||||
if (ret < 0)
|
size_t num_read = fread(inbuf, sizeof(char), infile_size, infile);
|
||||||
return (EXIT_FAILURE);
|
|
||||||
|
|
||||||
do {
|
if (num_read != infile_size) {
|
||||||
if (compressed_fpga_stream.avail_in == 0) {
|
|
||||||
compressed_fpga_stream.next_in = inbuf;
|
|
||||||
uint16_t i = 0;
|
|
||||||
do {
|
|
||||||
int32_t c = fgetc(infile);
|
|
||||||
if (!feof(infile)) {
|
|
||||||
inbuf[i++] = c & 0xFF;
|
|
||||||
compressed_fpga_stream.avail_in++;
|
|
||||||
} else {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} while (i < DECOMPRESS_BUF_SIZE);
|
|
||||||
}
|
|
||||||
|
|
||||||
ret = inflate(&compressed_fpga_stream, Z_SYNC_FLUSH);
|
|
||||||
|
|
||||||
if (ret != Z_OK && ret != Z_STREAM_END) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (compressed_fpga_stream.avail_out == 0) {
|
|
||||||
for (uint16_t i = 0; i < DECOMPRESS_BUF_SIZE; i++) {
|
|
||||||
fputc(outbuf[i], outfile);
|
|
||||||
}
|
|
||||||
compressed_fpga_stream.avail_out = DECOMPRESS_BUF_SIZE;
|
|
||||||
compressed_fpga_stream.next_out = outbuf;
|
|
||||||
}
|
|
||||||
} while (ret == Z_OK);
|
|
||||||
|
|
||||||
if (ret == Z_STREAM_END) { // reached end of input
|
|
||||||
uint16_t i = 0;
|
|
||||||
while (compressed_fpga_stream.avail_out < DECOMPRESS_BUF_SIZE) {
|
|
||||||
fputc(outbuf[i++], outfile);
|
|
||||||
compressed_fpga_stream.avail_out++;
|
|
||||||
}
|
|
||||||
fclose(outfile);
|
|
||||||
fclose(infile);
|
|
||||||
return (EXIT_SUCCESS);
|
|
||||||
} else {
|
|
||||||
fprintf(stderr, "Error. Inflate() returned error %d, %s", ret, compressed_fpga_stream.msg);
|
|
||||||
fclose(outfile);
|
|
||||||
fclose(infile);
|
|
||||||
return (EXIT_FAILURE);
|
return (EXIT_FAILURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lz4_stream compressed_fpga_stream;
|
||||||
|
// initialize lz4 structures
|
||||||
|
compressed_fpga_stream.lz4StreamDecode = &lz4StreamDecode_body;
|
||||||
|
compressed_fpga_stream.next_in = inbuf;
|
||||||
|
compressed_fpga_stream.avail_in = infile_size;
|
||||||
|
|
||||||
|
int total_size = 0;
|
||||||
|
while (compressed_fpga_stream.avail_in > 0) {
|
||||||
|
int cmp_bytes;
|
||||||
|
memcpy(&cmp_bytes, compressed_fpga_stream.next_in, sizeof(int));
|
||||||
|
compressed_fpga_stream.next_in += 4;
|
||||||
|
compressed_fpga_stream.avail_in -= cmp_bytes + 4;
|
||||||
|
const int decBytes = LZ4_decompress_safe_continue(compressed_fpga_stream.lz4StreamDecode, compressed_fpga_stream.next_in, outbuf, cmp_bytes, FPGA_RING_BUFFER_BYTES);
|
||||||
|
if (decBytes <= 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
fwrite(outbuf, decBytes, sizeof(char), outfile);
|
||||||
|
total_size += decBytes;
|
||||||
|
compressed_fpga_stream.next_in += cmp_bytes;
|
||||||
|
}
|
||||||
|
printf("uncompressed %li input bytes to %i output bytes\n", infile_size, total_size);
|
||||||
|
fclose(outfile);
|
||||||
|
fclose(infile);
|
||||||
|
return (EXIT_SUCCESS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -342,7 +280,7 @@ static int FpgaGatherVersion(FILE *infile, char *infile_name, char *dst, int len
|
||||||
strncat(dst, " on ", len - strlen(dst) - 1);
|
strncat(dst, " on ", len - strlen(dst) - 1);
|
||||||
for (uint16_t i = 0; i < fpga_info_len; i++) {
|
for (uint16_t i = 0; i < fpga_info_len; i++) {
|
||||||
char c = (char)fgetc(infile);
|
char c = (char)fgetc(infile);
|
||||||
if (i < sizeof(tempstr)) {
|
if (i < sizeof(tempstr)) {
|
||||||
if (c == '/') c = '-';
|
if (c == '/') c = '-';
|
||||||
if (c == ' ') c = '0';
|
if (c == ' ') c = '0';
|
||||||
tempstr[i] = c;
|
tempstr[i] = c;
|
||||||
|
|
Loading…
Reference in a new issue