mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-08-08 02:40:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Clean up commit for zstd

Browse source 
* deleted unused files: examples, tests and windows stuff
* moved the main header files up a level into third_party/zstd
* removed clang-format line on non-C/C++ files
This commit is contained in:
Farid Zakaria 2023-06-26 21:51:14 +00:00
parent 32028f01a2
commit 3d59e9ba09
291 changed files with 30 additions and 44293 deletions
Download patch file Download diff file Expand all files Collapse all files

1
third_party/zstd/CHANGELOG vendored
View file

@ -1,4 +1,3 @@
// clang-format off
v1.5.5 (Apr 2023)
fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln)
perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln)

6
third_party/zstd/CODE_OF_CONDUCT.md vendored
View file

@ -1,6 +0,0 @@
// clang-format off
# Code of Conduct
Facebook has adopted a Code of Conduct that we expect project participants to adhere to.
Please read the [full text](https://code.fb.com/codeofconduct/)
so that you can understand what actions will and will not be tolerated.

1
third_party/zstd/COPYING vendored
View file

@ -1,4 +1,3 @@
// clang-format off
GNU GENERAL PUBLIC LICENSE
Version 2, June 1991

1
third_party/zstd/LICENSE vendored
View file

@ -1,4 +1,3 @@
// clang-format off
BSD License
For Zstandard software

2
third_party/zstd/README.cosmo vendored
View file

@ -9,4 +9,4 @@ ORIGIN
LOCAL CHANGES
None.
Changed layout of files slightly to better suit Cosmopolitan.

1
third_party/zstd/README.md vendored
View file

@ -1,4 +1,3 @@
// clang-format off
<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p>
__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm,

16
third_party/zstd/examples/.gitignore vendored
View file

@ -1,16 +0,0 @@
// clang-format off
#build
simple_compression
simple_decompression
multiple_simple_compression
dictionary_compression
dictionary_decompression
streaming_compression
streaming_decompression
multiple_streaming_compression
streaming_memory_usage
#test artefact
tmp*
test*
*.zst

94
third_party/zstd/examples/Makefile vendored
View file

@ -1,94 +0,0 @@
// clang-format off
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
LIBDIR =../lib
CPPFLAGS += -I$(LIBDIR)
LIB = $(LIBDIR)/libzstd.a
.PHONY: default
default: all
.PHONY: all
all: simple_compression simple_decompression \
multiple_simple_compression\
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression \
multiple_streaming_compression streaming_memory_usage
$(LIB) :
$(MAKE) -C $(LIBDIR) libzstd.a
simple_compression.o: common.h
simple_compression : $(LIB)
simple_decompression.o: common.h
simple_decompression : $(LIB)
multiple_simple_compression.o: common.h
multiple_simple_compression : $(LIB)
dictionary_compression.o: common.h
dictionary_compression : $(LIB)
dictionary_decompression.o: common.h
dictionary_decompression : $(LIB)
streaming_compression.o: common.h
streaming_compression : $(LIB)
multiple_streaming_compression.o: common.h
multiple_streaming_compression : $(LIB)
streaming_decompression.o: common.h
streaming_decompression : $(LIB)
streaming_memory_usage.o: common.h
streaming_memory_usage : $(LIB)
.PHONY:clean
clean:
@$(RM) core *.o tmp* result* *.zst \
simple_compression simple_decompression \
multiple_simple_compression \
dictionary_compression dictionary_decompression \
streaming_compression streaming_decompression \
multiple_streaming_compression streaming_memory_usage
@echo Cleaning completed
.PHONY:test
test: all
cp README.md tmp
cp Makefile tmp2
@echo -- Simple compression tests
./simple_compression tmp
./simple_decompression tmp.zst
./multiple_simple_compression *.c
./streaming_decompression tmp.zst > /dev/null
@echo -- Streaming memory usage
./streaming_memory_usage
@echo -- Streaming compression tests
./streaming_compression tmp
./streaming_decompression tmp.zst > /dev/null
@echo -- Edge cases detection
! ./streaming_decompression tmp # invalid input, must fail
! ./simple_decompression tmp # invalid input, must fail
touch tmpNull # create 0-size file
./simple_compression tmpNull
./simple_decompression tmpNull.zst # 0-size frame : must work
@echo -- Multiple streaming tests
./multiple_streaming_compression *.c
@echo -- Dictionary compression tests
./dictionary_compression tmp2 tmp README.md
./dictionary_decompression tmp2.zst tmp.zst README.md
$(RM) tmp* *.zst
@echo tests completed

47
third_party/zstd/examples/README.md vendored
View file

@ -1,47 +0,0 @@
// clang-format off
Zstandard library : usage examples
==================================
- [Simple compression](simple_compression.c) :
Compress a single file.
Introduces usage of : `ZSTD_compress()`
- [Simple decompression](simple_decompression.c) :
Decompress a single file.
Only compatible with simple compression.
Result remains in memory.
Introduces usage of : `ZSTD_decompress()`
- [Multiple simple compression](multiple_simple_compression.c) :
Compress multiple files (in simple mode) in a single command line.
Demonstrates memory preservation technique that
minimizes malloc()/free() calls by re-using existing resources.
Introduces usage of : `ZSTD_compressCCtx()`
- [Streaming memory usage](streaming_memory_usage.c) :
Provides amount of memory used by streaming context.
Introduces usage of : `ZSTD_sizeof_CStream()`
- [Streaming compression](streaming_compression.c) :
Compress a single file.
Introduces usage of : `ZSTD_compressStream()`
- [Multiple Streaming compression](multiple_streaming_compression.c) :
Compress multiple files (in streaming mode) in a single command line.
Introduces memory usage preservation technique,
reducing impact of malloc()/free() and memset() by re-using existing resources.
- [Streaming decompression](streaming_decompression.c) :
Decompress a single file compressed by zstd.
Compatible with both simple and streaming compression.
Result is sent to stdout.
Introduces usage of : `ZSTD_decompressStream()`
- [Dictionary compression](dictionary_compression.c) :
Compress multiple files using the same dictionary.
Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()`
- [Dictionary decompression](dictionary_decompression.c) :
Decompress multiple files using the same dictionary.
Result remains in memory.
Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`

277
third_party/zstd/examples/common.h vendored
View file

@ -1,277 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/*
* This header file has common utility functions used in examples.
*/
#ifndef COMMON_H
#define COMMON_H
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // malloc, free, exit
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // fprintf, perror, fopen, etc.
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // strerror
#include "libc/errno.h" // errno
#include "libc/calls/calls.h"
#include "libc/calls/struct/stat.h"
#include "libc/calls/struct/stat.macros.h"
#include "libc/calls/struct/timespec.h"
#include "libc/calls/weirdtypes.h"
#include "libc/sysv/consts/s.h"
#include "libc/sysv/consts/utime.h"
#include "libc/time/time.h" // stat
// MISSING #include <zstd.h>
/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */
#if defined(__GNUC__)
# define UNUSED_ATTR __attribute__((unused))
#else
# define UNUSED_ATTR
#endif
#define HEADER_FUNCTION static UNUSED_ATTR
/*
* Define the returned error code from utility functions.
*/
typedef enum {
ERROR_fsize = 1,
ERROR_fopen = 2,
ERROR_fclose = 3,
ERROR_fread = 4,
ERROR_fwrite = 5,
ERROR_loadFile = 6,
ERROR_saveFile = 7,
ERROR_malloc = 8,
ERROR_largeFile = 9,
} COMMON_ErrorCode;
/*! CHECK
* Check that the condition holds. If it doesn't print a message and die.
*/
#define CHECK(cond, ...) \
do { \
if (!(cond)) { \
fprintf(stderr, \
"%s:%d CHECK(%s) failed: ", \
__FILE__, \
__LINE__, \
#cond); \
fprintf(stderr, "" __VA_ARGS__); \
fprintf(stderr, "\n"); \
exit(1); \
} \
} while (0)
/*! CHECK_ZSTD
* Check the zstd error code and die if an error occurred after printing a
* message.
*/
#define CHECK_ZSTD(fn) \
do { \
size_t const err = (fn); \
CHECK(!ZSTD_isError(err), "%s", ZSTD_getErrorName(err)); \
} while (0)
/*! fsize_orDie() :
* Get the size of a given file path.
*
* @return The size of a given file path.
*/
HEADER_FUNCTION size_t fsize_orDie(const char *filename)
{
struct stat st;
if (stat(filename, &st) != 0) {
/* error */
perror(filename);
exit(ERROR_fsize);
}
off_t const fileSize = st.st_size;
size_t const size = (size_t)fileSize;
/* 1. fileSize should be non-negative,
* 2. if off_t -> size_t type conversion results in discrepancy,
* the file size is too large for type size_t.
*/
if ((fileSize < 0) || (fileSize != (off_t)size)) {
fprintf(stderr, "%s : filesize too large \n", filename);
exit(ERROR_largeFile);
}
return size;
}
/*! fopen_orDie() :
* Open a file using given file path and open option.
*
* @return If successful this function will return a FILE pointer to an
* opened file otherwise it sends an error to stderr and exits.
*/
HEADER_FUNCTION FILE* fopen_orDie(const char *filename, const char *instruction)
{
FILE* const inFile = fopen(filename, instruction);
if (inFile) return inFile;
/* error */
perror(filename);
exit(ERROR_fopen);
}
/*! fclose_orDie() :
* Close an opened file using given FILE pointer.
*/
HEADER_FUNCTION void fclose_orDie(FILE* file)
{
if (!fclose(file)) { return; };
/* error */
perror("fclose");
exit(ERROR_fclose);
}
/*! fread_orDie() :
*
* Read sizeToRead bytes from a given file, storing them at the
* location given by buffer.
*
* @return The number of bytes read.
*/
HEADER_FUNCTION size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
{
size_t const readSize = fread(buffer, 1, sizeToRead, file);
if (readSize == sizeToRead) return readSize; /* good */
if (feof(file)) return readSize; /* good, reached end of file */
/* error */
perror("fread");
exit(ERROR_fread);
}
/*! fwrite_orDie() :
*
* Write sizeToWrite bytes to a file pointed to by file, obtaining
* them from a location given by buffer.
*
* Note: This function will send an error to stderr and exit if it
* cannot write data to the given file pointer.
*
* @return The number of bytes written.
*/
HEADER_FUNCTION size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
{
size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
if (writtenSize == sizeToWrite) return sizeToWrite; /* good */
/* error */
perror("fwrite");
exit(ERROR_fwrite);
}
/*! malloc_orDie() :
* Allocate memory.
*
* @return If successful this function returns a pointer to allo-
* cated memory. If there is an error, this function will send that
* error to stderr and exit.
*/
HEADER_FUNCTION void* malloc_orDie(size_t size)
{
void* const buff = malloc(size);
if (buff) return buff;
/* error */
perror("malloc");
exit(ERROR_malloc);
}
/*! loadFile_orDie() :
* load file into buffer (memory).
*
* Note: This function will send an error to stderr and exit if it
* cannot read data from the given file path.
*
* @return If successful this function will load file into buffer and
* return file size, otherwise it will printout an error to stderr and exit.
*/
HEADER_FUNCTION size_t loadFile_orDie(const char* fileName, void* buffer, size_t bufferSize)
{
size_t const fileSize = fsize_orDie(fileName);
CHECK(fileSize <= bufferSize, "File too large!");
FILE* const inFile = fopen_orDie(fileName, "rb");
size_t const readSize = fread(buffer, 1, fileSize, inFile);
if (readSize != (size_t)fileSize) {
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(ERROR_fread);
}
fclose(inFile); /* can't fail, read only */
return fileSize;
}
/*! mallocAndLoadFile_orDie() :
* allocate memory buffer and then load file into it.
*
* Note: This function will send an error to stderr and exit if memory allocation
* fails or it cannot read data from the given file path.
*
* @return If successful this function will return buffer and bufferSize(=fileSize),
* otherwise it will printout an error to stderr and exit.
*/
HEADER_FUNCTION void* mallocAndLoadFile_orDie(const char* fileName, size_t* bufferSize)
{
size_t const fileSize = fsize_orDie(fileName);
*bufferSize = fileSize;
void* const buffer = malloc_orDie(*bufferSize);
loadFile_orDie(fileName, buffer, *bufferSize);
return buffer;
}
/*! saveFile_orDie() :
*
* Save buffSize bytes to a given file path, obtaining them from a location pointed
* to by buff.
*
* Note: This function will send an error to stderr and exit if it
* cannot write to a given file.
*/
HEADER_FUNCTION void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
{
FILE* const oFile = fopen_orDie(fileName, "wb");
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
if (wSize != (size_t)buffSize) {
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
exit(ERROR_fwrite);
}
if (fclose(oFile)) {
perror(fileName);
exit(ERROR_fclose);
}
}
#endif

131
third_party/zstd/examples/dictionary_compression.c vendored
View file

@ -1,131 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
**/
/* This example deals with Dictionary compression,
* its counterpart is `examples/dictionary_decompression.c` .
* These examples presume that a dictionary already exists.
* The main method to create a dictionary is `zstd --train`,
* look at the CLI documentation for details.
* Another possible method is to employ dictionary training API,
* published in `lib/zdict.h` .
**/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // memset, strcat
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/* createDict() :
** `dictFileName` is supposed already created using `zstd --train` */
static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
CHECK(cdict != NULL, "ZSTD_createCDict() failed!");
free(dictBuffer);
return cdict;
}
static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
{
size_t fSize;
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);
/* Compress using the dictionary.
* This function writes the dictionary id, and content size into the header.
* But, it doesn't use a checksum. You can control these options using the
* advanced API: ZSTD_CCtx_setParameter(), ZSTD_CCtx_refCDict(),
* and ZSTD_compress2().
*/
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
CHECK_ZSTD(cSize);
saveFile_orDie(oname, cBuff, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
ZSTD_freeCCtx(cctx); /* never fails */
free(fBuff);
free(cBuff);
}
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
int const cLevel = 3;
if (argc<3) {
fprintf(stderr, "wrong arguments\n");
fprintf(stderr, "usage:\n");
fprintf(stderr, "%s [FILES] dictionary\n", exeName);
return 1;
}
/* load dictionary only once */
const char* const dictName = argv[argc-1];
ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
int u;
for (u=1; u<argc-1; u++) {
const char* inFilename = argv[u];
char* const outFilename = createOutFilename_orDie(inFilename);
compress(inFilename, outFilename, dictPtr);
free(outFilename);
}
ZSTD_freeCDict(dictPtr);
printf("All %u files compressed. \n", argc-2);
return 0;
}

121
third_party/zstd/examples/dictionary_decompression.c vendored
View file

@ -1,121 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/* createDict() :
`dictFileName` is supposed to have been created using `zstd --train` */
static ZSTD_DDict* createDict_orDie(const char* dictFileName)
{
size_t dictSize;
printf("loading dictionary %s \n", dictFileName);
void* const dictBuffer = mallocAndLoadFile_orDie(dictFileName, &dictSize);
ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
CHECK(ddict != NULL, "ZSTD_createDDict() failed!");
free(dictBuffer);
return ddict;
}
static void decompress(const char* fname, const ZSTD_DDict* ddict)
{
size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
/* Read the content size from the frame header. For simplicity we require
* that it is always present. By default, zstd will write the content size
* in the header when it is known. If you can't guarantee that the frame
* content size is always written into the header, either use streaming
* decompression, or ZSTD_decompressBound().
*/
unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
void* const rBuff = malloc_orDie((size_t)rSize);
/* Check that the dictionary ID matches.
* If a non-zstd dictionary is used, then both will be zero.
* By default zstd always writes the dictionary ID into the frame.
* Zstd will check if there is a dictionary ID mismatch as well.
*/
unsigned const expectedDictID = ZSTD_getDictID_fromDDict(ddict);
unsigned const actualDictID = ZSTD_getDictID_fromFrame(cBuff, cSize);
CHECK(actualDictID == expectedDictID,
"DictID mismatch: expected %u got %u",
expectedDictID,
actualDictID);
/* Decompress using the dictionary.
* If you need to control the decompression parameters, then use the
* advanced API: ZSTD_DCtx_setParameter(), ZSTD_DCtx_refDDict(), and
* ZSTD_decompressDCtx().
*/
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
CHECK_ZSTD(dSize);
/* When zstd knows the content size, it will error if it doesn't match. */
CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
ZSTD_freeDCtx(dctx);
free(rBuff);
free(cBuff);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc<3) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s [FILES] dictionary\n", exeName);
return 1;
}
/* load dictionary only once */
const char* const dictName = argv[argc-1];
ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
int u;
for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
ZSTD_freeDDict(dictPtr);
printf("All %u files correctly decoded (in memory) \n", argc-2);
return 0;
}

140
third_party/zstd/examples/multiple_simple_compression.c vendored
View file

@ -1,140 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // memcpy, strlen
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
typedef struct {
void* fBuffer;
void* cBuffer;
size_t fBufferSize;
size_t cBufferSize;
ZSTD_CCtx* cctx;
} resources;
/*
* allocate memory for buffers big enough to compress all files
* as well as memory for output file name (ofn)
*/
static resources createResources_orDie(int argc, const char** argv, char **ofn, size_t* ofnBufferLen)
{
size_t maxFilenameLength=0;
size_t maxFileSize = 0;
int argNb;
for (argNb = 1; argNb < argc; argNb++) {
const char* const filename = argv[argNb];
size_t const filenameLength = strlen(filename);
size_t const fileSize = fsize_orDie(filename);
if (filenameLength > maxFilenameLength) maxFilenameLength = filenameLength;
if (fileSize > maxFileSize) maxFileSize = fileSize;
}
resources ress;
ress.fBufferSize = maxFileSize;
ress.cBufferSize = ZSTD_compressBound(maxFileSize);
*ofnBufferLen = maxFilenameLength + 5;
*ofn = (char*)malloc_orDie(*ofnBufferLen);
ress.fBuffer = malloc_orDie(ress.fBufferSize);
ress.cBuffer = malloc_orDie(ress.cBufferSize);
ress.cctx = ZSTD_createCCtx();
CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
return ress;
}
static void freeResources(resources ress, char *outFilename)
{
free(ress.fBuffer);
free(ress.cBuffer);
ZSTD_freeCCtx(ress.cctx); /* never fails */
free(outFilename);
}
/* compress with pre-allocated context (ZSTD_CCtx) and input/output buffers*/
static void compressFile_orDie(resources ress, const char* fname, const char* oname)
{
size_t fSize = loadFile_orDie(fname, ress.fBuffer, ress.fBufferSize);
/* Compress using the context.
* If you need more control over parameters, use the advanced API:
* ZSTD_CCtx_setParameter(), and ZSTD_compress2().
*/
size_t const cSize = ZSTD_compressCCtx(ress.cctx, ress.cBuffer, ress.cBufferSize, ress.fBuffer, fSize, 1);
CHECK_ZSTD(cSize);
saveFile_orDie(oname, ress.cBuffer, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc<2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE(s)\n", exeName);
return 1;
}
/* memory allocation for outFilename and resources */
char* outFilename;
size_t outFilenameBufferLen;
resources const ress = createResources_orDie(argc, argv, &outFilename, &outFilenameBufferLen);
/* compress files with shared context, input and output buffers */
int argNb;
for (argNb = 1; argNb < argc; argNb++) {
const char* const inFilename = argv[argNb];
size_t const inFilenameLen = strlen(inFilename);
CHECK(inFilenameLen + 5 <= outFilenameBufferLen, "File name too long!");
memcpy(outFilename, inFilename, inFilenameLen);
memcpy(outFilename+inFilenameLen, ".zst", 5);
compressFile_orDie(ress, inFilename, outFilename);
}
/* free memory */
freeResources(ress,outFilename);
printf("compressed %i files \n", argc-1);
return 0;
}

157
third_party/zstd/examples/multiple_streaming_compression.c vendored
View file

@ -1,157 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* The objective of this example is to show of to compress multiple successive files
* while preserving memory management.
* All structures and buffers will be created only once,
* and shared across all compression operations */
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // memset, strcat
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
typedef struct {
void* buffIn;
void* buffOut;
size_t buffInSize;
size_t buffOutSize;
ZSTD_CCtx* cctx;
} resources;
static resources createResources_orDie(int cLevel)
{
resources ress;
ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */
ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */
ress.buffIn = malloc_orDie(ress.buffInSize);
ress.buffOut= malloc_orDie(ress.buffOutSize);
ress.cctx = ZSTD_createCCtx();
CHECK(ress.cctx != NULL, "ZSTD_createCCtx() failed!");
/* Set any compression parameters you want here.
* They will persist for every compression operation.
* Here we set the compression level, and enable the checksum.
*/
CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
CHECK_ZSTD( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, 1) );
return ress;
}
static void freeResources(resources ress)
{
ZSTD_freeCCtx(ress.cctx);
free(ress.buffIn);
free(ress.buffOut);
}
static void compressFile_orDie(resources ress, const char* fname, const char* outName)
{
// Open the input and output files.
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
/* Reset the context to a clean state to start a new compression operation.
* The parameters are sticky, so we keep the compression level and extra
* parameters that we set in createResources_orDie().
*/
CHECK_ZSTD( ZSTD_CCtx_reset(ress.cctx, ZSTD_reset_session_only) );
size_t const toRead = ress.buffInSize;
size_t read;
while ( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
/* This loop is the same as streaming_compression.c.
* See that file for detailed comments.
*/
int const lastChunk = (read < toRead);
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
ZSTD_inBuffer input = { ress.buffIn, read, 0 };
int finished;
do {
ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
size_t const remaining = ZSTD_compressStream2(ress.cctx, &output, &input, mode);
CHECK_ZSTD(remaining);
fwrite_orDie(ress.buffOut, output.pos, fout);
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished);
CHECK(input.pos == input.size,
"Impossible: zstd only returns 0 when the input is completely consumed!");
}
fclose_orDie(fout);
fclose_orDie(fin);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc<2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE(s)\n", exeName);
return 1;
}
int const cLevel = 7;
resources const ress = createResources_orDie(cLevel);
void* ofnBuffer = NULL;
size_t ofnbSize = 0;
int argNb;
for (argNb = 1; argNb < argc; argNb++) {
const char* const ifn = argv[argNb];
size_t const ifnSize = strlen(ifn);
size_t const ofnSize = ifnSize + 5;
if (ofnbSize <= ofnSize) {
ofnbSize = ofnSize + 16;
free(ofnBuffer);
ofnBuffer = malloc_orDie(ofnbSize);
}
memset(ofnBuffer, 0, ofnSize);
strcat(ofnBuffer, ifn);
strcat(ofnBuffer, ".zst");
compressFile_orDie(ress, ifn, ofnBuffer);
}
freeResources(ress);
free(ofnBuffer);
printf("compressed %i files \n", argc-1);
return 0;
}

92
third_party/zstd/examples/simple_compression.c vendored
View file

@ -1,92 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // strlen, strcat, memset
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void compress_orDie(const char* fname, const char* oname)
{
size_t fSize;
void* const fBuff = mallocAndLoadFile_orDie(fname, &fSize);
size_t const cBuffSize = ZSTD_compressBound(fSize);
void* const cBuff = malloc_orDie(cBuffSize);
/* Compress.
* If you are doing many compressions, you may want to reuse the context.
* See the multiple_simple_compression.c example.
*/
size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
CHECK_ZSTD(cSize);
saveFile_orDie(oname, cBuff, cSize);
/* success */
printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
free(fBuff);
free(cBuff);
}
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* const outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
const char* const inFilename = argv[1];
char* const outFilename = createOutFilename_orDie(inFilename);
compress_orDie(inFilename, outFilename);
free(outFilename);
return 0;
}

87
third_party/zstd/examples/simple_decompression.c vendored
View file

@ -1,87 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void decompress(const char* fname)
{
size_t cSize;
void* const cBuff = mallocAndLoadFile_orDie(fname, &cSize);
/* Read the content size from the frame header. For simplicity we require
* that it is always present. By default, zstd will write the content size
* in the header when it is known. If you can't guarantee that the frame
* content size is always written into the header, either use streaming
* decompression, or ZSTD_decompressBound().
*/
unsigned long long const rSize = ZSTD_getFrameContentSize(cBuff, cSize);
CHECK(rSize != ZSTD_CONTENTSIZE_ERROR, "%s: not compressed by zstd!", fname);
CHECK(rSize != ZSTD_CONTENTSIZE_UNKNOWN, "%s: original size unknown!", fname);
void* const rBuff = malloc_orDie((size_t)rSize);
/* Decompress.
* If you are doing many decompressions, you may want to reuse the context
* and use ZSTD_decompressDCtx(). If you want to set advanced parameters,
* use ZSTD_DCtx_setParameter().
*/
size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
CHECK_ZSTD(dSize);
/* When zstd knows the content size, it will error if it doesn't match. */
CHECK(dSize == rSize, "Impossible because zstd will check this condition!");
/* success */
printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
free(rBuff);
free(cBuff);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc!=2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE\n", exeName);
return 1;
}
decompress(argv[1]);
printf("%s correctly decoded (in memory). \n", argv[1]);
return 0;
}

170
third_party/zstd/examples/streaming_compression.c vendored
View file

@ -1,170 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // memset, strcat, strlen
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void compressFile_orDie(const char* fname, const char* outName, int cLevel,
int nbThreads)
{
fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n",
fname, cLevel, nbThreads);
/* Open the input and output files. */
FILE* const fin = fopen_orDie(fname, "rb");
FILE* const fout = fopen_orDie(outName, "wb");
/* Create the input and output buffers.
* They may be any size, but we recommend using these functions to size them.
* Performance will only suffer significantly for very tiny buffers.
*/
size_t const buffInSize = ZSTD_CStreamInSize();
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize();
void* const buffOut = malloc_orDie(buffOutSize);
/* Create the context. */
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
/* Set any parameters you want.
* Here we set the compression level, and enable the checksum.
*/
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel) );
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
if (nbThreads > 1) {
size_t const r = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
if (ZSTD_isError(r)) {
fprintf (stderr, "Note: the linked libzstd library doesn't support multithreading. "
"Reverting to single-thread mode. \n");
}
}
/* This loop read from the input file, compresses that entire chunk,
* and writes all output produced to the output file.
*/
size_t const toRead = buffInSize;
for (;;) {
size_t read = fread_orDie(buffIn, toRead, fin);
/* Select the flush mode.
* If the read may not be finished (read == toRead) we use
* ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
* Zstd optimizes the case where the first flush mode is ZSTD_e_end,
* since it knows it is compressing the entire source in one pass.
*/
int const lastChunk = (read < toRead);
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
/* Set the input buffer to what we just read.
* We compress until the input buffer is empty, each time flushing the
* output.
*/
ZSTD_inBuffer input = { buffIn, read, 0 };
int finished;
do {
/* Compress into the output buffer and write all of the output to
* the file so we can reuse the buffer next iteration.
*/
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
CHECK_ZSTD(remaining);
fwrite_orDie(buffOut, output.pos, fout);
/* If we're on the last chunk we're finished when zstd returns 0,
* which means its consumed all the input AND finished the frame.
* Otherwise, we're finished when we've consumed all the input.
*/
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished);
CHECK(input.pos == input.size,
"Impossible: zstd only returns 0 when the input is completely consumed!");
if (lastChunk) {
break;
}
}
ZSTD_freeCCtx(cctx);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
}
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* const outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc < 2) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s FILE [LEVEL] [THREADS]\n", exeName);
return 1;
}
int cLevel = 1;
int nbThreads = 1;
if (argc >= 3) {
cLevel = atoi (argv[2]);
CHECK(cLevel != 0, "can't parse LEVEL!");
}
if (argc >= 4) {
nbThreads = atoi (argv[3]);
CHECK(nbThreads != 0, "can't parse THREADS!");
}
const char* const inFilename = argv[1];
char* const outFilename = createOutFilename_orDie(inFilename);
compressFile_orDie(inFilename, outFilename, cLevel, nbThreads);
free(outFilename); /* not strictly required, since program execution stops there,
* but some static analyzer may complain otherwise */
return 0;
}

207
third_party/zstd/examples/streaming_compression_thread_pool.c vendored
View file

@ -1,207 +0,0 @@
// clang-format off
/*
* Copyright (c) Martin Liska, SUSE, Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
#include "libc/mem/alg.h"
#include "libc/mem/mem.h"
#include "libc/str/str.h" // memset, strcat, strlen
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
#include "libc/calls/weirdtypes.h"
#include "libc/sysv/consts/clock.h"
#include "libc/thread/thread.h"
#include "libc/thread/thread2.h"
typedef struct compress_args
{
const char *fname;
char *outName;
int cLevel;
#if defined(ZSTD_STATIC_LINKING_ONLY)
ZSTD_threadPool *pool;
#endif
} compress_args_t;
static void *compressFile_orDie(void *data)
{
const int nbThreads = 16;
compress_args_t *args = (compress_args_t *)data;
fprintf (stderr, "Starting compression of %s with level %d, using %d threads\n", args->fname, args->cLevel, nbThreads);
/* Open the input and output files. */
FILE* const fin = fopen_orDie(args->fname, "rb");
FILE* const fout = fopen_orDie(args->outName, "wb");
/* Create the input and output buffers.
* They may be any size, but we recommend using these functions to size them.
* Performance will only suffer significantly for very tiny buffers.
*/
size_t const buffInSize = ZSTD_CStreamInSize();
void* const buffIn = malloc_orDie(buffInSize);
size_t const buffOutSize = ZSTD_CStreamOutSize();
void* const buffOut = malloc_orDie(buffOutSize);
/* Create the context. */
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
#if defined(ZSTD_STATIC_LINKING_ONLY)
size_t r = ZSTD_CCtx_refThreadPool(cctx, args->pool);
CHECK(r == 0, "ZSTD_CCtx_refThreadPool failed!");
#endif
/* Set any parameters you want.
* Here we set the compression level, and enable the checksum.
*/
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, args->cLevel) );
CHECK_ZSTD( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads);
/* This loop reads from the input file, compresses that entire chunk,
* and writes all output produced to the output file.
*/
size_t const toRead = buffInSize;
for (;;) {
size_t read = fread_orDie(buffIn, toRead, fin);
/* Select the flush mode.
* If the read may not be finished (read == toRead) we use
* ZSTD_e_continue. If this is the last chunk, we use ZSTD_e_end.
* Zstd optimizes the case where the first flush mode is ZSTD_e_end,
* since it knows it is compressing the entire source in one pass.
*/
int const lastChunk = (read < toRead);
ZSTD_EndDirective const mode = lastChunk ? ZSTD_e_end : ZSTD_e_continue;
/* Set the input buffer to what we just read.
* We compress until the input buffer is empty, each time flushing the
* output.
*/
ZSTD_inBuffer input = { buffIn, read, 0 };
int finished;
do {
/* Compress into the output buffer and write all of the output to
* the file so we can reuse the buffer next iteration.
*/
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
size_t const remaining = ZSTD_compressStream2(cctx, &output , &input, mode);
CHECK_ZSTD(remaining);
fwrite_orDie(buffOut, output.pos, fout);
/* If we're on the last chunk we're finished when zstd returns 0,
* which means its consumed all the input AND finished the frame.
* Otherwise, we're finished when we've consumed all the input.
*/
finished = lastChunk ? (remaining == 0) : (input.pos == input.size);
} while (!finished);
CHECK(input.pos == input.size,
"Impossible: zstd only returns 0 when the input is completely consumed!");
if (lastChunk) {
break;
}
}
fprintf (stderr, "Finishing compression of %s\n", args->outName);
ZSTD_freeCCtx(cctx);
fclose_orDie(fout);
fclose_orDie(fin);
free(buffIn);
free(buffOut);
free(args->outName);
return NULL;
}
static char* createOutFilename_orDie(const char* filename)
{
size_t const inL = strlen(filename);
size_t const outL = inL + 5;
void* const outSpace = malloc_orDie(outL);
memset(outSpace, 0, outL);
strcat(outSpace, filename);
strcat(outSpace, ".zst");
return (char*)outSpace;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc<=3) {
printf("wrong arguments\n");
printf("usage:\n");
printf("%s POOL_SIZE LEVEL FILES\n", exeName);
return 1;
}
int pool_size = atoi (argv[1]);
CHECK(pool_size != 0, "can't parse POOL_SIZE!");
int level = atoi (argv[2]);
CHECK(level != 0, "can't parse LEVEL!");
argc -= 3;
argv += 3;
#if defined(ZSTD_STATIC_LINKING_ONLY)
ZSTD_threadPool *pool = ZSTD_createThreadPool (pool_size);
CHECK(pool != NULL, "ZSTD_createThreadPool() failed!");
fprintf (stderr, "Using shared thread pool of size %d\n", pool_size);
#else
fprintf (stderr, "All threads use its own thread pool\n");
#endif
pthread_t *threads = malloc_orDie(argc * sizeof(pthread_t));
compress_args_t *args = malloc_orDie(argc * sizeof(compress_args_t));
for (unsigned i = 0; i < argc; i++)
{
args[i].fname = argv[i];
args[i].outName = createOutFilename_orDie(args[i].fname);
args[i].cLevel = level;
#if defined(ZSTD_STATIC_LINKING_ONLY)
args[i].pool = pool;
#endif
pthread_create (&threads[i], NULL, compressFile_orDie, &args[i]);
}
for (unsigned i = 0; i < argc; i++)
pthread_join (threads[i], NULL);
#if defined(ZSTD_STATIC_LINKING_ONLY)
ZSTD_freeThreadPool (pool);
#endif
return 0;
}

122
third_party/zstd/examples/streaming_decompression.c vendored
View file

@ -1,122 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // fprintf
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/fmt/conv.h"
#include "libc/limits.h"
#include "libc/mem/alg.h"
#include "libc/mem/alloca.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/rand.h"
#include "libc/stdio/temp.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/musl/crypt.h"
#include "third_party/musl/rand48.h" // free
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
static void decompressFile_orDie(const char* fname)
{
FILE* const fin = fopen_orDie(fname, "rb");
size_t const buffInSize = ZSTD_DStreamInSize();
void* const buffIn = malloc_orDie(buffInSize);
FILE* const fout = stdout;
size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
void* const buffOut = malloc_orDie(buffOutSize);
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
/* This loop assumes that the input file is one or more concatenated zstd
* streams. This example won't work if there is trailing non-zstd data at
* the end, but streaming decompression in general handles this case.
* ZSTD_decompressStream() returns 0 exactly when the frame is completed,
* and doesn't consume input after the frame.
*/
size_t const toRead = buffInSize;
size_t read;
size_t lastRet = 0;
int isEmpty = 1;
while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
isEmpty = 0;
ZSTD_inBuffer input = { buffIn, read, 0 };
/* Given a valid frame, zstd won't consume the last byte of the frame
* until it has flushed all of the decompressed data of the frame.
* Therefore, instead of checking if the return code is 0, we can
* decompress just check if input.pos < input.size.
*/
while (input.pos < input.size) {
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
/* The return code is zero if the frame is complete, but there may
* be multiple frames concatenated together. Zstd will automatically
* reset the context when a frame is complete. Still, calling
* ZSTD_DCtx_reset() can be useful to reset the context to a clean
* state, for instance if the last decompression call returned an
* error.
*/
size_t const ret = ZSTD_decompressStream(dctx, &output , &input);
CHECK_ZSTD(ret);
fwrite_orDie(buffOut, output.pos, fout);
lastRet = ret;
}
}
if (isEmpty) {
fprintf(stderr, "input is empty\n");
exit(1);
}
if (lastRet != 0) {
/* The last return value from ZSTD_decompressStream did not end on a
* frame, but we reached the end of the file! We assume this is an
* error, and the input was truncated.
*/
fprintf(stderr, "EOF before end of stream: %zu\n", lastRet);
exit(1);
}
ZSTD_freeDCtx(dctx);
fclose_orDie(fin);
fclose_orDie(fout);
free(buffIn);
free(buffOut);
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
if (argc!=2) {
fprintf(stderr, "wrong arguments\n");
fprintf(stderr, "usage:\n");
fprintf(stderr, "%s FILE\n", exeName);
return 1;
}
const char* const inFilename = argv[1];
decompressFile_orDie(inFilename);
return 0;
}

144
third_party/zstd/examples/streaming_memory_usage.c vendored
View file

@ -1,144 +0,0 @@
// clang-format off
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/*=== Tuning parameter ===*/
#ifndef MAX_TESTED_LEVEL
#define MAX_TESTED_LEVEL 12
#endif
/*=== Dependencies ===*/
#include "libc/calls/calls.h"
#include "libc/calls/weirdtypes.h"
#include "libc/fmt/fmt.h"
#include "libc/stdio/dprintf.h"
#include "libc/stdio/stdio.h"
#include "libc/stdio/temp.h"
#include "third_party/musl/tempnam.h" // printf
#define ZSTD_STATIC_LINKING_ONLY
// MISSING #include <zstd.h> // presumes zstd library is installed
#include "third_party/zstd/examples/common.h" // Helper functions, CHECK(), and CHECK_ZSTD()
/*=== functions ===*/
/*! readU32FromChar() :
@return : unsigned integer value read from input in `char` format
allows and interprets K, KB, KiB, M, MB and MiB suffix.
Will also modify `*stringPtr`, advancing it to position where it stopped reading.
Note : function result can overflow if digit string > MAX_UINT */
static unsigned readU32FromChar(const char** stringPtr)
{
unsigned result = 0;
while ((**stringPtr >='0') && (**stringPtr <='9'))
result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
if ((**stringPtr=='K') || (**stringPtr=='M')) {
result <<= 10;
if (**stringPtr=='M') result <<= 10;
(*stringPtr)++ ;
if (**stringPtr=='i') (*stringPtr)++;
if (**stringPtr=='B') (*stringPtr)++;
}
return result;
}
int main(int argc, char const *argv[]) {
printf("\n Zstandard (v%s) memory usage for streaming : \n\n", ZSTD_versionString());
unsigned wLog = 0;
if (argc > 1) {
const char* valStr = argv[1];
wLog = readU32FromChar(&valStr);
}
int compressionLevel;
for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) {
#define INPUT_SIZE 5
#define COMPRESSED_SIZE 128
char const dataToCompress[INPUT_SIZE] = "abcde";
char compressedData[COMPRESSED_SIZE];
char decompressedData[INPUT_SIZE];
/* the ZSTD_CCtx_params structure is a way to save parameters and use
* them across multiple contexts. We use them here so we can call the
* function ZSTD_estimateCStreamSize_usingCCtxParams().
*/
ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams();
CHECK(cctxParams != NULL, "ZSTD_createCCtxParams() failed!");
/* Set the compression level. */
CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, compressionLevel) );
/* Set the window log.
* The value 0 means use the default window log, which is equivalent to
* not setting it.
*/
CHECK_ZSTD( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, wLog) );
/* Force the compressor to allocate the maximum memory size for a given
* level by not providing the pledged source size, or calling
* ZSTD_compressStream2() with ZSTD_e_end.
*/
ZSTD_CCtx* const cctx = ZSTD_createCCtx();
CHECK(cctx != NULL, "ZSTD_createCCtx() failed!");
CHECK_ZSTD( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) );
size_t compressedSize;
{
ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
CHECK_ZSTD( ZSTD_compressStream(cctx, &outBuff, &inBuff) );
size_t const remaining = ZSTD_endStream(cctx, &outBuff);
CHECK_ZSTD(remaining);
CHECK(remaining == 0, "Frame not flushed!");
compressedSize = outBuff.pos;
}
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
CHECK(dctx != NULL, "ZSTD_createDCtx() failed!");
/* Set the maximum allowed window log.
* The value 0 means use the default window log, which is equivalent to
* not setting it.
*/
CHECK_ZSTD( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, wLog) );
/* forces decompressor to use maximum memory size, since the
* decompressed size is not stored in the frame header.
*/
{ ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
size_t const remaining = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
CHECK_ZSTD(remaining);
CHECK(remaining == 0, "Frame not complete!");
CHECK(outBuff.pos == sizeof(dataToCompress), "Bad decompression!");
}
size_t const cstreamSize = ZSTD_sizeof_CStream(cctx);
size_t const cstreamEstimatedSize = ZSTD_estimateCStreamSize_usingCCtxParams(cctxParams);
size_t const dstreamSize = ZSTD_sizeof_DStream(dctx);
size_t const dstreamEstimatedSize = ZSTD_estimateDStreamSize_fromFrame(compressedData, compressedSize);
CHECK(cstreamSize <= cstreamEstimatedSize, "Compression mem (%u) > estimated (%u)",
(unsigned)cstreamSize, (unsigned)cstreamEstimatedSize);
CHECK(dstreamSize <= dstreamEstimatedSize, "Decompression mem (%u) > estimated (%u)",
(unsigned)dstreamSize, (unsigned)dstreamEstimatedSize);
printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB (estimated : %5u KB)\n",
compressionLevel,
(unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10),
(unsigned)(dstreamSize>>10), (unsigned)(dstreamEstimatedSize>>10));
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtx(cctx);
ZSTD_freeCCtxParams(cctxParams);
if (wLog) break; /* single test */
}
return 0;
}

234
third_party/zstd/lib/README.md vendored
View file

@ -1,234 +0,0 @@
// clang-format off
Zstandard library files
================================
The __lib__ directory is split into several sub-directories,
in order to make it easier to select or exclude features.
#### Building
`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
including commands variables, staged install, directory variables and standard targets.
- `make` : generates both static and dynamic libraries
- `make install` : install libraries and headers in target system directories
`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
and support for decoding legacy formats >= v0.5.0.
The scope can be reduced on demand (see paragraph _modular build_).
#### Multithreading support
When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons).
Enabling multithreading requires 2 conditions :
- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
For convenience, we provide a build target to generate multi and single threaded libraries:
- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`.
- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`.
- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`.
When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to invoke the `-pthread` flag during link stage.
Multithreading capabilities are exposed
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351).
#### API
Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
#### Advanced API
Optional advanced features are exposed via :
- `lib/zstd_errors.h` : translates `size_t` function results
into a `ZSTD_ErrorCode`, for accurate error handling.
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
it unlocks access to the experimental API,
exposed in the second part of `zstd.h`.
All definitions in the experimental APIs are unstable,
they may still change in the future, or even be removed.
As a consequence, experimental definitions shall ___never be used with dynamic library___ !
Only static linking is allowed.
#### Modular build
It's possible to compile only a limited set of features within `libzstd`.
The file structure is designed to make this selection manually achievable for any build system :
- Directory `lib/common` is always required, for all variants.
- Compression source code lies in `lib/compress`
- Decompression source code lies in `lib/decompress`
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
The API is exposed in `lib/dictBuilder/zdict.h`.
This module depends on both `lib/common` and `lib/compress` .
- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
This module depends on `lib/common` and `lib/decompress`.
To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
Specifying a number limits versions supported to that version onward.
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
Decoding supported legacy format is a transparent capability triggered within decompression functions.
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
Each version does also provide its own set of advanced API.
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
- While invoking `make libzstd`, it's possible to define build macros
`ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the
corresponding features. This will also disable compilation of all
dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable
dictBuilder).
- There are a number of options that can help minimize the binary size of
`libzstd`.
The first step is to select the components needed (using the above-described
`ZSTD_LIB_COMPRESSION` etc.).
The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This
disables various optional components and changes the compilation flags to
prioritize space-saving.
Detailed options: Zstandard's code and build environment is set up by default
to optimize above all else for performance. In pursuit of this goal, Zstandard
makes significant trade-offs in code size. For example, Zstandard often has
more than one implementation of a particular component, with each
implementation optimized for different scenarios. For example, the Huffman
decoder has complementary implementations that decode the stream one symbol at
a time or two symbols at a time. Zstd normally includes both (and dispatches
between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or
`HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
only one or the other of two decompression implementations. The smallest
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
On the compressor side, Zstd's compression levels map to several internal
strategies. In environments where the higher compression levels aren't used,
it is possible to exclude all but the fastest strategy with
`ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP=1`. (Note that this will change
the behavior of the default compression level.) Or if you want to retain the
default compressor as well, you can set
`ZSTD_LIB_EXCLUDE_COMPRESSORS_GREEDY_AND_UP=1`, at the cost of an additional
~20KB or so.
For squeezing the last ounce of size out, you can also define
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
which removes the error messages that are otherwise returned by
`ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`).
Finally, when integrating into your application, make sure you're doing link-
time optimization and unused symbol garbage collection (via some combination of,
e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`,
`-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`,
`-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands
the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your
compiler's documentation.
- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1`
will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in
the shared library, which is now hidden by default.
- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries
which can detect at runtime the presence of BMI2 instructions, and use them only if present.
These instructions contribute to better performance, notably on the decoder side.
By default, this feature is automatically enabled on detecting
the right instruction set (x64) and compiler (clang or gcc >= 5).
It's obviously disabled for different cpus,
or when BMI2 instruction set is _required_ by the compiler command line
(in this case, only the BMI2 code path is generated).
Setting this macro will either force to generate the BMI2 dispatcher (1)
or prevent it (0). It overrides automatic detection.
- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions
that zstd does not use. Not all unused functions are hidden, but they can be if needed.
Currently, this macro will hide function definitions in FSE and HUF that use an excessive
amount of stack space.
- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics.
Compiler builtins are still used.
- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control
the amount of extra memory used during decompression to store literals.
This defaults to 64kB. Reducing this value reduces the memory footprint of
`ZSTD_DCtx` decompression contexts,
but might also result in a small decompression speed cost.
- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE`
can be overridden to control the visibility of zstd's API. Additionally,
`ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility
of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols
from the shared library. These macros default to `ZSTDLIB_VISIBILITY`,
`ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility
with the old macro names.
#### Windows : using MinGW+MSYS to create DLL
DLL can be created using MinGW+MSYS with the `make libzstd` command.
This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`.
The import library is only required with Visual C++.
The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to
compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses ZSTD consists of a single `test-dll.c`
file it should be linked with `dll\libzstd.dll`. For example:
```
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll
```
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
#### Advanced Build options
The build system requires a hash function in order to
separate object files created with different compilation flags.
By default, it tries to use `md5sum` or equivalent.
The hash function can be manually switched by setting the `HASH` variable.
For example : `make HASH=xxhsum`
The hash function needs to generate at least 64-bit using hexadecimal format.
When no hash function is found,
the Makefile just generates all object files into the same default directory,
irrespective of compilation flags.
This functionality only matters if `libzstd` is compiled multiple times
with different build flags.
The build directory, where object files are stored
can also be manually controlled using variable `BUILD_DIR`,
for example `make BUILD_DIR=objectDir/v1`.
In which case, the hash function doesn't matter.
#### Deprecated API
Obsolete API on their way out are stored in directory `lib/deprecated`.
At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`.
These prototypes will be removed in some future version.
Consider migrating code towards supported streaming API exposed in `zstd.h`.
#### Miscellaneous
The other files are not source code. There are :
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
- `README.md` : this file
- `dll/` : resources directory for Windows compilation
- `libzstd.pc.in` : script for `pkg-config` (used in `make install`)

2
third_party/zstd/lib/common/allocations.h vendored
View file

@ -17,7 +17,7 @@
#include "third_party/zstd/lib/common/compiler.h" /* MEM_STATIC */
#define ZSTD_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/zstd.h" /* ZSTD_customMem */
#include "third_party/zstd/zstd.h" /* ZSTD_customMem */
#ifndef ZSTD_ALLOCATIONS_H
#define ZSTD_ALLOCATIONS_H

2
third_party/zstd/lib/common/error_private.h vendored
View file

@ -22,7 +22,7 @@ extern "C" {
/* ****************************************
* Dependencies
******************************************/
#include "third_party/zstd/lib/zstd_errors.h" /* enum list */
#include "third_party/zstd/zstd_errors.h" /* enum list */
#include "third_party/zstd/lib/common/compiler.h"
#include "third_party/zstd/lib/common/debug.h"
#include "third_party/zstd/lib/common/zstd_deps.h" /* size_t */

2
third_party/zstd/lib/common/pool.h vendored
View file

@ -19,7 +19,7 @@ extern "C" {
#include "third_party/zstd/lib/common/zstd_deps.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
#include "libc/assert.h"
typedef struct POOL_ctx_s POOL_ctx;

2
third_party/zstd/lib/common/zstd_internal.h vendored
View file

@ -26,7 +26,7 @@
#include "third_party/zstd/lib/common/debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */
#include "third_party/zstd/lib/common/error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
#define FSE_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/common/fse.h"
#include "third_party/zstd/lib/common/huf.h"

2
third_party/zstd/lib/compress/clevels.h vendored
View file

@ -13,7 +13,7 @@
#define ZSTD_CLEVELS_H
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
/*-===== Pre-defined compression levels =====-*/

2
third_party/zstd/lib/compress/zstd_compress_superblock.h vendored
View file

@ -16,7 +16,7 @@
* Dependencies
***************************************/
#include "third_party/zstd/lib/zstd.h" /* ZSTD_CCtx */
#include "third_party/zstd/zstd.h" /* ZSTD_CCtx */
/*-*************************************
* Target Compressed Block Size

2
third_party/zstd/lib/compress/zstd_ldm.h vendored
View file

@ -17,7 +17,7 @@ extern "C" {
#endif
#include "third_party/zstd/lib/compress/zstd_compress_internal.h" /* ldmParams_t, U32 */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_CCtx, size_t */
#include "third_party/zstd/zstd.h" /* ZSTD_CCtx, size_t */
/*-*************************************
* Long distance matching

2
third_party/zstd/lib/compress/zstdmt_compress.h vendored
View file

@ -29,7 +29,7 @@
/* === Dependencies === */
#include "third_party/zstd/lib/common/zstd_deps.h" /* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
#include "third_party/zstd/zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Constants === */

2
third_party/zstd/lib/decompress/zstd_ddict.h vendored
View file

@ -17,7 +17,7 @@
* Dependencies
*********************************************************/
#include "third_party/zstd/lib/common/zstd_deps.h" /* size_t */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_DDict, and several public functions */
#include "third_party/zstd/zstd.h" /* ZSTD_DDict, and several public functions */
/*-*******************************************************

2
third_party/zstd/lib/decompress/zstd_decompress_block.h vendored
View file

@ -17,7 +17,7 @@
* Dependencies
*********************************************************/
#include "third_party/zstd/lib/common/zstd_deps.h" /* size_t */
#include "third_party/zstd/lib/zstd.h" /* DCtx, and some public functions */
#include "third_party/zstd/zstd.h" /* DCtx, and some public functions */
#include "third_party/zstd/lib/common/zstd_internal.h" /* blockProperties_t, and some public functions */
#include "third_party/zstd/lib/decompress/zstd_decompress_internal.h" /* ZSTD_seqSymbol */

4
third_party/zstd/lib/deprecated/zbuff.h vendored
View file

@ -29,7 +29,7 @@ extern "C" {
* Dependencies
***************************************/
/* size_t */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
#include "third_party/zstd/zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
/* ***************************************************************
@ -187,7 +187,7 @@ ZBUFF_DEPRECATED("use ZSTD_DStreamOutSize") size_t ZBUFF_recommendedDOutSize(voi
/*--- Dependency ---*/
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
/*--- Custom memory allocator ---*/

2
third_party/zstd/lib/deprecated/zbuff_decompress.c vendored
View file

@ -15,7 +15,7 @@
* Dependencies
***************************************/
#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* suppress warning on ZSTD_initDStream_usingDict */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
#include "third_party/zstd/zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
#define ZBUFF_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/deprecated/zbuff.h"

2
third_party/zstd/lib/dictBuilder/cover.c vendored
View file

@ -67,7 +67,7 @@
#include "third_party/zstd/lib/common/threading.h"
#include "third_party/zstd/lib/common/zstd_internal.h" /* includes zstd.h */
#include "third_party/zstd/lib/common/bits.h" /* ZSTD_highbit32 */
#include "third_party/zstd/lib/zdict.h"
#include "third_party/zstd/zdict.h"
#include "third_party/zstd/lib/dictBuilder/cover.h"
/*-*************************************

2
third_party/zstd/lib/dictBuilder/cover.h vendored
View file

@ -52,7 +52,7 @@
#include "third_party/zstd/lib/common/pool.h"
#include "third_party/zstd/lib/common/threading.h"
#include "third_party/zstd/lib/common/zstd_internal.h" /* includes zstd.h */
#include "third_party/zstd/lib/zdict.h"
#include "third_party/zstd/zdict.h"
/**
* COVER_best_t is used for two purposes:

2
third_party/zstd/lib/dictBuilder/fastcover.c vendored
View file

@ -57,7 +57,7 @@
#include "third_party/zstd/lib/common/threading.h"
#include "third_party/zstd/lib/common/zstd_internal.h" /* includes zstd.h */
#include "third_party/zstd/lib/compress/zstd_compress_internal.h" /* ZSTD_hash*() */
#include "third_party/zstd/lib/zdict.h"
#include "third_party/zstd/zdict.h"
#include "third_party/zstd/lib/dictBuilder/cover.h"

2
third_party/zstd/lib/dictBuilder/zdict.c vendored
View file

@ -83,7 +83,7 @@
#include "third_party/zstd/lib/common/zstd_internal.h" /* includes zstd.h */
#include "third_party/zstd/lib/common/xxhash.h" /* XXH64 */
#include "third_party/zstd/lib/compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
#include "third_party/zstd/lib/zdict.h"
#include "third_party/zstd/zdict.h"
#include "third_party/zstd/lib/dictBuilder/divsufsort.h"
#include "third_party/zstd/lib/common/bits.h" /* ZSTD_NbCommonBytes */

49
third_party/zstd/lib/dll/example/Makefile vendored
View file

@ -1,49 +0,0 @@
// clang-format off
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
VOID := /dev/null
ZSTDDIR := ../include
LIBDIR := ../static
DLLDIR := ../dll
CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make
CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \
-Wdeclaration-after-statement -Wstrict-prototypes \
-Wpointer-arith -Wstrict-aliasing=1
CFLAGS += $(MOREFLAGS)
CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_
FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
else
EXT =
endif
.PHONY: default fullbench-dll fullbench-lib
default: all
all: fullbench-dll fullbench-lib
fullbench-lib: fullbench.c datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib
fullbench-dll: fullbench.c datagen.c
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll
clean:
@$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \
@echo Cleaning completed

64
third_party/zstd/lib/dll/example/README.md vendored
View file

@ -1,64 +0,0 @@
// clang-format off
# ZSTD Windows binary package
## The package contents
- `zstd.exe` : Command Line Utility, supporting gzip-like arguments
- `dll\libzstd.dll` : The ZSTD dynamic library (DLL)
- `dll\libzstd.lib` : The import library of the ZSTD dynamic library (DLL) for Visual C++
- `example\` : The example of usage of the ZSTD library
- `include\` : Header files required by the ZSTD library
- `static\libzstd_static.lib` : The static ZSTD library (LIB)
## Usage of Command Line Interface
Command Line Interface (CLI) supports gzip-like arguments.
By default CLI takes an input file and compresses it to an output file:
Usage: zstd [arg] [input] [output]
The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can
be improved with commands from `-3` to `-16` but higher levels also have slower
compression. CLI includes in-memory compression benchmark module with compression
levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds.
CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined
into `-b1e18i1`.
## The example of usage of static and dynamic ZSTD libraries with gcc/MinGW
Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`.
`fullbench-dll` uses a dynamic ZSTD library from the `dll` directory.
`fullbench-lib` uses a static ZSTD library from the `lib` directory.
## Using ZSTD DLL with gcc/MinGW
The header files from `include\` and the dynamic library `dll\libzstd.dll`
are required to compile a project using gcc/MinGW.
The dynamic library has to be added to linking options.
It means that if a project that uses ZSTD consists of a single `test-dll.c`
file it should be linked with `dll\libzstd.dll`. For example:
gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\libzstd.dll
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
## The example of usage of static and dynamic ZSTD libraries with Visual C++
Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a
dynamic ZSTD library from the `dll` directory. The solution works with Visual C++
2010 or newer. When one will open the solution with Visual C++ newer than 2010
then the solution will be upgraded to the current version.
## Using ZSTD DLL with Visual C++
The header files from `include\` and the import library `dll\libzstd.lib`
are required to compile a project using Visual C++.
1. The path to header files should be added to `Additional Include Directories` that can
be found in project properties `C/C++` then `General`.
2. The import library has to be added to `Additional Dependencies` that can
be found in project properties `Linker` then `Input`.
If one will provide only the name `libzstd.lib` without a full path to the library
the directory has to be added to `Linker\General\Additional Library Directories`.
The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.

21
third_party/zstd/lib/dll/example/build_package.bat vendored
View file

@ -1,21 +0,0 @@
// clang-format off
@ECHO OFF
MKDIR bin\dll bin\static bin\example bin\include
COPY tests\fullbench.c bin\example\
COPY programs\datagen.c bin\example\
COPY programs\datagen.h bin\example\
COPY programs\util.h bin\example\
COPY programs\platform.h bin\example\
COPY lib\common\mem.h bin\example\
COPY lib\common\zstd_internal.h bin\example\
COPY lib\common\error_private.h bin\example\
COPY lib\common\xxhash.h bin\example\
COPY lib\libzstd.a bin\static\libzstd_static.lib
COPY lib\dll\libzstd.* bin\dll\
COPY lib\dll\example\Makefile bin\example\
COPY lib\dll\example\fullbench-dll.* bin\example\
COPY lib\dll\example\README.md bin\
COPY lib\zstd.h bin\include\
COPY lib\common\zstd_errors.h bin\include\
COPY lib\dictBuilder\zdict.h bin\include\
COPY programs\zstd.exe bin\zstd.exe

26
third_party/zstd/lib/dll/example/fullbench-dll.sln vendored
View file

@ -1,26 +0,0 @@
// clang-format off
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Express 2012 for Windows Desktop
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32
{13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32
{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64
{13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64
{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32
{13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32
{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64
{13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

182
third_party/zstd/lib/dll/example/fullbench-dll.vcxproj vendored
View file

@ -1,182 +0,0 @@
// clang-format off
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{00000000-1CC8-4FD7-9281-6B8DBB9D3DF8}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>fullbench-dll</RootNamespace>
<OutDir>$(SolutionDir)bin\$(Platform)_$(Configuration)\</OutDir>
<IntDir>$(SolutionDir)bin\obj\$(RootNamespace)_$(Platform)_$(Configuration)\</IntDir>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>MultiByte</CharacterSet>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(IncludePath);$(SolutionDir)..\..\lib;$(SolutionDir)..\..\programs;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
<RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
<ImageHasSafeExceptionHandlers>false</ImageHasSafeExceptionHandlers>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level4</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;ZSTD_DLL_IMPORT=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<AdditionalIncludeDirectories>..\include</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<AdditionalLibraryDirectories>$(SolutionDir)..\dll;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>libzstd.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="datagen.c" />
<ClCompile Include="fullbench.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\include\zstd.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

42
third_party/zstd/programs/.gitignore vendored
View file

@ -1,42 +0,0 @@
// clang-format off
# local binary (Makefile)
zstd
zstd32
zstd4
zstd-compress
zstd-decompress
zstd-frugal
zstd-small
zstd-nolegacy
zstd-dictBuilder
zstd-dll
zstd_arm64
zstd_x64
# Object files
*.o
*.ko
default.profraw
have_zlib
# Executables
*.exe
*.out
*.app
# Default result files
dictionary
grillResults.txt
_*
tmp*
*.zst
result
out
# fuzzer
afl
# Misc files
*.bat
!windres/generate_res.bat
dirTest*

45
third_party/zstd/programs/BUCK vendored
View file

@ -1,45 +0,0 @@
// clang-format off
cxx_binary(
name='zstd',
headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']),
srcs=glob(['*.c'], excludes=['datagen.c']),
deps=[
':datagen',
':util',
'//lib:zstd',
'//lib:zdict',
'//lib:mem',
'//lib:xxhash',
],
preprocessor_flags=[
'-DZSTD_GZCOMPRESS',
'-DZSTD_GZDECOMPRESS',
'-DZSTD_LZMACOMPRESS',
'-DZSTD_LZMADECOMPRES',
'-DZSTD_LZ4COMPRESS',
'-DZSTD_LZ4DECOMPRES',
],
linker_flags=[
'-lz',
'-llzma',
'-llz4',
],
)
cxx_library(
name='datagen',
visibility=['PUBLIC'],
header_namespace='',
exported_headers=['datagen.h'],
srcs=['datagen.c'],
deps=['//lib:mem'],
)
cxx_library(
name='util',
visibility=['PUBLIC'],
header_namespace='',
exported_headers=['util.h', 'platform.h'],
deps=['//lib:mem'],
)

447
third_party/zstd/programs/Makefile vendored
View file

@ -1,447 +0,0 @@
// clang-format off
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ##########################################################################
# zstd : Command Line Utility, supporting gzip-like arguments
# zstd32 : Same as zstd, but forced to compile in 32-bits mode
# zstd-nolegacy : zstd without support of decompression of legacy versions
# zstd-small : minimal zstd without dictionary builder and benchmark
# zstd-compress : compressor-only version of zstd
# zstd-decompress : decompressor-only version of zstd
# ##########################################################################
.PHONY: default
default: zstd-release
LIBZSTD := ../lib
include $(LIBZSTD)/libzstd.mk
ifeq ($(shell $(CC) -v 2>&1 | $(GREP) -c "gcc version "), 1)
ALIGN_LOOP = -falign-loops=32
else
ALIGN_LOOP =
endif
ZSTDLIB_COMMON_SRC := $(sort $(ZSTD_COMMON_FILES))
ZSTDLIB_COMPRESS_SRC := $(sort $(ZSTD_COMPRESS_FILES))
ZSTDLIB_DECOMPRESS_SRC := $(sort $(ZSTD_DECOMPRESS_FILES))
ZSTDLIB_CORE_SRC := $(sort $(ZSTD_DECOMPRESS_FILES) $(ZSTD_COMMON_FILES) $(ZSTD_COMPRESS_FILES))
ZDICT_SRC := $(sort $(ZSTD_DICTBUILDER_FILES))
ZSTDLEGACY_SRC := $(sort $(ZSTD_LEGACY_FILES))
# Sort files in alphabetical order for reproducible builds
ZSTDLIB_FULL_SRC = $(sort $(ZSTDLIB_CORE_SRC) $(ZSTDLEGACY_SRC) $(ZDICT_SRC))
ZSTDLIB_LOCAL_SRC = $(notdir $(ZSTDLIB_FULL_SRC))
ZSTDLIB_LOCAL_OBJ0 := $(ZSTDLIB_LOCAL_SRC:.c=.o)
ZSTDLIB_LOCAL_OBJ := $(ZSTDLIB_LOCAL_OBJ0:.S=.o)
ZSTD_CLI_SRC := $(sort $(wildcard *.c))
ZSTD_CLI_OBJ := $(ZSTD_CLI_SRC:.c=.o)
ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
ZSTD_ALL_OBJ0 := $(ZSTD_ALL_SRC:.c=.o)
ZSTD_ALL_OBJ := $(ZSTD_ALL_OBJ0:.S=.o)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
RES64_FILE = windres/zstd64.res
RES32_FILE = windres/zstd32.res
ifneq (,$(filter x86_64%,$(shell $(CC) -dumpmachine)))
RES_FILE = $(RES64_FILE)
else
RES_FILE = $(RES32_FILE)
endif
else
EXT =
endif
# thread detection
NO_THREAD_MSG := ==> no threads, building without multithreading support
HAVE_PTHREAD := $(shell printf '$(NUM_SYMBOL)include <pthread.h>\nint main(void) { return 0; }' > have_pthread.c && $(CC) $(FLAGS) -o have_pthread$(EXT) have_pthread.c -pthread 2> $(VOID) && rm have_pthread$(EXT) && echo 1 || echo 0; rm have_pthread.c)
HAVE_THREAD := $(shell [ "$(HAVE_PTHREAD)" -eq "1" -o -n "$(filter Windows%,$(OS))" ] && echo 1 || echo 0)
ifeq ($(HAVE_THREAD), 1)
THREAD_MSG := ==> building with threading support
THREAD_CPP := -DZSTD_MULTITHREAD
THREAD_LD := -pthread
else
THREAD_MSG := $(NO_THREAD_MSG)
endif
# zlib detection
NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
HAVE_ZLIB ?= $(shell printf '$(NUM_SYMBOL)include <zlib.h>\nint main(void) { return 0; }' > have_zlib.c && $(CC) $(FLAGS) -o have_zlib$(EXT) have_zlib.c -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0; rm have_zlib.c)
ifeq ($(HAVE_ZLIB), 1)
ZLIB_MSG := ==> building zstd with .gz compression support
ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
ZLIBLD = -lz
else
ZLIB_MSG := $(NO_ZLIB_MSG)
endif
# lzma detection
NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
HAVE_LZMA ?= $(shell printf '$(NUM_SYMBOL)include <lzma.h>\nint main(void) { return 0; }' > have_lzma.c && $(CC) $(FLAGS) -o have_lzma$(EXT) have_lzma.c -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0; rm have_lzma.c)
ifeq ($(HAVE_LZMA), 1)
LZMA_MSG := ==> building zstd with .xz/.lzma compression support
LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
LZMALD = -llzma
else
LZMA_MSG := $(NO_LZMA_MSG)
endif
# lz4 detection
NO_LZ4_MSG := ==> no liblz4, building zstd without .lz4 support
HAVE_LZ4 ?= $(shell printf '$(NUM_SYMBOL)include <lz4frame.h>\n$(NUM_SYMBOL)include <lz4.h>\nint main(void) { return 0; }' > have_lz4.c && $(CC) $(FLAGS) -o have_lz4$(EXT) have_lz4.c -llz4 2> $(VOID) && rm have_lz4$(EXT) && echo 1 || echo 0; rm have_lz4.c)
ifeq ($(HAVE_LZ4), 1)
LZ4_MSG := ==> building zstd with .lz4 compression support
LZ4CPP = -DZSTD_LZ4COMPRESS -DZSTD_LZ4DECOMPRESS
LZ4LD = -llz4
else
LZ4_MSG := $(NO_LZ4_MSG)
endif
# explicit backtrace enable/disable for Linux & Darwin
ifeq ($(BACKTRACE), 0)
DEBUGFLAGS += -DBACKTRACE_ENABLE=0
endif
ifeq (,$(filter Windows%, $(OS)))
ifeq ($(BACKTRACE), 1)
DEBUGFLAGS += -DBACKTRACE_ENABLE=1
DEBUGFLAGS_LD += -rdynamic
endif
endif
SET_CACHE_DIRECTORY = \
+$(MAKE) --no-print-directory $@ \
BUILD_DIR=obj/$(HASH_DIR) \
CPPFLAGS="$(CPPFLAGS)" \
CFLAGS="$(CFLAGS)" \
LDFLAGS="$(LDFLAGS)" \
LDLIBS="$(LDLIBS)" \
ZSTD_ALL_SRC="$(ZSTD_ALL_SRC)"
.PHONY: all
all: zstd zstd-compress zstd-decompress zstd-small
.PHONY: allVariants
allVariants: all zstd-frugal zstd-nolegacy zstd-dictBuilder
.PHONY: zstd # must always be run
zstd : CPPFLAGS += $(THREAD_CPP) $(ZLIBCPP) $(LZMACPP) $(LZ4CPP)
zstd : LDFLAGS += $(THREAD_LD) $(DEBUGFLAGS_LD)
zstd : LDLIBS += $(ZLIBLD) $(LZMALD) $(LZ4LD)
zstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ifneq (,$(filter Windows%,$(OS)))
zstd : $(RES_FILE)
endif
ifndef BUILD_DIR
# generate BUILD_DIR from flags
zstd:
$(SET_CACHE_DIRECTORY)
else
# BUILD_DIR is defined
ZSTD_OBJ := $(addprefix $(BUILD_DIR)/, $(ZSTD_ALL_OBJ))
$(BUILD_DIR)/zstd : $(ZSTD_OBJ)
@echo "$(THREAD_MSG)"
@echo "$(ZLIB_MSG)"
@echo "$(LZMA_MSG)"
@echo "$(LZ4_MSG)"
@echo LINK $@
$(CC) $(FLAGS) $^ $(LDLIBS) -o $@$(EXT)
ifeq ($(HAVE_HASH),1)
SRCBIN_HASH = $(shell cat $(BUILD_DIR)/zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
DSTBIN_HASH = $(shell cat zstd$(EXT) 2> $(VOID) | $(HASH) | cut -f 1 -d " ")
BIN_ISDIFFERENT = $(if $(filter $(SRCBIN_HASH),$(DSTBIN_HASH)),0,1)
else
BIN_ISDIFFERENT = 1
endif
zstd : $(BUILD_DIR)/zstd
if [ $(BIN_ISDIFFERENT) -eq 1 ]; then \
cp -f $<$(EXT) $@$(EXT); \
echo zstd build completed; \
else \
echo zstd already built; \
fi
endif # BUILD_DIR
CLEAN += zstd
.PHONY: zstd-release
zstd-release: DEBUGFLAGS := -DBACKTRACE_ENABLE=0
zstd-release: DEBUGFLAGS_LD :=
zstd-release: zstd
CLEAN += zstd32
zstd32 : CPPFLAGS += $(THREAD_CPP)
zstd32 : LDFLAGS += $(THREAD_LD)
zstd32 : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
ifneq (,$(filter Windows%,$(OS)))
zstd32 : $(RES32_FILE)
endif
zstd32 : $(ZSTDLIB_FULL_SRC) $(ZSTD_CLI_SRC)
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
## zstd-nolegacy: same scope as zstd, with removed support of legacy formats
CLEAN += zstd-nolegacy
zstd-nolegacy : LDFLAGS += $(THREAD_LD) $(ZLIBLD) $(LZMALD) $(LZ4LD) $(DEBUGFLAGS_LD)
zstd-nolegacy : CPPFLAGS += -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0
zstd-nolegacy : $(ZSTDLIB_CORE_SRC) $(ZDICT_SRC) $(ZSTD_CLI_OBJ)
$(CC) $(FLAGS) $^ -o $@$(EXT) $(LDFLAGS)
.PHONY: zstd-nomt
zstd-nomt : THREAD_CPP :=
zstd-nomt : THREAD_LD :=
zstd-nomt : THREAD_MSG := - multi-threading disabled
zstd-nomt : zstd
.PHONY: zstd-nogz
zstd-nogz : ZLIBCPP :=
zstd-nogz : ZLIBLD :=
zstd-nogz : ZLIB_MSG := - gzip support is disabled
zstd-nogz : zstd
.PHONY: zstd-noxz
zstd-noxz : LZMACPP :=
zstd-noxz : LZMALD :=
zstd-noxz : LZMA_MSG := - xz/lzma support is disabled
zstd-noxz : zstd
## zstd-dll: zstd executable linked to dynamic library libzstd (must have same version)
.PHONY: zstd-dll
zstd-dll : LDFLAGS+= -L$(LIBZSTD)
zstd-dll : LDLIBS += -lzstd
zstd-dll : ZSTDLIB_LOCAL_SRC = xxhash.c pool.c threading.c
zstd-dll : zstd
## zstd-pgo: zstd executable optimized with PGO.
.PHONY: zstd-pgo
zstd-pgo : LLVM_PROFDATA?=llvm-profdata
zstd-pgo : PROF_GENERATE_FLAGS=-fprofile-generate $(if $(findstring gcc,$(CC)),-fprofile-dir=.)
zstd-pgo : PROF_USE_FLAGS=-fprofile-use $(if $(findstring gcc,$(CC)),-fprofile-dir=. -Werror=missing-profile -Wno-error=coverage-mismatch)
zstd-pgo :
$(MAKE) clean HASH_DIR=$(HASH_DIR)
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_GENERATE_FLAGS)"
./zstd -b19i1 $(PROFILE_WITH)
./zstd -b16i1 $(PROFILE_WITH)
./zstd -b9i2 $(PROFILE_WITH)
./zstd -b $(PROFILE_WITH)
./zstd -b7i2 $(PROFILE_WITH)
./zstd -b5 $(PROFILE_WITH)
ifndef BUILD_DIR
$(RM) zstd obj/$(HASH_DIR)/zstd obj/$(HASH_DIR)/*.o
else
$(RM) zstd $(BUILD_DIR)/zstd $(BUILD_DIR)/*.o
endif
case $(CC) in *clang*) if ! [ -e default.profdata ]; then $(LLVM_PROFDATA) merge -output=default.profdata default*.profraw; fi ;; esac
$(MAKE) zstd HASH_DIR=$(HASH_DIR) MOREFLAGS="$(PROF_USE_FLAGS)"
## zstd-small: minimal target, supporting only zstd compression and decompression. no bench. no legacy. no other format.
CLEAN += zstd-small zstd-frugal
zstd-small: CFLAGS = -Os -Wl,-s
zstd-frugal zstd-small: $(ZSTDLIB_CORE_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
CLEAN += zstd-decompress
zstd-decompress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_DECOMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
CLEAN += zstd-compress
zstd-compress: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_NOTRACE -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
## zstd-dictBuilder: executable supporting dictionary creation and compression (only)
CLEAN += zstd-dictBuilder
zstd-dictBuilder: $(ZSTDLIB_COMMON_SRC) $(ZSTDLIB_COMPRESS_SRC) $(ZDICT_SRC) zstdcli.c util.c timefn.c fileio.c fileio_asyncio.c dibio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODECOMPRESS -DZSTD_NOTRACE $^ -o $@$(EXT)
CLEAN += zstdmt
zstdmt: zstd
ln -sf zstd zstdmt
.PHONY: generate_res
generate_res: $(RES64_FILE) $(RES32_FILE)
ifneq (,$(filter Windows%,$(OS)))
RC ?= windres
# https://stackoverflow.com/questions/708238/how-do-i-add-an-icon-to-a-mingw-gcc-compiled-executable
$(RES64_FILE): windres/zstd.rc
$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-x86-64
$(RES32_FILE): windres/zstd.rc
$(RC) -o $@ -I ../lib -I windres -i $< -O coff -F pe-i386
endif
.PHONY: clean
clean:
$(RM) $(CLEAN) core *.o tmp* result* dictionary *.zst \
*.gcda default*.profraw default.profdata have_zlib
$(RM) -r obj/*
@echo Cleaning completed
MD2ROFF = ronn
MD2ROFF_FLAGS = --roff --warnings --manual="User Commands" --organization="zstd $(ZSTD_VERSION)"
zstd.1: zstd.1.md ../lib/zstd.h
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
zstdgrep.1: zstdgrep.1.md ../lib/zstd.h
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
zstdless.1: zstdless.1.md ../lib/zstd.h
cat $< | $(MD2ROFF) $(MD2ROFF_FLAGS) | sed -n '/^\.\\\".*/!p' > $@
.PHONY: man
man: zstd.1 zstdgrep.1 zstdless.1
.PHONY: clean-man
clean-man:
$(RM) zstd.1
$(RM) zstdgrep.1
$(RM) zstdless.1
.PHONY: preview-man
preview-man: clean-man man
man ./zstd.1
man ./zstdgrep.1
man ./zstdless.1
# Generate .h dependencies automatically
DEPFLAGS = -MT $@ -MMD -MP -MF
$(BUILD_DIR)/%.o : %.c $(BUILD_DIR)/%.d | $(BUILD_DIR)
@echo CC $@
$(COMPILE.c) $(DEPFLAGS) $(BUILD_DIR)/$*.d $(OUTPUT_OPTION) $<
$(BUILD_DIR)/%.o : %.S | $(BUILD_DIR)
@echo AS $@
$(COMPILE.S) $(OUTPUT_OPTION) $<
MKDIR ?= mkdir
$(BUILD_DIR): ; $(MKDIR) -p $@
DEPFILES := $(ZSTD_OBJ:.o=.d)
$(DEPFILES):
include $(wildcard $(DEPFILES))
#-----------------------------------------------------------------------------
# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX))
HAVE_COLORNEVER = $(shell echo a | egrep --color=never a > /dev/null 2> /dev/null && echo 1 || echo 0)
EGREP_OPTIONS ?=
ifeq ($(HAVE_COLORNEVER), 1)
EGREP_OPTIONS += --color=never
endif
EGREP = egrep $(EGREP_OPTIONS)
AWK = awk
# Print a two column output of targets and their description. To add a target description, put a
# comment in the Makefile with the format "## <TARGET>: <DESCRIPTION>". For example:
#
## list: Print all targets and their descriptions (if provided)
.PHONY: list
list:
TARGETS=$$($(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null \
| $(AWK) -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' \
| $(EGREP) -v -e '^[^[:alnum:]]' | sort); \
{ \
printf "Target Name\tDescription\n"; \
printf "%0.s-" {1..16}; printf "\t"; printf "%0.s-" {1..40}; printf "\n"; \
for target in $$TARGETS; do \
line=$$($(EGREP) "^##[[:space:]]+$$target:" $(lastword $(MAKEFILE_LIST))); \
description=$$(echo $$line | $(AWK) '{i=index($$0,":"); print substr($$0,i+1)}' | xargs); \
printf "$$target\t$$description\n"; \
done \
} | column -t -s $$'\t'
DESTDIR ?=
# directory variables : GNU conventions prefer lowercase
# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
# support both lower and uppercase (BSD), use uppercase in script
prefix ?= /usr/local
PREFIX ?= $(prefix)
exec_prefix ?= $(PREFIX)
bindir ?= $(exec_prefix)/bin
BINDIR ?= $(bindir)
datarootdir ?= $(PREFIX)/share
mandir ?= $(datarootdir)/man
man1dir ?= $(mandir)/man1
ifneq (,$(filter $(UNAME),OpenBSD FreeBSD NetBSD DragonFly SunOS))
MANDIR ?= $(PREFIX)/man
MAN1DIR ?= $(MANDIR)/man1
else
MAN1DIR ?= $(man1dir)
endif
ifneq (,$(filter $(UNAME),SunOS))
INSTALL ?= ginstall
else
INSTALL ?= install
endif
INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_SCRIPT ?= $(INSTALL_PROGRAM)
INSTALL_DATA ?= $(INSTALL) -m 644
INSTALL_MAN ?= $(INSTALL_DATA)
.PHONY: install
install:
# generate zstd only if not already present
[ -e zstd ] || $(MAKE) zstd-release
[ -e $(DESTDIR)$(BINDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
[ -e $(DESTDIR)$(MAN1DIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(MAN1DIR)/
@echo Installing binaries
$(INSTALL_PROGRAM) zstd$(EXT) $(DESTDIR)$(BINDIR)/zstd$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdcat$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/unzstd$(EXT)
ln -sf zstd$(EXT) $(DESTDIR)$(BINDIR)/zstdmt$(EXT)
$(INSTALL_SCRIPT) zstdless $(DESTDIR)$(BINDIR)/zstdless
$(INSTALL_SCRIPT) zstdgrep $(DESTDIR)$(BINDIR)/zstdgrep
@echo Installing man pages
$(INSTALL_MAN) zstd.1 $(DESTDIR)$(MAN1DIR)/zstd.1
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/zstdcat.1
ln -sf zstd.1 $(DESTDIR)$(MAN1DIR)/unzstd.1
$(INSTALL_MAN) zstdgrep.1 $(DESTDIR)$(MAN1DIR)/zstdgrep.1
$(INSTALL_MAN) zstdless.1 $(DESTDIR)$(MAN1DIR)/zstdless.1
@echo zstd installation completed
.PHONY: uninstall
uninstall:
$(RM) $(DESTDIR)$(BINDIR)/zstdgrep
$(RM) $(DESTDIR)$(BINDIR)/zstdless
$(RM) $(DESTDIR)$(BINDIR)/zstdcat
$(RM) $(DESTDIR)$(BINDIR)/unzstd
$(RM) $(DESTDIR)$(BINDIR)/zstdmt
$(RM) $(DESTDIR)$(BINDIR)/zstd
$(RM) $(DESTDIR)$(MAN1DIR)/zstdless.1
$(RM) $(DESTDIR)$(MAN1DIR)/zstdgrep.1
$(RM) $(DESTDIR)$(MAN1DIR)/zstdcat.1
$(RM) $(DESTDIR)$(MAN1DIR)/unzstd.1
$(RM) $(DESTDIR)$(MAN1DIR)/zstd.1
@echo zstd programs successfully uninstalled
endif

1
third_party/zstd/programs/README.md vendored
View file

@ -1,4 +1,3 @@
// clang-format off
Command Line Interface for Zstandard library
============================================

4
third_party/zstd/programs/benchzstd.c vendored
View file

@ -58,14 +58,14 @@
#ifndef ZSTD_STATIC_LINKING_ONLY
#define ZSTD_STATIC_LINKING_ONLY
#endif
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
#include "third_party/zstd/programs/datagen.h" /* RDG_genBuffer */
#ifndef XXH_INLINE_ALL
#define XXH_INLINE_ALL
#endif
#include "third_party/zstd/lib/common/xxhash.h"
#include "third_party/zstd/programs/benchzstd.h"
#include "third_party/zstd/lib/zstd_errors.h"
#include "third_party/zstd/zstd_errors.h"
/* *************************************

2
third_party/zstd/programs/benchzstd.h vendored
View file

@ -25,7 +25,7 @@ extern "C" {
/* === Dependencies === */
/* size_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_compressionParameters */
#include "third_party/zstd/zstd.h" /* ZSTD_compressionParameters */
/* === Constants === */

2
third_party/zstd/programs/dibio.c vendored
View file

@ -56,7 +56,7 @@
#include "third_party/zstd/programs/timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
#include "third_party/zstd/lib/common/debug.h" /* assert */
#include "third_party/zstd/lib/common/mem.h" /* read */
#include "third_party/zstd/lib/zstd_errors.h"
#include "third_party/zstd/zstd_errors.h"
#include "third_party/zstd/programs/dibio.h"

2
third_party/zstd/programs/dibio.h vendored
View file

@ -20,7 +20,7 @@
* Dependencies
***************************************/
#define ZDICT_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/zdict.h" /* ZDICT_params_t */
#include "third_party/zstd/zdict.h" /* ZDICT_params_t */
/*-*************************************

4
third_party/zstd/programs/fileio.c vendored
View file

@ -108,8 +108,8 @@ FIO_display_prefs_t g_display_prefs = {2, FIO_ps_auto};
UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
#include "third_party/zstd/zstd.h"
#include "third_party/zstd/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
// MISSING #include <zlib.h>

2
third_party/zstd/programs/fileio.h vendored
View file

@ -16,7 +16,7 @@
#include "third_party/zstd/programs/fileio_types.h"
#include "third_party/zstd/programs/util.h" /* FileNamesTable */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_* */
#include "third_party/zstd/zstd.h" /* ZSTD_* */
#if defined (__cplusplus)
extern "C" {

2
third_party/zstd/programs/fileio_types.h vendored
View file

@ -13,7 +13,7 @@
#define FILEIO_TYPES_HEADER
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */
#include "third_party/zstd/lib/zstd.h" /* ZSTD_* */
#include "third_party/zstd/zstd.h" /* ZSTD_* */
/*-*************************************
* Parameters: FIO_prefs_t

17
third_party/zstd/programs/windres/verrsrc.h vendored
View file

@ -1,17 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* minimal set of defines required to generate zstd.res from zstd.rc */
#define VS_VERSION_INFO 1
#define VS_FFI_FILEFLAGSMASK 0x0000003FL
#define VOS_NT_WINDOWS32 0x00040004L
#define VFT_DLL 0x00000002L
#define VFT2_UNKNOWN 0x00000000L

51
third_party/zstd/programs/windres/zstd.rc vendored
View file

@ -1,51 +0,0 @@
// Microsoft Visual C++ generated resource script.
//
#include "zstd.h" /* ZSTD_VERSION_STRING */
#define APSTUDIO_READONLY_SYMBOLS
#include "verrsrc.h"
#undef APSTUDIO_READONLY_SYMBOLS
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
LANGUAGE 9, 1
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_DLL
FILESUBTYPE VFT2_UNKNOWN
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904B0"
BEGIN
VALUE "CompanyName", "Meta Platforms, Inc."
VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm"
VALUE "FileVersion", ZSTD_VERSION_STRING
VALUE "InternalName", "zstd.exe"
VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates."
VALUE "OriginalFilename", "zstd.exe"
VALUE "ProductName", "Zstandard"
VALUE "ProductVersion", ZSTD_VERSION_STRING
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x0409, 1200
END
END
#endif

BIN
third_party/zstd/programs/windres/zstd32.res vendored
View file

Binary file not shown.

BIN
third_party/zstd/programs/windres/zstd64.res vendored
View file

Binary file not shown.

382
third_party/zstd/programs/zstd.1 vendored
View file

@ -1,382 +0,0 @@
// clang-format off
.TH "ZSTD" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
.SH "SYNOPSIS"
.TS
allbox;
\fBzstd\fR [\fIOPTIONS\fR] [\- \fIINPUT\-FILE\fR] [\-o \fIOUTPUT\-FILE\fR]
.TE
.P
\fBzstdmt\fR is equivalent to \fBzstd \-T0\fR
.P
\fBunzstd\fR is equivalent to \fBzstd \-d\fR
.P
\fBzstdcat\fR is equivalent to \fBzstd \-dcf\fR
.SH "DESCRIPTION"
\fBzstd\fR is a fast lossless compression algorithm and data compression tool, with command line syntax similar to \fBgzip\fR(1) and \fBxz\fR(1)\. It is based on the \fBLZ77\fR family, with further FSE & huff0 entropy stages\. \fBzstd\fR offers highly configurable compression speed, from fast modes at > 200 MB/s per core, to strong modes with excellent compression ratios\. It also features a very fast decoder, with speeds > 500 MB/s per core\.
.P
\fBzstd\fR command line syntax is generally similar to gzip, but features the following differences:
.IP "\[ci]" 4
Source files are preserved by default\. It\'s possible to remove them automatically by using the \fB\-\-rm\fR command\.
.IP "\[ci]" 4
When compressing a single file, \fBzstd\fR displays progress notifications and result summary by default\. Use \fB\-q\fR to turn them off\.
.IP "\[ci]" 4
\fBzstd\fR displays a short help page when command line is an error\. Use \fB\-q\fR to turn it off\.
.IP "\[ci]" 4
\fBzstd\fR does not accept input from console, though it does accept \fBstdin\fR when it\'s not the console\.
.IP "\[ci]" 4
\fBzstd\fR does not store the input\'s filename or attributes, only its contents\.
.IP "" 0
.P
\fBzstd\fR processes each \fIfile\fR according to the selected operation mode\. If no \fIfiles\fR are given or \fIfile\fR is \fB\-\fR, \fBzstd\fR reads from standard input and writes the processed data to standard output\. \fBzstd\fR will refuse to write compressed data to standard output if it is a terminal: it will display an error message and skip the file\. Similarly, \fBzstd\fR will refuse to read compressed data from standard input if it is a terminal\.
.P
Unless \fB\-\-stdout\fR or \fB\-o\fR is specified, \fIfiles\fR are written to a new file whose name is derived from the source \fIfile\fR name:
.IP "\[ci]" 4
When compressing, the suffix \fB\.zst\fR is appended to the source filename to get the target filename\.
.IP "\[ci]" 4
When decompressing, the \fB\.zst\fR suffix is removed from the source filename to get the target filename
.IP "" 0
.SS "Concatenation with \.zst Files"
It is possible to concatenate multiple \fB\.zst\fR files\. \fBzstd\fR will decompress such agglomerated file as if it was a single \fB\.zst\fR file\.
.SH "OPTIONS"
.SS "Integer Suffixes and Special Values"
In most places where an integer argument is expected, an optional suffix is supported to easily indicate large integers\. There must be no space between the integer and the suffix\.
.TP
\fBKiB\fR
Multiply the integer by 1,024 (2\e^10)\. \fBKi\fR, \fBK\fR, and \fBKB\fR are accepted as synonyms for \fBKiB\fR\.
.TP
\fBMiB\fR
Multiply the integer by 1,048,576 (2\e^20)\. \fBMi\fR, \fBM\fR, and \fBMB\fR are accepted as synonyms for \fBMiB\fR\.
.SS "Operation Mode"
If multiple operation mode options are given, the last one takes effect\.
.TP
\fB\-z\fR, \fB\-\-compress\fR
Compress\. This is the default operation mode when no operation mode option is specified and no other operation mode is implied from the command name (for example, \fBunzstd\fR implies \fB\-\-decompress\fR)\.
.TP
\fB\-d\fR, \fB\-\-decompress\fR, \fB\-\-uncompress\fR
Decompress\.
.TP
\fB\-t\fR, \fB\-\-test\fR
Test the integrity of compressed \fIfiles\fR\. This option is equivalent to \fB\-\-decompress \-\-stdout > /dev/null\fR, decompressed data is discarded and checksummed for errors\. No files are created or removed\.
.TP
\fB\-b#\fR
Benchmark file(s) using compression level \fI#\fR\. See \fIBENCHMARK\fR below for a description of this operation\.
.TP
\fB\-\-train FILES\fR
Use \fIFILES\fR as a training set to create a dictionary\. The training set should contain a lot of small files (> 100)\. See \fIDICTIONARY BUILDER\fR below for a description of this operation\.
.TP
\fB\-l\fR, \fB\-\-list\fR
Display information related to a zstd compressed file, such as size, ratio, and checksum\. Some of these fields may not be available\. This command\'s output can be augmented with the \fB\-v\fR modifier\.
.SS "Operation Modifiers"
.IP "\[ci]" 4
\fB\-#\fR: selects \fB#\fR compression level [1\-19] (default: 3)
.IP "\[ci]" 4
\fB\-\-ultra\fR: unlocks high compression levels 20+ (maximum 22), using a lot more memory\. Note that decompression will also require more memory when using these levels\.
.IP "\[ci]" 4
\fB\-\-fast[=#]\fR: switch to ultra\-fast compression levels\. If \fB=#\fR is not present, it defaults to \fB1\fR\. The higher the value, the faster the compression speed, at the cost of some compression ratio\. This setting overwrites compression level if one was set previously\. Similarly, if a compression level is set after \fB\-\-fast\fR, it overrides it\.
.IP "\[ci]" 4
\fB\-T#\fR, \fB\-\-threads=#\fR: Compress using \fB#\fR working threads (default: 1)\. If \fB#\fR is 0, attempt to detect and use the number of physical CPU cores\. In all cases, the nb of threads is capped to \fBZSTDMT_NBWORKERS_MAX\fR, which is either 64 in 32\-bit mode, or 256 for 64\-bit environments\. This modifier does nothing if \fBzstd\fR is compiled without multithread support\.
.IP "\[ci]" 4
\fB\-\-single\-thread\fR: Use a single thread for both I/O and compression\. As compression is serialized with I/O, this can be slightly slower\. Single\-thread mode features significantly lower memory usage, which can be useful for systems with limited amount of memory, such as 32\-bit systems\.
.IP
Note 1: this mode is the only available one when multithread support is disabled\.
.IP
Note 2: this mode is different from \fB\-T1\fR, which spawns 1 compression thread in parallel with I/O\. Final compressed result is also slightly different from \fB\-T1\fR\.
.IP "\[ci]" 4
\fB\-\-auto\-threads={physical,logical} (default: physical)\fR: When using a default amount of threads via \fB\-T0\fR, choose the default based on the number of detected physical or logical cores\.
.IP "\[ci]" 4
\fB\-\-adapt[=min=#,max=#]\fR: \fBzstd\fR will dynamically adapt compression level to perceived I/O conditions\. Compression level adaptation can be observed live by using command \fB\-v\fR\. Adaptation can be constrained between supplied \fBmin\fR and \fBmax\fR levels\. The feature works when combined with multi\-threading and \fB\-\-long\fR mode\. It does not work with \fB\-\-single\-thread\fR\. It sets window size to 8 MiB by default (can be changed manually, see \fBwlog\fR)\. Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible\.
.IP
\fINote\fR: at the time of this writing, \fB\-\-adapt\fR can remain stuck at low speed when combined with multiple worker threads (>=2)\.
.IP "\[ci]" 4
\fB\-\-long[=#]\fR: enables long distance matching with \fB#\fR \fBwindowLog\fR, if \fB#\fR is not present it defaults to \fB27\fR\. This increases the window size (\fBwindowLog\fR) and memory usage for both the compressor and decompressor\. This setting is designed to improve the compression ratio for files with long matches at a large distance\.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.IP "\[ci]" 4
\fB\-D DICT\fR: use \fBDICT\fR as Dictionary to compress or decompress FILE(s)
.IP "\[ci]" 4
\fB\-\-patch\-from FILE\fR: Specify the file to be used as a reference point for zstd\'s diff engine\. This is effectively dictionary compression with some convenient parameter selection, namely that \fIwindowSize\fR > \fIsrcSize\fR\.
.IP
Note: cannot use both this and \fB\-D\fR together\.
.IP
Note: \fB\-\-long\fR mode will be automatically activated if \fIchainLog\fR < \fIfileLog\fR (\fIfileLog\fR being the \fIwindowLog\fR required to cover the whole file)\. You can also manually force it\.
.IP
Note: for all levels, you can use \fB\-\-patch\-from\fR in \fB\-\-single\-thread\fR mode to improve compression ratio at the cost of speed\.
.IP
Note: for level 19, you can get increased compression ratio at the cost of speed by specifying \fB\-\-zstd=targetLength=\fR to be something large (i\.e\. 4096), and by setting a large \fB\-\-zstd=chainLog=\fR\.
.IP "\[ci]" 4
\fB\-\-rsyncable\fR: \fBzstd\fR will periodically synchronize the compression state to make the compressed file more rsync\-friendly\. There is a negligible impact to compression ratio, and a potential impact to compression speed, perceptible at higher speeds, for example when combining \fB\-\-rsyncable\fR with many parallel worker threads\. This feature does not work with \fB\-\-single\-thread\fR\. You probably don\'t want to use it with long range mode, since it will decrease the effectiveness of the synchronization points, but your mileage may vary\.
.IP "\[ci]" 4
\fB\-C\fR, \fB\-\-[no\-]check\fR: add integrity check computed from uncompressed data (default: enabled)
.IP "\[ci]" 4
\fB\-\-[no\-]content\-size\fR: enable / disable whether or not the original size of the file is placed in the header of the compressed file\. The default option is \fB\-\-content\-size\fR (meaning that the original size will be placed in the header)\.
.IP "\[ci]" 4
\fB\-\-no\-dictID\fR: do not store dictionary ID within frame header (dictionary compression)\. The decoder will have to rely on implicit knowledge about which dictionary to use, it won\'t be able to check if it\'s correct\.
.IP "\[ci]" 4
\fB\-M#\fR, \fB\-\-memory=#\fR: Set a memory usage limit\. By default, \fBzstd\fR uses 128 MiB for decompression as the maximum amount of memory the decompressor is allowed to use, but you can override this manually if need be in either direction (i\.e\. you can increase or decrease it)\.
.IP
This is also used during compression when using with \fB\-\-patch\-from=\fR\. In this case, this parameter overrides that maximum size allowed for a dictionary\. (128 MiB)\.
.IP
Additionally, this can be used to limit memory for dictionary training\. This parameter overrides the default limit of 2 GiB\. zstd will load training samples up to the memory limit and ignore the rest\.
.IP "\[ci]" 4
\fB\-\-stream\-size=#\fR: Sets the pledged source size of input coming from a stream\. This value must be exact, as it will be included in the produced frame header\. Incorrect stream sizes will cause an error\. This information will be used to better optimize compression parameters, resulting in better and potentially faster compression, especially for smaller source sizes\.
.IP "\[ci]" 4
\fB\-\-size\-hint=#\fR: When handling input from a stream, \fBzstd\fR must guess how large the source size will be when optimizing compression parameters\. If the stream size is relatively small, this guess may be a poor one, resulting in a higher compression ratio than expected\. This feature allows for controlling the guess when needed\. Exact guesses result in better compression ratios\. Overestimates result in slightly degraded compression ratios, while underestimates may result in significant degradation\.
.IP "\[ci]" 4
\fB\-o FILE\fR: save result into \fBFILE\fR\.
.IP "\[ci]" 4
\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. During decompression and when the output destination is stdout, pass\-through unrecognized formats as\-is\.
.IP "\[ci]" 4
\fB\-c\fR, \fB\-\-stdout\fR: write to standard output (even if it is the console); keep original files unchanged\.
.IP "\[ci]" 4
\fB\-\-[no\-]sparse\fR: enable / disable sparse FS support, to make files with many zeroes smaller on disk\. Creating sparse files may save disk space and speed up decompression by reducing the amount of disk I/O\. default: enabled when output is into a file, and disabled when output is stdout\. This setting overrides default and can force sparse mode over stdout\.
.IP "\[ci]" 4
\fB\-\-[no\-]pass\-through\fR enable / disable passing through uncompressed files as\-is\. During decompression when pass\-through is enabled, unrecognized formats will be copied as\-is from the input to the output\. By default, pass\-through will occur when the output destination is stdout and the force (\fB\-f\fR) option is set\.
.IP "\[ci]" 4
\fB\-\-rm\fR: remove source file(s) after successful compression or decompression\. This command is silently ignored if output is \fBstdout\fR\. If used in combination with \fB\-o\fR, triggers a confirmation prompt (which can be silenced with \fB\-f\fR), as this is a destructive operation\.
.IP "\[ci]" 4
\fB\-k\fR, \fB\-\-keep\fR: keep source file(s) after successful compression or decompression\. This is the default behavior\.
.IP "\[ci]" 4
\fB\-r\fR: operate recursively on directories\. It selects all files in the named directory and all its subdirectories\. This can be useful both to reduce command line typing, and to circumvent shell expansion limitations, when there are a lot of files and naming breaks the maximum size of a command line\.
.IP "\[ci]" 4
\fB\-\-filelist FILE\fR read a list of files to process as content from \fBFILE\fR\. Format is compatible with \fBls\fR output, with one file per line\.
.IP "\[ci]" 4
\fB\-\-output\-dir\-flat DIR\fR: resulting files are stored into target \fBDIR\fR directory, instead of same directory as origin file\. Be aware that this command can introduce name collision issues, if multiple files, from different directories, end up having the same name\. Collision resolution ensures first file with a given name will be present in \fBDIR\fR, while in combination with \fB\-f\fR, the last file will be present instead\.
.IP "\[ci]" 4
\fB\-\-output\-dir\-mirror DIR\fR: similar to \fB\-\-output\-dir\-flat\fR, the output files are stored underneath target \fBDIR\fR directory, but this option will replicate input directory hierarchy into output \fBDIR\fR\.
.IP
If input directory contains "\.\.", the files in this directory will be ignored\. If input directory is an absolute directory (i\.e\. "/var/tmp/abc"), it will be stored into the "output\-dir/var/tmp/abc"\. If there are multiple input files or directories, name collision resolution will follow the same rules as \fB\-\-output\-dir\-flat\fR\.
.IP "\[ci]" 4
\fB\-\-format=FORMAT\fR: compress and decompress in other formats\. If compiled with support, zstd can compress to or decompress from other compression algorithm formats\. Possibly available options are \fBzstd\fR, \fBgzip\fR, \fBxz\fR, \fBlzma\fR, and \fBlz4\fR\. If no such format is provided, \fBzstd\fR is the default\.
.IP "\[ci]" 4
\fB\-h\fR/\fB\-H\fR, \fB\-\-help\fR: display help/long help and exit
.IP "\[ci]" 4
\fB\-V\fR, \fB\-\-version\fR: display version number and exit\. Advanced: \fB\-vV\fR also displays supported formats\. \fB\-vvV\fR also displays POSIX support\. \fB\-q\fR will only display the version number, suitable for machine reading\.
.IP "\[ci]" 4
\fB\-v\fR, \fB\-\-verbose\fR: verbose mode, display more information
.IP "\[ci]" 4
\fB\-q\fR, \fB\-\-quiet\fR: suppress warnings, interactivity, and notifications\. specify twice to suppress errors too\.
.IP "\[ci]" 4
\fB\-\-no\-progress\fR: do not display the progress bar, but keep all other messages\.
.IP "\[ci]" 4
\fB\-\-show\-default\-cparams\fR: shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size\. If the provided file is not a regular file (e\.g\. a pipe), this flag will output the parameters used for inputs of unknown size\.
.IP "\[ci]" 4
\fB\-\-\fR: All arguments after \fB\-\-\fR are treated as files
.IP "" 0
.SS "gzip Operation Modifiers"
When invoked via a \fBgzip\fR symlink, \fBzstd\fR will support further options that intend to mimic the \fBgzip\fR behavior:
.TP
\fB\-n\fR, \fB\-\-no\-name\fR
do not store the original filename and timestamps when compressing a file\. This is the default behavior and hence a no\-op\.
.TP
\fB\-\-best\fR
alias to the option \fB\-9\fR\.
.SS "Environment Variables"
Employing environment variables to set parameters has security implications\. Therefore, this avenue is intentionally limited\. Only \fBZSTD_CLEVEL\fR and \fBZSTD_NBTHREADS\fR are currently supported\. They set the compression level and number of threads to use during compression, respectively\.
.P
\fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\.
.P
\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \fBZSTD_NBTHREADS\fR has a default value of (\fB1\fR), and is capped at ZSTDMT_NBWORKERS_MAX==200\. \fBzstd\fR must be compiled with multithread support for this to have any effect\.
.P
They can both be overridden by corresponding command line arguments: \fB\-#\fR for compression level and \fB\-T#\fR for number of compression threads\.
.SH "DICTIONARY BUILDER"
\fBzstd\fR offers \fIdictionary\fR compression, which greatly improves efficiency on small files and messages\. It\'s possible to train \fBzstd\fR with a set of samples, the result of which is saved into a file called a \fBdictionary\fR\. Then, during compression and decompression, reference the same dictionary, using command \fB\-D dictionaryFileName\fR\. Compression of small files similar to the sample set will be greatly improved\.
.TP
\fB\-\-train FILEs\fR
Use FILEs as training set to create a dictionary\. The training set should ideally contain a lot of samples (> 100), and weight typically 100x the target dictionary size (for example, ~10 MB for a 100 KB dictionary)\. \fB\-\-train\fR can be combined with \fB\-r\fR to indicate a directory rather than listing all the files, which can be useful to circumvent shell expansion limits\.
.IP
Since dictionary compression is mostly effective for small files, the expectation is that the training set will only contain small files\. In the case where some samples happen to be large, only the first 128 KiB of these samples will be used for training\.
.IP
\fB\-\-train\fR supports multithreading if \fBzstd\fR is compiled with threading support (default)\. Additional advanced parameters can be specified with \fB\-\-train\-fastcover\fR\. The legacy dictionary builder can be accessed with \fB\-\-train\-legacy\fR\. The slower cover dictionary builder can be accessed with \fB\-\-train\-cover\fR\. Default \fB\-\-train\fR is equivalent to \fB\-\-train\-fastcover=d=8,steps=4\fR\.
.TP
\fB\-o FILE\fR
Dictionary saved into \fBFILE\fR (default name: dictionary)\.
.TP
\fB\-\-maxdict=#\fR
Limit dictionary to specified size (default: 112640 bytes)\. As usual, quantities are expressed in bytes by default, and it\'s possible to employ suffixes (like \fBKB\fR or \fBMB\fR) to specify larger values\.
.TP
\fB\-#\fR
Use \fB#\fR compression level during training (optional)\. Will generate statistics more tuned for selected compression level, resulting in a \fIsmall\fR compression ratio improvement for this level\.
.TP
\fB\-B#\fR
Split input files into blocks of size # (default: no split)
.TP
\fB\-M#\fR, \fB\-\-memory=#\fR
Limit the amount of sample data loaded for training (default: 2 GB)\. Note that the default (2 GB) is also the maximum\. This parameter can be useful in situations where the training set size is not well controlled and could be potentially very large\. Since speed of the training process is directly correlated to the size of the training sample set, a smaller sample set leads to faster training\.
.IP
In situations where the training set is larger than maximum memory, the CLI will randomly select samples among the available ones, up to the maximum allowed memory budget\. This is meant to improve dictionary relevance by mitigating the potential impact of clustering, such as selecting only files from the beginning of a list sorted by modification date, or sorted by alphabetical order\. The randomization process is deterministic, so training of the same list of files with the same parameters will lead to the creation of the same dictionary\.
.TP
\fB\-\-dictID=#\fR
A dictionary ID is a locally unique ID\. The decoder will use this value to verify it is using the right dictionary\. By default, zstd will create a 4\-bytes random number ID\. It\'s possible to provide an explicit number ID instead\. It\'s up to the dictionary manager to not assign twice the same ID to 2 different dictionaries\. Note that short numbers have an advantage: an ID < 256 will only need 1 byte in the compressed frame header, and an ID < 65536 will only need 2 bytes\. This compares favorably to 4 bytes default\.
.IP
Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\e^31, so they should not be used in public\.
.TP
\fB\-\-train\-cover[=k#,d=#,steps=#,split=#,shrink[=#]]\fR
Select parameters for the default dictionary builder algorithm named cover\. If \fId\fR is not specified, then it tries \fId\fR = 6 and \fId\fR = 8\. If \fIk\fR is not specified, then it tries \fIsteps\fR values in the range [50, 2000]\. If \fIsteps\fR is not specified, then the default value of 40 is used\. If \fIsplit\fR is not specified or split <= 0, then the default value of 100 is used\. Requires that \fId\fR <= \fIk\fR\. If \fIshrink\fR flag is not used, then the default value for \fIshrinkDict\fR of 0 is used\. If \fIshrink\fR is not specified, then the default value for \fIshrinkDictMaxRegression\fR of 1 is used\.
.IP
Selects segments of size \fIk\fR with highest score to put in the dictionary\. The score of a segment is computed by the sum of the frequencies of all the subsegments of size \fId\fR\. Generally \fId\fR should be in the range [6, 8], occasionally up to 16, but the algorithm will run faster with d <= \fI8\fR\. Good values for \fIk\fR vary widely based on the input data, but a safe range is [2 * \fId\fR, 2000]\. If \fIsplit\fR is 100, all input samples are used for both training and testing to find optimal \fId\fR and \fIk\fR to build dictionary\. Supports multithreading if \fBzstd\fR is compiled with threading support\. Having \fIshrink\fR enabled takes a truncated dictionary of minimum size and doubles in size until compression ratio of the truncated dictionary is at most \fIshrinkDictMaxRegression%\fR worse than the compression ratio of the largest dictionary\.
.IP
Examples:
.IP
\fBzstd \-\-train\-cover FILEs\fR
.IP
\fBzstd \-\-train\-cover=k=50,d=8 FILEs\fR
.IP
\fBzstd \-\-train\-cover=d=8,steps=500 FILEs\fR
.IP
\fBzstd \-\-train\-cover=k=50 FILEs\fR
.IP
\fBzstd \-\-train\-cover=k=50,split=60 FILEs\fR
.IP
\fBzstd \-\-train\-cover=shrink FILEs\fR
.IP
\fBzstd \-\-train\-cover=shrink=2 FILEs\fR
.TP
\fB\-\-train\-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]\fR
Same as cover but with extra parameters \fIf\fR and \fIaccel\fR and different default value of split If \fIsplit\fR is not specified, then it tries \fIsplit\fR = 75\. If \fIf\fR is not specified, then it tries \fIf\fR = 20\. Requires that 0 < \fIf\fR < 32\. If \fIaccel\fR is not specified, then it tries \fIaccel\fR = 1\. Requires that 0 < \fIaccel\fR <= 10\. Requires that \fId\fR = 6 or \fId\fR = 8\.
.IP
\fIf\fR is log of size of array that keeps track of frequency of subsegments of size \fId\fR\. The subsegment is hashed to an index in the range [0,2^\fIf\fR \- 1]\. It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency\. Using a higher \fIf\fR reduces collision but takes longer\.
.IP
Examples:
.IP
\fBzstd \-\-train\-fastcover FILEs\fR
.IP
\fBzstd \-\-train\-fastcover=d=8,f=15,accel=2 FILEs\fR
.TP
\fB\-\-train\-legacy[=selectivity=#]\fR
Use legacy dictionary builder algorithm with the given dictionary \fIselectivity\fR (default: 9)\. The smaller the \fIselectivity\fR value, the denser the dictionary, improving its efficiency but reducing its achievable maximum size\. \fB\-\-train\-legacy=s=#\fR is also accepted\.
.IP
Examples:
.IP
\fBzstd \-\-train\-legacy FILEs\fR
.IP
\fBzstd \-\-train\-legacy=selectivity=8 FILEs\fR
.SH "BENCHMARK"
.TP
\fB\-b#\fR
benchmark file(s) using compression level #
.TP
\fB\-e#\fR
benchmark file(s) using multiple compression levels, from \fB\-b#\fR to \fB\-e#\fR (inclusive)
.TP
\fB\-i#\fR
minimum evaluation time, in seconds (default: 3s), benchmark mode only
.TP
\fB\-B#\fR, \fB\-\-block\-size=#\fR
cut file(s) into independent chunks of size # (default: no chunking)
.TP
\fB\-\-priority=rt\fR
set process priority to real\-time
.P
\fBOutput Format:\fR CompressionLevel#Filename: InputSize \-> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
.P
\fBMethodology:\fR For both compression and decompression speed, the entire input is compressed/decompressed in\-memory to measure speed\. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy\.
.SH "ADVANCED COMPRESSION OPTIONS"
### \-B#: Specify the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to non\-identical compressed frames\.
.SS "\-\-zstd[=options]:"
\fBzstd\fR provides 22 predefined regular compression levels plus the fast levels\. This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor\. (You can see the result of this translation with \fB\-\-show\-default\-cparams\fR\.) These specific parameters can be overridden with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
.TP
\fBstrategy\fR=\fIstrat\fR, \fBstrat\fR=\fIstrat\fR
Specify a strategy used by a match finder\.
.IP
There are 9 strategies numbered from 1 to 9, from fastest to strongest: 1=\fBZSTD_fast\fR, 2=\fBZSTD_dfast\fR, 3=\fBZSTD_greedy\fR, 4=\fBZSTD_lazy\fR, 5=\fBZSTD_lazy2\fR, 6=\fBZSTD_btlazy2\fR, 7=\fBZSTD_btopt\fR, 8=\fBZSTD_btultra\fR, 9=\fBZSTD_btultra2\fR\.
.TP
\fBwindowLog\fR=\fIwlog\fR, \fBwlog\fR=\fIwlog\fR
Specify the maximum number of bits for a match distance\.
.IP
The higher number of increases the chance to find a match which usually improves compression ratio\. It also increases memory requirements for the compressor and decompressor\. The minimum \fIwlog\fR is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32\-bit platforms and 31 (2 GiB) on 64\-bit platforms\.
.IP
Note: If \fBwindowLog\fR is set to larger than 27, \fB\-\-long=windowLog\fR or \fB\-\-memory=windowSize\fR needs to be passed to the decompressor\.
.TP
\fBhashLog\fR=\fIhlog\fR, \fBhlog\fR=\fIhlog\fR
Specify the maximum number of bits for a hash table\.
.IP
Bigger hash tables cause fewer collisions which usually makes compression faster, but requires more memory during compression\.
.IP
The minimum \fIhlog\fR is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB)\.
.TP
\fBchainLog\fR=\fIclog\fR, \fBclog\fR=\fIclog\fR
Specify the maximum number of bits for the secondary search structure, whose form depends on the selected \fBstrategy\fR\.
.IP
Higher numbers of bits increases the chance to find a match which usually improves compression ratio\. It also slows down compression speed and increases memory requirements for compression\. This option is ignored for the \fBZSTD_fast\fR \fBstrategy\fR, which only has the primary hash table\.
.IP
The minimum \fIclog\fR is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32\-bit platforms and 30 (1B entries / 4 GiB) on 64\-bit platforms\.
.TP
\fBsearchLog\fR=\fIslog\fR, \fBslog\fR=\fIslog\fR
Specify the maximum number of searches in a hash chain or a binary tree using logarithmic scale\.
.IP
More searches increases the chance to find a match which usually increases compression ratio but decreases compression speed\.
.IP
The minimum \fIslog\fR is 1 and the maximum is \'windowLog\' \- 1\.
.TP
\fBminMatch\fR=\fImml\fR, \fBmml\fR=\fImml\fR
Specify the minimum searched length of a match in a hash table\.
.IP
Larger search lengths usually decrease compression ratio but improve decompression speed\.
.IP
The minimum \fImml\fR is 3 and the maximum is 7\.
.TP
\fBtargetLength\fR=\fItlen\fR, \fBtlen\fR=\fItlen\fR
The impact of this field vary depending on selected strategy\.
.IP
For \fBZSTD_btopt\fR, \fBZSTD_btultra\fR and \fBZSTD_btultra2\fR, it specifies the minimum match length that causes match finder to stop searching\. A larger \fBtargetLength\fR usually improves compression ratio but decreases compression speed\.
.IP
For \fBZSTD_fast\fR, it triggers ultra\-fast mode when > 0\. The value represents the amount of data skipped between match sampling\. Impact is reversed: a larger \fBtargetLength\fR increases compression speed but decreases compression ratio\.
.IP
For all other strategies, this field has no impact\.
.IP
The minimum \fItlen\fR is 0 and the maximum is 128 KiB\.
.TP
\fBoverlapLog\fR=\fIovlog\fR, \fBovlog\fR=\fIovlog\fR
Determine \fBoverlapSize\fR, amount of data reloaded from previous job\. This parameter is only available when multithreading is enabled\. Reloading more data improves compression ratio, but decreases speed\.
.IP
The minimum \fIovlog\fR is 0, and the maximum is 9\. 1 means "no overlap", hence completely independent jobs\. 9 means "full overlap", meaning up to \fBwindowSize\fR is reloaded from previous job\. Reducing \fIovlog\fR by 1 reduces the reloaded amount by a factor 2\. For example, 8 means "windowSize/2", and 6 means "windowSize/8"\. Value 0 is special and means "default": \fIovlog\fR is automatically determined by \fBzstd\fR\. In which case, \fIovlog\fR will range from 6 to 9, depending on selected \fIstrat\fR\.
.TP
\fBldmHashLog\fR=\fIlhlog\fR, \fBlhlog\fR=\fIlhlog\fR
Specify the maximum size for a hash table used for long distance matching\.
.IP
This option is ignored unless long distance matching is enabled\.
.IP
Bigger hash tables usually improve compression ratio at the expense of more memory during compression and a decrease in compression speed\.
.IP
The minimum \fIlhlog\fR is 6 and the maximum is 30 (default: 20)\.
.TP
\fBldmMinMatch\fR=\fIlmml\fR, \fBlmml\fR=\fIlmml\fR
Specify the minimum searched length of a match for long distance matching\.
.IP
This option is ignored unless long distance matching is enabled\.
.IP
Larger/very small values usually decrease compression ratio\.
.IP
The minimum \fIlmml\fR is 4 and the maximum is 4096 (default: 64)\.
.TP
\fBldmBucketSizeLog\fR=\fIlblog\fR, \fBlblog\fR=\fIlblog\fR
Specify the size of each bucket for the hash table used for long distance matching\.
.IP
This option is ignored unless long distance matching is enabled\.
.IP
Larger bucket sizes improve collision resolution but decrease compression speed\.
.IP
The minimum \fIlblog\fR is 1 and the maximum is 8 (default: 3)\.
.TP
\fBldmHashRateLog\fR=\fIlhrlog\fR, \fBlhrlog\fR=\fIlhrlog\fR
Specify the frequency of inserting entries into the long distance matching hash table\.
.IP
This option is ignored unless long distance matching is enabled\.
.IP
Larger values will improve compression speed\. Deviating far from the default value will likely result in a decrease in compression ratio\.
.IP
The default value is \fBwlog \- lhlog\fR\.
.SS "Example"
The following parameters sets advanced compression options to something similar to predefined level 19 for files bigger than 256 KB:
.P
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.SH "SEE ALSO"
\fBzstdgrep\fR(1), \fBzstdless\fR(1), \fBgzip\fR(1), \fBxz\fR(1)
.P
The \fIzstandard\fR format is specified in Y\. Collet, "Zstandard Compression and the \'application/zstd\' Media Type", https://www\.ietf\.org/rfc/rfc8878\.txt, Internet RFC 8878 (February 2021)\.
.SH "BUGS"
Report bugs at: https://github\.com/facebook/zstd/issues
.SH "AUTHOR"
Yann Collet

673
third_party/zstd/programs/zstd.1.md vendored
View file

@ -1,673 +0,0 @@
// clang-format off
zstd(1) -- zstd, zstdmt, unzstd, zstdcat - Compress or decompress .zst files
============================================================================
SYNOPSIS
--------
`zstd` [<OPTIONS>] [-|<INPUT-FILE>] [-o <OUTPUT-FILE>]
`zstdmt` is equivalent to `zstd -T0`
`unzstd` is equivalent to `zstd -d`
`zstdcat` is equivalent to `zstd -dcf`
DESCRIPTION
-----------
`zstd` is a fast lossless compression algorithm and data compression tool,
with command line syntax similar to `gzip`(1) and `xz`(1).
It is based on the **LZ77** family, with further FSE & huff0 entropy stages.
`zstd` offers highly configurable compression speed,
from fast modes at > 200 MB/s per core,
to strong modes with excellent compression ratios.
It also features a very fast decoder, with speeds > 500 MB/s per core.
`zstd` command line syntax is generally similar to gzip,
but features the following differences:
- Source files are preserved by default.
It's possible to remove them automatically by using the `--rm` command.
- When compressing a single file, `zstd` displays progress notifications
and result summary by default.
Use `-q` to turn them off.
- `zstd` displays a short help page when command line is an error.
Use `-q` to turn it off.
- `zstd` does not accept input from console,
though it does accept `stdin` when it's not the console.
- `zstd` does not store the input's filename or attributes, only its contents.
`zstd` processes each _file_ according to the selected operation mode.
If no _files_ are given or _file_ is `-`, `zstd` reads from standard input
and writes the processed data to standard output.
`zstd` will refuse to write compressed data to standard output
if it is a terminal: it will display an error message and skip the file.
Similarly, `zstd` will refuse to read compressed data from standard input
if it is a terminal.
Unless `--stdout` or `-o` is specified, _files_ are written to a new file
whose name is derived from the source _file_ name:
* When compressing, the suffix `.zst` is appended to the source filename to
get the target filename.
* When decompressing, the `.zst` suffix is removed from the source filename to
get the target filename
### Concatenation with .zst Files
It is possible to concatenate multiple `.zst` files. `zstd` will decompress
such agglomerated file as if it was a single `.zst` file.
OPTIONS
-------
### Integer Suffixes and Special Values
In most places where an integer argument is expected,
an optional suffix is supported to easily indicate large integers.
There must be no space between the integer and the suffix.
* `KiB`:
Multiply the integer by 1,024 (2\^10).
`Ki`, `K`, and `KB` are accepted as synonyms for `KiB`.
* `MiB`:
Multiply the integer by 1,048,576 (2\^20).
`Mi`, `M`, and `MB` are accepted as synonyms for `MiB`.
### Operation Mode
If multiple operation mode options are given,
the last one takes effect.
* `-z`, `--compress`:
Compress.
This is the default operation mode when no operation mode option is specified
and no other operation mode is implied from the command name
(for example, `unzstd` implies `--decompress`).
* `-d`, `--decompress`, `--uncompress`:
Decompress.
* `-t`, `--test`:
Test the integrity of compressed _files_.
This option is equivalent to `--decompress --stdout > /dev/null`,
decompressed data is discarded and checksummed for errors.
No files are created or removed.
* `-b#`:
Benchmark file(s) using compression level _#_.
See _BENCHMARK_ below for a description of this operation.
* `--train FILES`:
Use _FILES_ as a training set to create a dictionary.
The training set should contain a lot of small files (> 100).
See _DICTIONARY BUILDER_ below for a description of this operation.
* `-l`, `--list`:
Display information related to a zstd compressed file, such as size, ratio, and checksum.
Some of these fields may not be available.
This command's output can be augmented with the `-v` modifier.
### Operation Modifiers
* `-#`:
selects `#` compression level \[1-19\] (default: 3)
* `--ultra`:
unlocks high compression levels 20+ (maximum 22), using a lot more memory.
Note that decompression will also require more memory when using these levels.
* `--fast[=#]`:
switch to ultra-fast compression levels.
If `=#` is not present, it defaults to `1`.
The higher the value, the faster the compression speed,
at the cost of some compression ratio.
This setting overwrites compression level if one was set previously.
Similarly, if a compression level is set after `--fast`, it overrides it.
* `-T#`, `--threads=#`:
Compress using `#` working threads (default: 1).
If `#` is 0, attempt to detect and use the number of physical CPU cores.
In all cases, the nb of threads is capped to `ZSTDMT_NBWORKERS_MAX`,
which is either 64 in 32-bit mode, or 256 for 64-bit environments.
This modifier does nothing if `zstd` is compiled without multithread support.
* `--single-thread`:
Use a single thread for both I/O and compression.
As compression is serialized with I/O, this can be slightly slower.
Single-thread mode features significantly lower memory usage,
which can be useful for systems with limited amount of memory, such as 32-bit systems.
Note 1: this mode is the only available one when multithread support is disabled.
Note 2: this mode is different from `-T1`, which spawns 1 compression thread in parallel with I/O.
Final compressed result is also slightly different from `-T1`.
* `--auto-threads={physical,logical} (default: physical)`:
When using a default amount of threads via `-T0`, choose the default based on the number
of detected physical or logical cores.
* `--adapt[=min=#,max=#]`:
`zstd` will dynamically adapt compression level to perceived I/O conditions.
Compression level adaptation can be observed live by using command `-v`.
Adaptation can be constrained between supplied `min` and `max` levels.
The feature works when combined with multi-threading and `--long` mode.
It does not work with `--single-thread`.
It sets window size to 8 MiB by default (can be changed manually, see `wlog`).
Due to the chaotic nature of dynamic adaptation, compressed result is not reproducible.
_Note_: at the time of this writing, `--adapt` can remain stuck at low speed
when combined with multiple worker threads (>=2).
* `--long[=#]`:
enables long distance matching with `#` `windowLog`, if `#` is not
present it defaults to `27`.
This increases the window size (`windowLog`) and memory usage for both the
compressor and decompressor.
This setting is designed to improve the compression ratio for files with
long matches at a large distance.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
* `-D DICT`:
use `DICT` as Dictionary to compress or decompress FILE(s)
* `--patch-from FILE`:
Specify the file to be used as a reference point for zstd's diff engine.
This is effectively dictionary compression with some convenient parameter
selection, namely that _windowSize_ > _srcSize_.
Note: cannot use both this and `-D` together.
Note: `--long` mode will be automatically activated if _chainLog_ < _fileLog_
(_fileLog_ being the _windowLog_ required to cover the whole file). You
can also manually force it.
Note: for all levels, you can use `--patch-from` in `--single-thread` mode
to improve compression ratio at the cost of speed.
Note: for level 19, you can get increased compression ratio at the cost
of speed by specifying `--zstd=targetLength=` to be something large
(i.e. 4096), and by setting a large `--zstd=chainLog=`.
* `--rsyncable`:
`zstd` will periodically synchronize the compression state to make the
compressed file more rsync-friendly.
There is a negligible impact to compression ratio,
and a potential impact to compression speed, perceptible at higher speeds,
for example when combining `--rsyncable` with many parallel worker threads.
This feature does not work with `--single-thread`. You probably don't want
to use it with long range mode, since it will decrease the effectiveness of
the synchronization points, but your mileage may vary.
* `-C`, `--[no-]check`:
add integrity check computed from uncompressed data (default: enabled)
* `--[no-]content-size`:
enable / disable whether or not the original size of the file is placed in
the header of the compressed file. The default option is
`--content-size` (meaning that the original size will be placed in the header).
* `--no-dictID`:
do not store dictionary ID within frame header (dictionary compression).
The decoder will have to rely on implicit knowledge about which dictionary to use,
it won't be able to check if it's correct.
* `-M#`, `--memory=#`:
Set a memory usage limit. By default, `zstd` uses 128 MiB for decompression
as the maximum amount of memory the decompressor is allowed to use, but you can
override this manually if need be in either direction (i.e. you can increase or
decrease it).
This is also used during compression when using with `--patch-from=`. In this case,
this parameter overrides that maximum size allowed for a dictionary. (128 MiB).
Additionally, this can be used to limit memory for dictionary training. This parameter
overrides the default limit of 2 GiB. zstd will load training samples up to the memory limit
and ignore the rest.
* `--stream-size=#`:
Sets the pledged source size of input coming from a stream. This value must be exact, as it
will be included in the produced frame header. Incorrect stream sizes will cause an error.
This information will be used to better optimize compression parameters, resulting in
better and potentially faster compression, especially for smaller source sizes.
* `--size-hint=#`:
When handling input from a stream, `zstd` must guess how large the source size
will be when optimizing compression parameters. If the stream size is relatively
small, this guess may be a poor one, resulting in a higher compression ratio than
expected. This feature allows for controlling the guess when needed.
Exact guesses result in better compression ratios. Overestimates result in slightly
degraded compression ratios, while underestimates may result in significant degradation.
* `-o FILE`:
save result into `FILE`.
* `-f`, `--force`:
disable input and output checks. Allows overwriting existing files, input
from console, output to stdout, operating on links, block devices, etc.
During decompression and when the output destination is stdout, pass-through
unrecognized formats as-is.
* `-c`, `--stdout`:
write to standard output (even if it is the console); keep original files unchanged.
* `--[no-]sparse`:
enable / disable sparse FS support,
to make files with many zeroes smaller on disk.
Creating sparse files may save disk space and speed up decompression by
reducing the amount of disk I/O.
default: enabled when output is into a file,
and disabled when output is stdout.
This setting overrides default and can force sparse mode over stdout.
* `--[no-]pass-through`
enable / disable passing through uncompressed files as-is. During
decompression when pass-through is enabled, unrecognized formats will be
copied as-is from the input to the output. By default, pass-through will
occur when the output destination is stdout and the force (`-f`) option is
set.
* `--rm`:
remove source file(s) after successful compression or decompression.
This command is silently ignored if output is `stdout`.
If used in combination with `-o`,
triggers a confirmation prompt (which can be silenced with `-f`), as this is a destructive operation.
* `-k`, `--keep`:
keep source file(s) after successful compression or decompression.
This is the default behavior.
* `-r`:
operate recursively on directories.
It selects all files in the named directory and all its subdirectories.
This can be useful both to reduce command line typing,
and to circumvent shell expansion limitations,
when there are a lot of files and naming breaks the maximum size of a command line.
* `--filelist FILE`
read a list of files to process as content from `FILE`.
Format is compatible with `ls` output, with one file per line.
* `--output-dir-flat DIR`:
resulting files are stored into target `DIR` directory,
instead of same directory as origin file.
Be aware that this command can introduce name collision issues,
if multiple files, from different directories, end up having the same name.
Collision resolution ensures first file with a given name will be present in `DIR`,
while in combination with `-f`, the last file will be present instead.
* `--output-dir-mirror DIR`:
similar to `--output-dir-flat`,
the output files are stored underneath target `DIR` directory,
but this option will replicate input directory hierarchy into output `DIR`.
If input directory contains "..", the files in this directory will be ignored.
If input directory is an absolute directory (i.e. "/var/tmp/abc"),
it will be stored into the "output-dir/var/tmp/abc".
If there are multiple input files or directories,
name collision resolution will follow the same rules as `--output-dir-flat`.
* `--format=FORMAT`:
compress and decompress in other formats. If compiled with
support, zstd can compress to or decompress from other compression algorithm
formats. Possibly available options are `zstd`, `gzip`, `xz`, `lzma`, and `lz4`.
If no such format is provided, `zstd` is the default.
* `-h`/`-H`, `--help`:
display help/long help and exit
* `-V`, `--version`:
display version number and exit.
Advanced: `-vV` also displays supported formats.
`-vvV` also displays POSIX support.
`-q` will only display the version number, suitable for machine reading.
* `-v`, `--verbose`:
verbose mode, display more information
* `-q`, `--quiet`:
suppress warnings, interactivity, and notifications.
specify twice to suppress errors too.
* `--no-progress`:
do not display the progress bar, but keep all other messages.
* `--show-default-cparams`:
shows the default compression parameters that will be used for a particular input file, based on the provided compression level and the input size.
If the provided file is not a regular file (e.g. a pipe), this flag will output the parameters used for inputs of unknown size.
* `--`:
All arguments after `--` are treated as files
### gzip Operation Modifiers
When invoked via a `gzip` symlink, `zstd` will support further
options that intend to mimic the `gzip` behavior:
* `-n`, `--no-name`:
do not store the original filename and timestamps when compressing
a file. This is the default behavior and hence a no-op.
* `--best`:
alias to the option `-9`.
### Environment Variables
Employing environment variables to set parameters has security implications.
Therefore, this avenue is intentionally limited.
Only `ZSTD_CLEVEL` and `ZSTD_NBTHREADS` are currently supported.
They set the compression level and number of threads to use during compression, respectively.
`ZSTD_CLEVEL` can be used to set the level between 1 and 19 (the "normal" range).
If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a warning message.
`ZSTD_CLEVEL` just replaces the default compression level (`3`).
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
`zstd` must be compiled with multithread support for this to have any effect.
They can both be overridden by corresponding command line arguments:
`-#` for compression level and `-T#` for number of compression threads.
DICTIONARY BUILDER
------------------
`zstd` offers _dictionary_ compression,
which greatly improves efficiency on small files and messages.
It's possible to train `zstd` with a set of samples,
the result of which is saved into a file called a `dictionary`.
Then, during compression and decompression, reference the same dictionary,
using command `-D dictionaryFileName`.
Compression of small files similar to the sample set will be greatly improved.
* `--train FILEs`:
Use FILEs as training set to create a dictionary.
The training set should ideally contain a lot of samples (> 100),
and weight typically 100x the target dictionary size
(for example, ~10 MB for a 100 KB dictionary).
`--train` can be combined with `-r` to indicate a directory rather than listing all the files,
which can be useful to circumvent shell expansion limits.
Since dictionary compression is mostly effective for small files,
the expectation is that the training set will only contain small files.
In the case where some samples happen to be large,
only the first 128 KiB of these samples will be used for training.
`--train` supports multithreading if `zstd` is compiled with threading support (default).
Additional advanced parameters can be specified with `--train-fastcover`.
The legacy dictionary builder can be accessed with `--train-legacy`.
The slower cover dictionary builder can be accessed with `--train-cover`.
Default `--train` is equivalent to `--train-fastcover=d=8,steps=4`.
* `-o FILE`:
Dictionary saved into `FILE` (default name: dictionary).
* `--maxdict=#`:
Limit dictionary to specified size (default: 112640 bytes).
As usual, quantities are expressed in bytes by default,
and it's possible to employ suffixes (like `KB` or `MB`)
to specify larger values.
* `-#`:
Use `#` compression level during training (optional).
Will generate statistics more tuned for selected compression level,
resulting in a _small_ compression ratio improvement for this level.
* `-B#`:
Split input files into blocks of size # (default: no split)
* `-M#`, `--memory=#`:
Limit the amount of sample data loaded for training (default: 2 GB).
Note that the default (2 GB) is also the maximum.
This parameter can be useful in situations where the training set size
is not well controlled and could be potentially very large.
Since speed of the training process is directly correlated to
the size of the training sample set,
a smaller sample set leads to faster training.
In situations where the training set is larger than maximum memory,
the CLI will randomly select samples among the available ones,
up to the maximum allowed memory budget.
This is meant to improve dictionary relevance
by mitigating the potential impact of clustering,
such as selecting only files from the beginning of a list
sorted by modification date, or sorted by alphabetical order.
The randomization process is deterministic, so
training of the same list of files with the same parameters
will lead to the creation of the same dictionary.
* `--dictID=#`:
A dictionary ID is a locally unique ID.
The decoder will use this value to verify it is using the right dictionary.
By default, zstd will create a 4-bytes random number ID.
It's possible to provide an explicit number ID instead.
It's up to the dictionary manager to not assign twice the same ID to
2 different dictionaries.
Note that short numbers have an advantage:
an ID < 256 will only need 1 byte in the compressed frame header,
and an ID < 65536 will only need 2 bytes.
This compares favorably to 4 bytes default.
Note that RFC8878 reserves IDs less than 32768 and greater than or equal to 2\^31, so they should not be used in public.
* `--train-cover[=k#,d=#,steps=#,split=#,shrink[=#]]`:
Select parameters for the default dictionary builder algorithm named cover.
If _d_ is not specified, then it tries _d_ = 6 and _d_ = 8.
If _k_ is not specified, then it tries _steps_ values in the range [50, 2000].
If _steps_ is not specified, then the default value of 40 is used.
If _split_ is not specified or split <= 0, then the default value of 100 is used.
Requires that _d_ <= _k_.
If _shrink_ flag is not used, then the default value for _shrinkDict_ of 0 is used.
If _shrink_ is not specified, then the default value for _shrinkDictMaxRegression_ of 1 is used.
Selects segments of size _k_ with highest score to put in the dictionary.
The score of a segment is computed by the sum of the frequencies of all the
subsegments of size _d_.
Generally _d_ should be in the range [6, 8], occasionally up to 16, but the
algorithm will run faster with d <= _8_.
Good values for _k_ vary widely based on the input data, but a safe range is
[2 * _d_, 2000].
If _split_ is 100, all input samples are used for both training and testing
to find optimal _d_ and _k_ to build dictionary.
Supports multithreading if `zstd` is compiled with threading support.
Having _shrink_ enabled takes a truncated dictionary of minimum size and doubles
in size until compression ratio of the truncated dictionary is at most
_shrinkDictMaxRegression%_ worse than the compression ratio of the largest dictionary.
Examples:
`zstd --train-cover FILEs`
`zstd --train-cover=k=50,d=8 FILEs`
`zstd --train-cover=d=8,steps=500 FILEs`
`zstd --train-cover=k=50 FILEs`
`zstd --train-cover=k=50,split=60 FILEs`
`zstd --train-cover=shrink FILEs`
`zstd --train-cover=shrink=2 FILEs`
* `--train-fastcover[=k#,d=#,f=#,steps=#,split=#,accel=#]`:
Same as cover but with extra parameters _f_ and _accel_ and different default value of split
If _split_ is not specified, then it tries _split_ = 75.
If _f_ is not specified, then it tries _f_ = 20.
Requires that 0 < _f_ < 32.
If _accel_ is not specified, then it tries _accel_ = 1.
Requires that 0 < _accel_ <= 10.
Requires that _d_ = 6 or _d_ = 8.
_f_ is log of size of array that keeps track of frequency of subsegments of size _d_.
The subsegment is hashed to an index in the range [0,2^_f_ - 1].
It is possible that 2 different subsegments are hashed to the same index, and they are considered as the same subsegment when computing frequency.
Using a higher _f_ reduces collision but takes longer.
Examples:
`zstd --train-fastcover FILEs`
`zstd --train-fastcover=d=8,f=15,accel=2 FILEs`
* `--train-legacy[=selectivity=#]`:
Use legacy dictionary builder algorithm with the given dictionary
_selectivity_ (default: 9).
The smaller the _selectivity_ value, the denser the dictionary,
improving its efficiency but reducing its achievable maximum size.
`--train-legacy=s=#` is also accepted.
Examples:
`zstd --train-legacy FILEs`
`zstd --train-legacy=selectivity=8 FILEs`
BENCHMARK
---------
* `-b#`:
benchmark file(s) using compression level #
* `-e#`:
benchmark file(s) using multiple compression levels, from `-b#` to `-e#` (inclusive)
* `-i#`:
minimum evaluation time, in seconds (default: 3s), benchmark mode only
* `-B#`, `--block-size=#`:
cut file(s) into independent chunks of size # (default: no chunking)
* `--priority=rt`:
set process priority to real-time
**Output Format:** CompressionLevel#Filename: InputSize -> OutputSize (CompressionRatio), CompressionSpeed, DecompressionSpeed
**Methodology:** For both compression and decompression speed, the entire input is compressed/decompressed in-memory to measure speed. A run lasts at least 1 sec, so when files are small, they are compressed/decompressed several times per run, in order to improve measurement accuracy.
ADVANCED COMPRESSION OPTIONS
----------------------------
### -B#:
Specify the size of each compression job.
This parameter is only available when multi-threading is enabled.
Each compression job is run in parallel, so this value indirectly impacts the nb of active threads.
Default job size varies depending on compression level (generally `4 * windowSize`).
`-B#` makes it possible to manually select a custom size.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to non-identical compressed frames.
### --zstd[=options]:
`zstd` provides 22 predefined regular compression levels plus the fast levels.
This compression level is translated internally into a number of specific parameters that actually control the behavior of the compressor.
(You can see the result of this translation with `--show-default-cparams`.)
These specific parameters can be overridden with advanced compression options.
The _options_ are provided as a comma-separated list.
You may specify only the options you want to change and the rest will be
taken from the selected or default compression level.
The list of available _options_:
- `strategy`=_strat_, `strat`=_strat_:
Specify a strategy used by a match finder.
There are 9 strategies numbered from 1 to 9, from fastest to strongest:
1=`ZSTD_fast`, 2=`ZSTD_dfast`, 3=`ZSTD_greedy`,
4=`ZSTD_lazy`, 5=`ZSTD_lazy2`, 6=`ZSTD_btlazy2`,
7=`ZSTD_btopt`, 8=`ZSTD_btultra`, 9=`ZSTD_btultra2`.
- `windowLog`=_wlog_, `wlog`=_wlog_:
Specify the maximum number of bits for a match distance.
The higher number of increases the chance to find a match which usually
improves compression ratio.
It also increases memory requirements for the compressor and decompressor.
The minimum _wlog_ is 10 (1 KiB) and the maximum is 30 (1 GiB) on 32-bit
platforms and 31 (2 GiB) on 64-bit platforms.
Note: If `windowLog` is set to larger than 27, `--long=windowLog` or
`--memory=windowSize` needs to be passed to the decompressor.
- `hashLog`=_hlog_, `hlog`=_hlog_:
Specify the maximum number of bits for a hash table.
Bigger hash tables cause fewer collisions which usually makes compression
faster, but requires more memory during compression.
The minimum _hlog_ is 6 (64 entries / 256 B) and the maximum is 30 (1B entries / 4 GiB).
- `chainLog`=_clog_, `clog`=_clog_:
Specify the maximum number of bits for the secondary search structure,
whose form depends on the selected `strategy`.
Higher numbers of bits increases the chance to find a match which usually
improves compression ratio.
It also slows down compression speed and increases memory requirements for
compression.
This option is ignored for the `ZSTD_fast` `strategy`, which only has the primary hash table.
The minimum _clog_ is 6 (64 entries / 256 B) and the maximum is 29 (512M entries / 2 GiB) on 32-bit platforms
and 30 (1B entries / 4 GiB) on 64-bit platforms.
- `searchLog`=_slog_, `slog`=_slog_:
Specify the maximum number of searches in a hash chain or a binary tree
using logarithmic scale.
More searches increases the chance to find a match which usually increases
compression ratio but decreases compression speed.
The minimum _slog_ is 1 and the maximum is 'windowLog' - 1.
- `minMatch`=_mml_, `mml`=_mml_:
Specify the minimum searched length of a match in a hash table.
Larger search lengths usually decrease compression ratio but improve
decompression speed.
The minimum _mml_ is 3 and the maximum is 7.
- `targetLength`=_tlen_, `tlen`=_tlen_:
The impact of this field vary depending on selected strategy.
For `ZSTD_btopt`, `ZSTD_btultra` and `ZSTD_btultra2`, it specifies
the minimum match length that causes match finder to stop searching.
A larger `targetLength` usually improves compression ratio
but decreases compression speed.
For `ZSTD_fast`, it triggers ultra-fast mode when > 0.
The value represents the amount of data skipped between match sampling.
Impact is reversed: a larger `targetLength` increases compression speed
but decreases compression ratio.
For all other strategies, this field has no impact.
The minimum _tlen_ is 0 and the maximum is 128 KiB.
- `overlapLog`=_ovlog_, `ovlog`=_ovlog_:
Determine `overlapSize`, amount of data reloaded from previous job.
This parameter is only available when multithreading is enabled.
Reloading more data improves compression ratio, but decreases speed.
The minimum _ovlog_ is 0, and the maximum is 9.
1 means "no overlap", hence completely independent jobs.
9 means "full overlap", meaning up to `windowSize` is reloaded from previous job.
Reducing _ovlog_ by 1 reduces the reloaded amount by a factor 2.
For example, 8 means "windowSize/2", and 6 means "windowSize/8".
Value 0 is special and means "default": _ovlog_ is automatically determined by `zstd`.
In which case, _ovlog_ will range from 6 to 9, depending on selected _strat_.
- `ldmHashLog`=_lhlog_, `lhlog`=_lhlog_:
Specify the maximum size for a hash table used for long distance matching.
This option is ignored unless long distance matching is enabled.
Bigger hash tables usually improve compression ratio at the expense of more
memory during compression and a decrease in compression speed.
The minimum _lhlog_ is 6 and the maximum is 30 (default: 20).
- `ldmMinMatch`=_lmml_, `lmml`=_lmml_:
Specify the minimum searched length of a match for long distance matching.
This option is ignored unless long distance matching is enabled.
Larger/very small values usually decrease compression ratio.
The minimum _lmml_ is 4 and the maximum is 4096 (default: 64).
- `ldmBucketSizeLog`=_lblog_, `lblog`=_lblog_:
Specify the size of each bucket for the hash table used for long distance
matching.
This option is ignored unless long distance matching is enabled.
Larger bucket sizes improve collision resolution but decrease compression
speed.
The minimum _lblog_ is 1 and the maximum is 8 (default: 3).
- `ldmHashRateLog`=_lhrlog_, `lhrlog`=_lhrlog_:
Specify the frequency of inserting entries into the long distance matching
hash table.
This option is ignored unless long distance matching is enabled.
Larger values will improve compression speed. Deviating far from the
default value will likely result in a decrease in compression ratio.
The default value is `wlog - lhlog`.
### Example
The following parameters sets advanced compression options to something
similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
SEE ALSO
--------
`zstdgrep`(1), `zstdless`(1), `gzip`(1), `xz`(1)
The <zstandard> format is specified in Y. Collet, "Zstandard Compression and the 'application/zstd' Media Type", https://www.ietf.org/rfc/rfc8878.txt, Internet RFC 8878 (February 2021).
BUGS
----
Report bugs at: https://github.com/facebook/zstd/issues
AUTHOR
------
Yann Collet

2
third_party/zstd/programs/zstdcli.c vendored
View file

@ -69,7 +69,7 @@
#ifndef ZSTD_NOTRACE
#include "third_party/zstd/programs/zstdcli_trace.h"
#endif
#include "third_party/zstd/lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
#include "third_party/zstd/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
#include "third_party/zstd/programs/fileio_asyncio.h"

2
third_party/zstd/programs/zstdcli_trace.c vendored
View file

@ -40,7 +40,7 @@
#include "third_party/zstd/programs/util.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/zstd.h"
#include "third_party/zstd/zstd.h"
/* We depend on the trace header to avoid duplicating the ZSTD_trace struct.
* But, we check the version so it is compatible with dynamic linking.
*/

18
third_party/zstd/programs/zstdgrep.1 vendored
View file

@ -1,18 +0,0 @@
// clang-format off
.TH "ZSTDGREP" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
.SH "SYNOPSIS"
\fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \|\.\|\.\|\.]
.SH "DESCRIPTION"
\fBzstdgrep\fR runs \fBgrep\fR(1) on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat\fR(1)\.
.P
The \fIgrep\-flags\fR and \fIpattern\fR arguments are passed on to \fBgrep\fR(1)\. If an \fB\-e\fR flag is found in the \fIgrep\-flags\fR, \fBzstdgrep\fR will not look for a \fIpattern\fR argument\.
.P
Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR(1)) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\.
.SH "EXIT STATUS"
In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\.
.SH "SEE ALSO"
\fBzstd\fR(1)
.SH "AUTHORS"
Thomas Klausner \fIwiz@NetBSD\.org\fR

31
third_party/zstd/programs/zstdgrep.1.md vendored
View file

@ -1,31 +0,0 @@
// clang-format off
zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files
============================================================================
SYNOPSIS
--------
`zstdgrep` [<grep-flags>] [--] <pattern> [<files> ...]
DESCRIPTION
-----------
`zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1).
The <grep-flags> and <pattern> arguments are passed on to `grep`(1). If an `-e` flag is found in the <grep-flags>, `zstdgrep` will not look for a <pattern> argument.
Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box,
and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches.
Note though that such alternatives may also feature some minor command line differences.
EXIT STATUS
-----------
In case of missing arguments or missing pattern, 1 will be returned, otherwise 0.
SEE ALSO
--------
`zstd`(1)
AUTHORS
-------
Thomas Klausner <wiz@NetBSD.org>

1
third_party/zstd/programs/zstdless vendored
View file

@ -1,4 +1,3 @@
// clang-format off
#!/bin/sh
zstd=${ZSTD:-zstd}

10
third_party/zstd/programs/zstdless.1 vendored
View file

@ -1,10 +0,0 @@
// clang-format off
.TH "ZSTDLESS" "1" "March 2023" "zstd 1.5.5" "User Commands"
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
.SH "SYNOPSIS"
\fBzstdless\fR [\fIflags\fR] [\fIfile\fR \|\.\|\.\|\.]
.SH "DESCRIPTION"
\fBzstdless\fR runs \fBless\fR(1) on files or stdin, if no \fIfile\fR argument is given, after decompressing them with \fBzstdcat\fR(1)\.
.SH "SEE ALSO"
\fBzstd\fR(1)

17
third_party/zstd/programs/zstdless.1.md vendored
View file

@ -1,17 +0,0 @@
// clang-format off
zstdless(1) -- view zstandard-compressed files
============================================================================
SYNOPSIS
--------
`zstdless` [<flags>] [<file> ...]
DESCRIPTION
-----------
`zstdless` runs `less`(1) on files or stdin, if no <file> argument is given, after decompressing them with `zstdcat`(1).
SEE ALSO
--------
`zstd`(1)

69
third_party/zstd/tests/.gitignore vendored
View file

@ -1,69 +0,0 @@
# local binary (Makefile)
fullbench
fullbench32
fullbench-lib
fuzzer
fuzzer32
fuzzer-dll
zbufftest
zbufftest32
zbufftest-dll
zstreamtest
zstreamtest32
zstreamtest_asan
zstreamtest_tsan
zstreamtest_ubsan
zstreamtest-dll
datagen
paramgrill
paramgrill32
roundTripCrash
longmatch
symbols
legacy
decodecorpus
pool
poolTests
invalidDictionaries
checkTag
zcat
zstdcat
tm
# test artifacts
dictionary
grillResults.txt
_*
tmp*
*.zst
*.gz
!gzip/hufts-segv.gz
result
out
*.zstd
hello*
world
# Tmp test directory
zstdtest
speedTest
versionsTest
namespaceTest
dirTest*
# fuzzer
afl
# Local script
startSpeedTest
speedTest.pid
*.bat
# Generic Object files
*.o
*.ko
# Generic Executables
*.exe
*.out
*.app

378
third_party/zstd/tests/DEPRECATED-test-zstd-speed.py vendored
View file

@ -1,378 +0,0 @@
#! /usr/bin/env python3
# THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ##########################################################################
# Limitations:
# - doesn't support filenames with spaces
# - dir1/zstd and dir2/zstd will be merged in a single results file
import argparse
import os # getloadavg
import string
import subprocess
import time # strftime
import traceback
import hashlib
import platform # system
script_version = 'v1.1.2 (2017-03-26)'
default_repo_url = 'https://github.com/facebook/zstd.git'
working_dir_name = 'speedTest'
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd
email_header = 'ZSTD_speedTest'
pid = str(os.getpid())
verbose = False
clang_version = "unknown"
gcc_version = "unknown"
args = None
def hashfile(hasher, fname, blocksize=65536):
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(blocksize), b""):
hasher.update(chunk)
return hasher.hexdigest()
def log(text):
print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
if print_command:
log("> " + command)
popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
stderr_lines = stderr_lines.decode("utf-8")
stdout_lines = stdout_lines.decode("utf-8")
if print_output:
if stdout_lines:
print(stdout_lines)
if stderr_lines:
print(stderr_lines)
if popen.returncode is not None and popen.returncode != 0:
if stderr_lines and not print_output and print_error:
print(stderr_lines)
raise RuntimeError(stdout_lines + stderr_lines)
return (stdout_lines + stderr_lines).splitlines()
execute.cwd = None
def does_command_exist(command):
try:
execute(command, verbose, False, False)
except Exception:
return False
return True
def send_email(emails, topic, text, have_mutt, have_mail):
logFileName = working_path + '/' + 'tmpEmailContent'
with open(logFileName, "w") as myfile:
myfile.writelines(text)
myfile.close()
if have_mutt:
execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
elif have_mail:
execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
else:
log("e-mail cannot be sent (mail or mutt not found)")
def send_email_with_attachments(branch, commit, last_commit, args, text, results_files,
logFileName, have_mutt, have_mail):
with open(logFileName, "w") as myfile:
myfile.writelines(text)
myfile.close()
email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
% (email_header, pid, branch, commit, last_commit,
args.lowerLimit, args.ratioLimit)
if have_mutt:
execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files
+ ' < ' + logFileName)
elif have_mail:
execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName)
else:
log("e-mail cannot be sent (mail or mutt not found)")
def git_get_branches():
execute('git fetch -p', verbose)
branches = execute('git branch -rl', verbose)
output = []
for line in branches:
if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line):
output.append(line.strip())
return output
def git_get_changes(branch, commit, last_commit):
fmt = '--format="%h: (%an) %s, %ar"'
if last_commit is None:
commits = execute('git log -n 10 %s %s' % (fmt, commit))
else:
commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
def get_last_results(resultsFileName):
if not os.path.isfile(resultsFileName):
return None, None, None, None
commit = None
csize = []
cspeed = []
dspeed = []
with open(resultsFileName, 'r') as f:
for line in f:
words = line.split()
if len(words) <= 4: # branch + commit + compilerVer + md5
commit = words[1]
csize = []
cspeed = []
dspeed = []
if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files"
csize.append(int(words[1]))
cspeed.append(float(words[3]))
dspeed.append(float(words[5]))
return commit, csize, cspeed, dspeed
def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName,
testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
sleepTime = 30
while os.getloadavg()[0] > args.maxLoadAvg:
log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds"
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
time.sleep(sleepTime)
start_load = str(os.getloadavg())
osType = platform.system()
if osType == 'Linux':
cpuSelector = "taskset --cpu-list 0"
else:
cpuSelector = ""
if args.dictionary:
result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
else:
result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True)
end_load = str(os.getloadavg())
linesExpected = args.lastCLevel + 1
if len(result) != linesExpected:
raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
with open(resultsFileName, "a") as myfile:
myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum))
myfile.write('\n'.join(result) + '\n')
myfile.close()
if (last_cspeed == None):
log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
return ""
commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
text = ""
for i in range(0, min(len(cspeed), len(last_cspeed))):
print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName))
if (cspeed[i]/last_cspeed[i] < args.lowerLimit):
text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
if (dspeed[i]/last_dspeed[i] < args.lowerLimit):
text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
if (float(last_csize[i])/csize[i] < args.ratioLimit):
text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
if text:
text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text
return text
def update_config_file(branch, commit):
last_commit = None
commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
if os.path.isfile(commitFileName):
with open(commitFileName, 'r') as infile:
last_commit = infile.read()
with open(commitFileName, 'w') as outfile:
outfile.write(commit)
return last_commit
def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName):
last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
if not args.dry_run:
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
if text:
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
return text
def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
local_branch = branch.split('/')[1]
version = local_branch.rpartition('-')[2] + '_' + commit
if not args.dry_run:
execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
'mv programs/zstd programs/zstd_clang && ' +
'make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version))
logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
text_to_send = []
results_files = ""
if args.dictionary:
dictName = args.dictionary.rpartition('/')[2]
else:
dictName = None
for filePath in testFilePaths:
fileName = filePath.rpartition('/')[2]
if dictName:
resultsFileName = working_path + "/" + dictName.replace(".", "_") + "_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
else:
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd', md5_zstd, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd32', md5_zstd32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName)
if text:
text_to_send.append(text)
results_files += resultsFileName + " "
if text_to_send:
send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('testFileNames', help='file or directory names list for speed benchmark')
parser.add_argument('emails', help='list of e-mail addresses to send warnings')
parser.add_argument('--dictionary', '-D', help='path to the dictionary')
parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="")
parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98)
parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999)
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
args = parser.parse_args()
verbose = args.verbose
# check if test files are accessible
testFileNames = args.testFileNames.split()
testFilePaths = []
for fileName in testFileNames:
fileName = os.path.expanduser(fileName)
if os.path.isfile(fileName) or os.path.isdir(fileName):
testFilePaths.append(os.path.abspath(fileName))
else:
log("ERROR: File/directory not found: " + fileName)
exit(1)
# check if dictionary is accessible
if args.dictionary:
args.dictionary = os.path.abspath(os.path.expanduser(args.dictionary))
if not os.path.isfile(args.dictionary):
log("ERROR: Dictionary not found: " + args.dictionary)
exit(1)
# check availability of e-mail senders
have_mutt = does_command_exist("mutt -h")
have_mail = does_command_exist("mail -V")
if not have_mutt and not have_mail:
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
exit(1)
clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
gcc_version = execute("gcc -dumpversion", verbose)[0];
if verbose:
print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
print("working_path=%s" % working_path)
print("clone_path=%s" % clone_path)
print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
print("message=%s" % args.message)
print("emails=%s" % args.emails)
print("dictionary=%s" % args.dictionary)
print("maxLoadAvg=%s" % args.maxLoadAvg)
print("lowerLimit=%s" % args.lowerLimit)
print("ratioLimit=%s" % args.ratioLimit)
print("lastCLevel=%s" % args.lastCLevel)
print("sleepTime=%s" % args.sleepTime)
print("timeout=%s" % args.timeout)
print("dry_run=%s" % args.dry_run)
print("verbose=%s" % args.verbose)
print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
# clone ZSTD repo if needed
if not os.path.isdir(working_path):
os.mkdir(working_path)
if not os.path.isdir(clone_path):
execute.cwd = working_path
execute('git clone ' + args.repoURL)
if not os.path.isdir(clone_path):
log("ERROR: ZSTD clone not found: " + clone_path)
exit(1)
execute.cwd = clone_path
# check if speedTest.pid already exists
pidfile = "./speedTest.pid"
if os.path.isfile(pidfile):
log("ERROR: %s already exists, exiting" % pidfile)
exit(1)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
with open(pidfile, 'w') as the_file:
the_file.write(pid)
branch = ""
commit = ""
first_time = True
while True:
try:
if first_time:
first_time = False
else:
time.sleep(args.sleepTime)
loadavg = os.getloadavg()[0]
if (loadavg <= args.maxLoadAvg):
branches = git_get_branches()
for branch in branches:
commit = execute('git show -s --format=%h ' + branch, verbose)[0]
last_commit = update_config_file(branch, commit)
if commit == last_commit:
log("skipping branch %s: head %s already processed" % (branch, commit))
else:
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
execute('git checkout -- . && git checkout ' + branch)
print(git_get_changes(branch, commit, last_commit))
test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
else:
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
if verbose:
log("sleep for %s seconds" % args.sleepTime)
except Exception as e:
stack = traceback.format_exc()
email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
print(stack)
except KeyboardInterrupt:
os.unlink(pidfile)
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
exit(0)

468
third_party/zstd/tests/Makefile vendored
View file

@ -1,468 +0,0 @@
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
# datagen : Synthetic and parametrable data generator, for tests
# fullbench : Precisely measure speed for each zstd inner functions
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
# fuzzer : Test tool, to check zstd integrity on target platform
# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode
# paramgrill : parameter tester for zstd
# test-zstd-speed.py : script for testing zstd speed difference between commits
# versionsTest : compatibility test between zstd versions stored on Github (v0.1+)
# zstreamtest : Fuzzer test tool for zstd streaming API
# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode
# ##########################################################################
LIBZSTD = ../lib
ZSTD_LEGACY_SUPPORT ?= 0
DEBUGLEVEL ?= 2
export DEBUGLEVEL # transmit value to sub-makefiles
include $(LIBZSTD)/libzstd.mk
ZSTDDIR = $(LIBZSTD)
PRGDIR = ../programs
PYTHON ?= python3
TESTARTEFACT := versionsTest
DEBUGFLAGS += -g -Wno-c++-compat
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
ZSTDCOMMON_FILES := $(sort $(ZSTD_COMMON_FILES))
ZSTDCOMP_FILES := $(sort $(ZSTD_COMPRESS_FILES))
ZSTDDECOMP_FILES := $(sort $(ZSTD_DECOMPRESS_FILES))
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZDICT_FILES := $(sort $(ZSTD_DICTBUILDER_FILES))
ZSTD_F1 := $(sort $(wildcard $(ZSTD_FILES)))
ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1))
ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1))
ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2))
ZSTD_OBJ4 := $(ZSTD_OBJ3:.c=.o)
ZSTD_OBJECTS := $(ZSTD_OBJ4:.S=.o)
ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1))
ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1))
ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2))
ZSTDMT_OBJ4 := $(ZSTDMT_OBJ3:.c=.o)
ZSTDMT_OBJECTS := $(ZSTDMT_OBJ4:.S=.o)
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
EXT =.exe
MULTITHREAD_CPP = -DZSTD_MULTITHREAD
MULTITHREAD_LD =
else
EXT =
MULTITHREAD_CPP = -DZSTD_MULTITHREAD
MULTITHREAD_LD = -pthread
endif
MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
VOID = /dev/null
ZSTREAM_TESTTIME ?= -T90s
FUZZERTEST ?= -T200s
ZSTDRTTEST = --test-large-data
DECODECORPUS_TESTTIME ?= -T30
.PHONY: default
default: fullbench
.PHONY: all
all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash poolTests
.PHONY: all32
all32: fullbench32 fuzzer32 zstreamtest32
.PHONY: allnothread
allnothread: MULTITHREAD_CPP=
allnothread: MULTITHREAD_LD=
allnothread: fullbench fuzzer paramgrill datagen decodecorpus
# note : broken : requires symbols unavailable from dynamic library
.PHONY: dll
dll: fuzzer-dll zstreamtest-dll
.PHONY: zstd zstd32 zstd-nolegacy # only external makefile knows how to build or update them
zstd zstd32 zstd-nolegacy zstd-dll:
$(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)"
.PHONY: libzstd
libzstd :
$(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS+="$(DEBUGFLAGS)"
%-dll : libzstd
%-dll : LDFLAGS += -L$(ZSTDDIR) -lzstd
$(ZSTDDIR)/libzstd.a :
$(MAKE) -C $(ZSTDDIR) libzstd.a
zstdm_%.o : $(ZSTDDIR)/common/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdc_%.o : $(ZSTDDIR)/compress/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdd_%.o : $(ZSTDDIR)/decompress/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdd_%.o : $(ZSTDDIR)/decompress/%.S
$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP)
zstdmt_m_%.o : $(ZSTDDIR)/common/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c
$(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@
zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.S
$(CC) -c $(CPPFLAGS) $(ASFLAGS) $< -o $@
FULLBENCHS := fullbench fullbench32
CLEAN += $(FULLBENCHS)
fullbench32: CPPFLAGS += -m32
$(FULLBENCHS) : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
$(FULLBENCHS) : LDFLAGS += $(MULTITHREAD_LD)
$(FULLBENCHS) : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements
$(FULLBENCHS) : $(ZSTD_FILES)
$(FULLBENCHS) : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += fullbench-lib
fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_
fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(ZSTDDIR)/libzstd.a fullbench.c
$(LINK.c) $^ -o $@$(EXT)
# note : broken : requires symbols unavailable from dynamic library
fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c
# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
$(LINK.c) $^ $(LDLIBS) -o $@$(EXT)
CLEAN += fuzzer fuzzer32
fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) -Wno-deprecated-declarations
fuzzer : LDFLAGS += $(MULTITHREAD_LD)
fuzzer : $(ZSTDMT_OBJECTS)
fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
fuzzer32 : CFLAGS += -m32 $(MULTITHREAD)
fuzzer32 : $(ZSTD_FILES)
$(LINK.c) $^ -o $@$(EXT)
# note : broken : requires symbols unavailable from dynamic library
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
CLEAN += zstreamtest zstreamtest32
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c external_matchfinder.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
zstreamtest32 : CFLAGS += -m32
zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP)
zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD)
zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES)
zstreamtest32 : $(ZSTREAMFILES)
zstreamtest zstreamtest32 :
$(LINK.c) $^ -o $@$(EXT)
CLEAN += zstreamtest_asan
zstreamtest_asan : CFLAGS += -fsanitize=address
zstreamtest_asan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
CLEAN += zstreamtest_tsan
zstreamtest_tsan : CFLAGS += -fsanitize=thread
zstreamtest_tsan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
CLEAN += zstreamtest_ubsan
zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
zstreamtest_ubsan : $(ZSTREAMFILES)
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
# note : broken : requires symbols unavailable from dynamic library
zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll
zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
CLEAN += paramgrill
paramgrill : DEBUGFLAGS = # turn off debug for speed measurements
paramgrill : LDLIBS += -lm
paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c
CLEAN += datagen
datagen : $(PRGDIR)/datagen.c datagencli.c
$(LINK.c) $^ -o $@$(EXT)
CLEAN += roundTripCrash
roundTripCrash: CFLAGS += $(MULTITHREAD)
roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c
CLEAN += longmatch
longmatch : $(ZSTD_OBJECTS) longmatch.c
CLEAN += bigdict
bigdict: CFLAGS += $(MULTITHREAD)
bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c
CLEAN += invalidDictionaries
invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c
CLEAN += legacy
legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -UZSTD_LEGACY_SUPPORT -DZSTD_LEGACY_SUPPORT=4
legacy : $(ZSTD_FILES) $(sort $(wildcard $(ZSTDDIR)/legacy/*.c)) legacy.c
CLEAN += decodecorpus
decodecorpus : LDLIBS += -lm
decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c
CLEAN += poolTests
poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
.PHONY: versionsTest
versionsTest: clean
$(PYTHON) test-zstd-versions.py
.PHONY: automated_benchmarking
automated_benchmarking: clean
$(PYTHON) automated_benchmarking.py
# make checkTag : check that release tag corresponds to release version
CLEAN += checkTag
checkTag.o : $(ZSTDDIR)/zstd.h
.PHONY: clean
clean:
$(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C $(PRGDIR) clean
$(RM) -fR $(TESTARTEFACT)
$(RM) -rf tmp* # some test directories are named tmp*
$(RM) $(CLEAN) core *.o *.tmp result* *.gcda dictionary *.zst \
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench-dll$(EXT) fuzzer-dll$(EXT) zstreamtest-dll$(EXT)
@echo Cleaning completed
#----------------------------------------------------------------------------------
# valgrind tests validated only for some posix platforms
#----------------------------------------------------------------------------------
UNAME := $(shell uname)
ifneq (,$(filter $(UNAME),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS AIX))
HOST_OS = POSIX
.PHONY: test-valgrind
test-valgrind: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1
test-valgrind: zstd datagen fuzzer fullbench
@echo "\n ---- valgrind tests : memory analyzer ----"
$(VALGRIND) ./datagen -g50M > $(VOID)
$(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi
./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID)
./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp
$(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID)
./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID)
$(RM) tmp
$(VALGRIND) ./fuzzer -T1mn -t1
$(VALGRIND) ./fullbench -i1
endif
ifneq (,$(filter MINGW% MSYS%,$(UNAME)))
HOST_OS = MSYS
endif
#-----------------------------------------------------------------------------
# make tests validated only for below targets
#-----------------------------------------------------------------------------
ifneq (,$(filter $(HOST_OS),MSYS POSIX))
DIFF:=diff
ifneq (,$(filter $(UNAME),SunOS))
DIFF:=gdiff
endif
.PHONY: list
list:
@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs
.PHONY: shortest
shortest: ZSTDRTTEST= # remove long tests
shortest: test-zstd
.PHONY: check
check: shortest
.PHONY: fuzztest
fuzztest: test-fuzzer test-zstream test-decodecorpus
.PHONY: test
test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
ifeq ($(QEMU_SYS),)
test: test-pool
endif
.PHONY: test32
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
.PHONY: test-all
test-all: test test32 test-decodecorpus-cli
.PHONY: test-zstd test-zstd32 test-zstd-nolegacy
test-zstd: ZSTD = $(PRGDIR)/zstd
test-zstd: zstd
.PHONY: test-zstd-dll
test-zstd-dll: ZSTD = $(PRGDIR)/zstd
test-zstd-dll: zstd-dll
test-zstd32: ZSTD = $(PRGDIR)/zstd32
test-zstd32: zstd32
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
test-zstd-nolegacy: zstd-nolegacy
test-zstd test-zstd32 test-zstd-nolegacy test-zstd-dll: datagen
file $(ZSTD)
EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
.PHONY: test-cli-tests
test-cli-tests: ZSTD = $(PRGDIR)/zstd
test-cli-tests: zstd datagen
file $(ZSTD)
./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
.PHONY: test-fullbench
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
$(QEMU_SYS) ./fullbench -i1 -P0
.PHONY: test-fullbench32
test-fullbench32: fullbench32 datagen
$(QEMU_SYS) ./fullbench32 -i1
$(QEMU_SYS) ./fullbench32 -i1 -P0
.PHONY: test-fuzzer
test-fuzzer: fuzzer
$(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS)
# Note : this test presumes `fuzzer` will be built
.PHONY: test-fuzzer-stackmode
test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0
test-fuzzer-stackmode: test-fuzzer
.PHONY: test-fuzzer32
test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
.PHONY: test-zstream
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
test-bigdict: bigdict
$(QEMU_SYS) ./bigdict
test-invalidDictionaries: invalidDictionaries
$(QEMU_SYS) ./invalidDictionaries
test-legacy: legacy
$(QEMU_SYS) ./legacy
test-decodecorpus: decodecorpus
$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
test-decodecorpus-cli: decodecorpus
@echo "\n ---- decodecorpus basic cli tests ----"
@mkdir testdir
./decodecorpus -n5 -otestdir -ptestdir
@cd testdir && \
$(ZSTD) -d z000000.zst -o tmp0 && \
$(ZSTD) -d z000001.zst -o tmp1 && \
$(ZSTD) -d z000002.zst -o tmp2 && \
$(ZSTD) -d z000003.zst -o tmp3 && \
$(ZSTD) -d z000004.zst -o tmp4 && \
diff z000000 tmp0 && \
diff z000001 tmp1 && \
diff z000002 tmp2 && \
diff z000003 tmp3 && \
diff z000004 tmp4 && \
rm ./* && \
cd ..
@echo "\n ---- decodecorpus dictionary cli tests ----"
./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB
@cd testdir && \
$(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \
$(ZSTD) -d z000001.zst -D dictionary -o tmp1 && \
$(ZSTD) -d z000002.zst -D dictionary -o tmp2 && \
$(ZSTD) -d z000003.zst -D dictionary -o tmp3 && \
$(ZSTD) -d z000004.zst -D dictionary -o tmp4 && \
diff z000000 tmp0 && \
diff z000001 tmp1 && \
diff z000002 tmp2 && \
diff z000003 tmp3 && \
diff z000004 tmp4 && \
cd ..
@rm -rf testdir
test-pool: poolTests
$(QEMU_SYS) ./poolTests
test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd
test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4
test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4
test-lz4: zstd decodecorpus datagen
[ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4
[ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4
./decodecorpus -ptmp
# lz4 -> zstd
lz4 < tmp | \
$(ZSTD) -d | \
cmp - tmp
lz4 < tmp | \
$(ZSTD_UNLZ4) | \
cmp - tmp
# zstd -> lz4
$(ZSTD) --format=lz4 < tmp | \
lz4 -d | \
cmp - tmp
$(ZSTD_LZ4) < tmp | \
lz4 -d | \
cmp - tmp
# zstd -> zstd
$(ZSTD) --format=lz4 < tmp | \
$(ZSTD) -d | \
cmp - tmp
# zstd -> zstd
$(ZSTD) < tmp | \
$(ZSTD) -d | \
cmp - tmp
./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null
rm tmp lz4 unlz4
endif

184
third_party/zstd/tests/README.md vendored
View file

@ -1,184 +0,0 @@
Programs and scripts for automated testing of Zstandard
=======================================================
This directory contains the following programs and scripts:
- `datagen` : Synthetic and parametrable data generator, for tests
- `fullbench` : Precisely measure speed for each zstd inner functions
- `fuzzer` : Test tool, to check zstd integrity on target platform
- `paramgrill` : parameter tester for zstd
- `test-zstd-speed.py` : script for testing zstd speed difference between commits
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
- `zstreamtest` : Fuzzer test tool for zstd streaming API
- `legacy` : Test tool to test decoding of legacy zstd frames
- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
This script creates `versionsTest` directory to which zstd repository is cloned.
Then all tagged (released) versions of zstd are compiled.
In the following step interoperability between zstd versions is checked.
#### `automated-benchmarking.py` - script for benchmarking zstd prs to dev
This script benchmarks facebook:dev and changes from pull requests made to zstd and compares
them against facebook:dev to detect regressions. This script currently runs on a dedicated
desktop machine for every pull request that is made to the zstd repo but can also
be run on any machine via the command line interface.
There are three modes of usage for this script: fastmode will just run a minimal single
build comparison (between facebook:dev and facebook:release), onetime will pull all the current
pull requests from the zstd repo and compare facebook:dev to all of them once, continuous
will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev.
```
Example usage: python automated_benchmarking.py
```
```
usage: automated_benchmarking.py [-h] [--directory DIRECTORY]
[--levels LEVELS] [--iterations ITERATIONS]
[--emails EMAILS] [--frequency FREQUENCY]
[--mode MODE] [--dict DICT]
optional arguments:
-h, --help show this help message and exit
--directory DIRECTORY
directory with files to benchmark
--levels LEVELS levels to test e.g. ('1,2,3')
--iterations ITERATIONS
number of benchmark iterations to run
--emails EMAILS email addresses of people who will be alerted upon
regression. Only for continuous mode
--frequency FREQUENCY
specifies the number of seconds to wait before each
successive check for new PRs in continuous mode
--mode MODE 'fastmode', 'onetime', 'current', or 'continuous' (see
README.md for details)
--dict DICT filename of dictionary to use (when set, this
dictionary will be used to compress the files provided
inside --directory)
```
#### `test-zstd-speed.py` - script for testing zstd speed difference between commits
DEPRECATED
This script creates `speedTest` directory to which zstd repository is cloned.
Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter).
After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits.
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
The results of the speed benchmark are compared to the previous results.
If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter).
Additional remarks:
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
- Using the script with virtual machines can lead to large variations of speed results
- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75)
- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning
The example usage with two test files, one e-mail address, and with an additional message:
```
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
```
To run the script in background please use:
```
nohup ./test-zstd-speed.py testFileNames emails &
```
The full list of parameters:
```
positional arguments:
testFileNames file names list for speed benchmark
emails list of e-mail addresses to send warnings
optional arguments:
-h, --help show this help message and exit
--message MESSAGE attach an additional message to e-mail
--lowerLimit LOWERLIMIT
send email if speed is lower than given limit
--maxLoadAvg MAXLOADAVG
maximum load average to start testing
--lastCLevel LASTCLEVEL
last compression level for testing
--sleepTime SLEEPTIME
frequency of repository checking in seconds
```
#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
Command line tool to generate test .zst files.
This tool will generate .zst files with checksums,
as well as optionally output the corresponding correct uncompressed data for
extra verification.
Example:
```
./decodecorpus -ptestfiles -otestfiles -n10000 -s5
```
will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
with the zstd checksum field set,
as well as the 10,000 original files for more detailed comparison of decompression results.
```
./decodecorpus -t -T1mn
```
will choose a random seed, and for 1 minute,
generate random test frames and ensure that the
zstd library correctly decompresses them in both simple and streaming modes.
#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
Full list of arguments
```
-T# : set level 1 speed objective
-B# : cut input into blocks of size # (default : single block)
-S : benchmarks a single run (example command: -Sl3w10h12)
w# - windowLog
h# - hashLog
c# - chainLog
s# - searchLog
l# - minMatch
t# - targetLength
S# - strategy
L# - level
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
(Added forceAttachDictionary / fadt)
When invoked with --optimize, this represents the sample to exceed.
--optimize= : find parameters to maximize compression ratio given parameters
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
cSpeed= : Minimum compression speed
dSpeed= : Minimum decompression speed
cMem= : Maximum compression memory
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
(Lower value will begin with stronger strategies) (Default 90%)
speedRatio= (accepts decimals)
: determines value of gains in speed vs gains in ratio
when determining overall winner (default 5 (1% ratio = 5% speed)).
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
Higher values will make optimizer run longer, more chances to find better solution.
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
Setting memLog = 0 turns off memoization
--display= : specify which parameters are included in the output
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
(Default: display all params available)
-P# : generated sample compressibility (when no file is provided)
-t# : Caps runtime of operation in seconds (default: 99999 seconds (about 27 hours))
-v : Prints Benchmarking output
-D : Next argument dictionary file
-s : Benchmark all files separately
-q : Quiet, repeat for more quiet
-q Prints parameters + results whenever a new best is found
-qq Only prints parameters whenever a new best is found, prints final parameters + results
-qqq Only print final parameters + results
-qqqq Only prints final parameter set in the form --zstd=
-v : Verbose, cancels quiet, repeat for more volume
-v Prints all candidate parameters and results
```
Any inputs afterwards are treated as files to benchmark.

326
third_party/zstd/tests/automated_benchmarking.py vendored
View file

@ -1,326 +0,0 @@
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ##########################################################################
import argparse
import glob
import json
import os
import time
import pickle as pk
import subprocess
import urllib.request
GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open"
GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd"
RELEASE_BUILD = {"user": "facebook", "branch": "dev", "hash": None}
# check to see if there are any new PRs every minute
DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60
PREVIOUS_PRS_FILENAME = "prev_prs.pk"
# Not sure what the threshold for triggering alarms should be
# 1% regression sounds like a little too sensitive but the desktop
# that I'm running it on is pretty stable so I think this is fine
CSPEED_REGRESSION_TOLERANCE = 0.01
DSPEED_REGRESSION_TOLERANCE = 0.01
def get_new_open_pr_builds(prev_state=True):
prev_prs = None
if os.path.exists(PREVIOUS_PRS_FILENAME):
with open(PREVIOUS_PRS_FILENAME, "rb") as f:
prev_prs = pk.load(f)
data = json.loads(urllib.request.urlopen(GITHUB_API_PR_URL).read().decode("utf-8"))
prs = {
d["url"]: {
"user": d["user"]["login"],
"branch": d["head"]["ref"],
"hash": d["head"]["sha"].strip(),
}
for d in data
}
with open(PREVIOUS_PRS_FILENAME, "wb") as f:
pk.dump(prs, f)
if not prev_state or prev_prs == None:
return list(prs.values())
return [pr for url, pr in prs.items() if url not in prev_prs or prev_prs[url] != pr]
def get_latest_hashes():
tmp = subprocess.run(["git", "log", "-1"], stdout=subprocess.PIPE).stdout.decode(
"utf-8"
)
sha1 = tmp.split("\n")[0].split(" ")[1]
tmp = subprocess.run(
["git", "show", "{}^1".format(sha1)], stdout=subprocess.PIPE
).stdout.decode("utf-8")
sha2 = tmp.split("\n")[0].split(" ")[1]
tmp = subprocess.run(
["git", "show", "{}^2".format(sha1)], stdout=subprocess.PIPE
).stdout.decode("utf-8")
sha3 = "" if len(tmp) == 0 else tmp.split("\n")[0].split(" ")[1]
return [sha1.strip(), sha2.strip(), sha3.strip()]
def get_builds_for_latest_hash():
hashes = get_latest_hashes()
for b in get_new_open_pr_builds(False):
if b["hash"] in hashes:
return [b]
return []
def clone_and_build(build):
if build["user"] != None:
github_url = GITHUB_URL_TEMPLATE.format(build["user"])
os.system(
"""
rm -rf zstd-{user}-{sha} &&
git clone {github_url} zstd-{user}-{sha} &&
cd zstd-{user}-{sha} &&
{checkout_command}
make -j &&
cd ../
""".format(
user=build["user"],
github_url=github_url,
sha=build["hash"],
checkout_command="git checkout {} &&".format(build["hash"])
if build["hash"] != None
else "",
)
)
return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"])
else:
os.system("cd ../ && make -j && cd tests")
return "../zstd"
def parse_benchmark_output(output):
idx = [i for i, d in enumerate(output) if d == "MB/s"]
return [float(output[idx[0] - 1]), float(output[idx[1] - 1])]
def benchmark_single(executable, level, filename):
return parse_benchmark_output((
subprocess.run(
[executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
.stdout.decode("utf-8")
.split(" ")
))
def benchmark_n(executable, level, filename, n):
speeds_arr = [benchmark_single(executable, level, filename) for _ in range(n)]
cspeed, dspeed = max(b[0] for b in speeds_arr), max(b[1] for b in speeds_arr)
print(
"Bench (executable={} level={} filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
os.path.basename(executable),
level,
os.path.basename(filename),
n,
cspeed,
dspeed,
)
)
return (cspeed, dspeed)
def benchmark(build, filenames, levels, iterations):
executable = clone_and_build(build)
return [
[benchmark_n(executable, l, f, iterations) for f in filenames] for l in levels
]
def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations):
cspeeds, dspeeds = [], []
for _ in range(iterations):
output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ")
cspeed, dspeed = parse_benchmark_output(output)
cspeeds.append(cspeed)
dspeeds.append(dspeed)
max_cspeed, max_dspeed = max(cspeeds), max(dspeeds)
print(
"Bench (executable={} level={} filenames_directory={}, dictionary_filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
os.path.basename(executable),
level,
os.path.basename(filenames_directory),
os.path.basename(dictionary_filename),
iterations,
max_cspeed,
max_dspeed,
)
)
return (max_cspeed, max_dspeed)
def benchmark_dictionary(build, filenames_directory, dictionary_filename, levels, iterations):
executable = clone_and_build(build)
return [benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, l, iterations) for l in levels]
def parse_regressions_and_labels(old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build):
cspeed_reg = (old_cspeed - new_cspeed) / old_cspeed
dspeed_reg = (old_dspeed - new_dspeed) / old_dspeed
baseline_label = "{}:{} ({})".format(
baseline_build["user"], baseline_build["branch"], baseline_build["hash"]
)
test_label = "{}:{} ({})".format(
test_build["user"], test_build["branch"], test_build["hash"]
)
return cspeed_reg, dspeed_reg, baseline_label, test_label
def get_regressions(baseline_build, test_build, iterations, filenames, levels):
old = benchmark(baseline_build, filenames, levels, iterations)
new = benchmark(test_build, filenames, levels, iterations)
regressions = []
for j, level in enumerate(levels):
for k, filename in enumerate(filenames):
old_cspeed, old_dspeed = old[j][k]
new_cspeed, new_dspeed = new[j][k]
cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels(
old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
)
if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
regressions.append(
"[COMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
level,
filename,
baseline_label,
test_label,
old_cspeed,
new_cspeed,
cspeed_reg * 100.0,
)
)
if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
regressions.append(
"[DECOMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
level,
filename,
baseline_label,
test_label,
old_dspeed,
new_dspeed,
dspeed_reg * 100.0,
)
)
return regressions
def get_regressions_dictionary(baseline_build, test_build, filenames_directory, dictionary_filename, levels, iterations):
old = benchmark_dictionary(baseline_build, filenames_directory, dictionary_filename, levels, iterations)
new = benchmark_dictionary(test_build, filenames_directory, dictionary_filename, levels, iterations)
regressions = []
for j, level in enumerate(levels):
old_cspeed, old_dspeed = old[j]
new_cspeed, new_dspeed = new[j]
cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels(
old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
)
if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
regressions.append(
"[COMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
level,
filenames_directory,
dictionary_filename,
baseline_label,
test_label,
old_cspeed,
new_cspeed,
cspeed_reg * 100.0,
)
)
if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
regressions.append(
"[DECOMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
level,
filenames_directory,
dictionary_filename,
baseline_label,
test_label,
old_dspeed,
new_dspeed,
dspeed_reg * 100.0,
)
)
return regressions
def main(filenames, levels, iterations, builds=None, emails=None, continuous=False, frequency=DEFAULT_MAX_API_CALL_FREQUENCY_SEC, dictionary_filename=None):
if builds == None:
builds = get_new_open_pr_builds()
while True:
for test_build in builds:
if dictionary_filename == None:
regressions = get_regressions(
RELEASE_BUILD, test_build, iterations, filenames, levels
)
else:
regressions = get_regressions_dictionary(
RELEASE_BUILD, test_build, filenames, dictionary_filename, levels, iterations
)
body = "\n".join(regressions)
if len(regressions) > 0:
if emails != None:
os.system(
"""
echo "{}" | mutt -s "[zstd regression] caused by new pr" {}
""".format(
body, emails
)
)
print("Emails sent to {}".format(emails))
print(body)
if not continuous:
break
time.sleep(frequency)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--directory", help="directory with files to benchmark", default="golden-compression")
parser.add_argument("--levels", help="levels to test e.g. ('1,2,3')", default="1")
parser.add_argument("--iterations", help="number of benchmark iterations to run", default="1")
parser.add_argument("--emails", help="email addresses of people who will be alerted upon regression. Only for continuous mode", default=None)
parser.add_argument("--frequency", help="specifies the number of seconds to wait before each successive check for new PRs in continuous mode", default=DEFAULT_MAX_API_CALL_FREQUENCY_SEC)
parser.add_argument("--mode", help="'fastmode', 'onetime', 'current', or 'continuous' (see README.md for details)", default="current")
parser.add_argument("--dict", help="filename of dictionary to use (when set, this dictionary will be used to compress the files provided inside --directory)", default=None)
args = parser.parse_args()
filenames = args.directory
levels = [int(l) for l in args.levels.split(",")]
mode = args.mode
iterations = int(args.iterations)
emails = args.emails
frequency = int(args.frequency)
dictionary_filename = args.dict
if dictionary_filename == None:
filenames = glob.glob("{}/**".format(filenames))
if (len(filenames) == 0):
print("0 files found")
quit()
if mode == "onetime":
main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename)
elif mode == "current":
builds = [{"user": None, "branch": "None", "hash": None}]
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
elif mode == "fastmode":
builds = [{"user": "facebook", "branch": "release", "hash": None}]
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
else:
main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename)

128
third_party/zstd/tests/bigdict.c vendored
View file

@ -1,128 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include <assert.h>
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>
#include "datagen.h"
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
static int
compress(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
void const* src, size_t srcSize,
void* roundtrip, ZSTD_EndDirective end)
{
ZSTD_inBuffer in = {src, srcSize, 0};
ZSTD_outBuffer out = {dst, dstCapacity, 0};
int ended = 0;
while (!ended && (in.pos < in.size || out.pos > 0)) {
size_t rc;
out.pos = 0;
rc = ZSTD_compressStream2(cctx, &out, &in, end);
if (ZSTD_isError(rc))
return 1;
if (end == ZSTD_e_end && rc == 0)
ended = 1;
{
ZSTD_inBuffer rtIn = {dst, out.pos, 0};
ZSTD_outBuffer rtOut = {roundtrip, srcSize, 0};
rc = 1;
while (rtIn.pos < rtIn.size || rtOut.pos > 0) {
rtOut.pos = 0;
rc = ZSTD_decompressStream(dctx, &rtOut, &rtIn);
if (ZSTD_isError(rc)) {
fprintf(stderr, "Decompression error: %s\n", ZSTD_getErrorName(rc));
return 1;
}
if (rc == 0)
break;
}
if (ended && rc != 0) {
fprintf(stderr, "Frame not finished!\n");
return 1;
}
}
}
return 0;
}
int main(int argc, const char** argv)
{
ZSTD_CCtx* cctx = ZSTD_createCCtx();
ZSTD_DCtx* dctx = ZSTD_createDCtx();
const size_t dataSize = (size_t)1 << 30;
const size_t outSize = ZSTD_compressBound(dataSize);
const size_t bufferSize = (size_t)1 << 31;
char* buffer = (char*)malloc(bufferSize);
void* out = malloc(outSize);
void* roundtrip = malloc(dataSize);
(void)argc;
(void)argv;
if (!buffer || !out || !roundtrip || !cctx || !dctx) {
fprintf(stderr, "Allocation failure\n");
return 1;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
return 1;
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
return 1;
if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
return 1;
RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
/* Compress 30 GB */
{
int i;
for (i = 0; i < 10; ++i) {
fprintf(stderr, "Compressing 1 GB\n");
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
return 1;
}
}
fprintf(stderr, "Compressing 1 GB\n");
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
return 1;
fprintf(stderr, "Success!\n");
free(roundtrip);
free(out);
free(buffer);
ZSTD_freeDCtx(dctx);
ZSTD_freeCCtx(cctx);
return 0;
}

65
third_party/zstd/tests/checkTag.c vendored
View file

@ -1,65 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* checkTag : validation tool for libzstd
* command :
* $ ./checkTag tag
* checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
* The tag is then compared to zstd version number.
* They are compatible if first 3 digits are identical.
* Anything beyond that is free, and doesn't impact validation.
* Example : tag v1.8.1.2 is compatible with version 1.8.1
* When tag and version are not compatible, program exits with error code 1.
* When they are compatible, it exists with a code 0.
* checkTag is intended to be used in automated testing environment.
*/
#include <stdio.h> /* printf */
#include <string.h> /* strlen, strncmp */
#include "zstd.h" /* ZSTD_VERSION_STRING */
/* validate() :
* @return 1 if tag is compatible, 0 if not.
*/
static int validate(const char* const tag)
{
size_t const tagLength = strlen(tag);
size_t const verLength = strlen(ZSTD_VERSION_STRING);
if (tagLength < 2) return 0;
if (tag[0] != 'v') return 0;
if (tagLength <= verLength) return 0;
if (strncmp(ZSTD_VERSION_STRING, tag+1, verLength)) return 0;
return 1;
}
int main(int argc, const char** argv)
{
const char* const exeName = argv[0];
const char* const tag = argv[1];
if (argc!=2) {
printf("incorrect usage : %s tag \n", exeName);
return 2;
}
printf("Version : %s \n", ZSTD_VERSION_STRING);
printf("Tag : %s \n", tag);
if (validate(tag)) {
printf("OK : tag is compatible with zstd version \n");
return 0;
}
printf("!! error : tag and versions are not compatible !! \n");
return 1;
}

31
third_party/zstd/tests/check_size.py vendored
View file

@ -1,31 +0,0 @@
#!/usr/bin/env python3
# ################################################################
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
import os
import subprocess
import sys
if len(sys.argv) != 3:
print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
sys.exit(1)
file = sys.argv[1]
limit = int(sys.argv[2])
if not os.path.exists(file):
print(f"{file} does not exist")
sys.exit(1)
size = os.path.getsize(file)
if size > limit:
print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
sys.exit(1)

6
third_party/zstd/tests/cli-tests/.gitignore vendored
View file

@ -1,6 +0,0 @@
!bin/
!datagen
!zstdcat
scratch/
bin/symlinks

258
third_party/zstd/tests/cli-tests/README.md vendored
View file

@ -1,258 +0,0 @@
# CLI tests
The CLI tests are focused on testing the zstd CLI.
They are intended to be simple tests that the CLI and arguments work as advertised.
They are not intended to test the library, only the code in `programs/`.
The library will get incidental coverage, but if you find yourself trying to trigger a specific condition in the library, this is the wrong tool.
## Test runner usage
The test runner `run.py` will run tests against the in-tree build of `zstd` and `datagen` by default. Which means that `zstd` and `datagen` must be built.
The `zstd` binary used can be passed with `--zstd /path/to/zstd`.
Additionally, to run `zstd` through a tool like `valgrind` or `qemu`, set the `--exec-prefix 'valgrind -q'` flag.
Similarly, the `--datagen`, and `--zstdgrep` flags can be set to specify
the paths to their respective binaries. However, these tools do not use
the `EXEC_PREFIX`.
Each test executes in its own scratch directory under `scratch/test/name`. E.g. `scratch/basic/help.sh/`. Normally these directories are removed after the test executes. However, the `--preserve` flag will preserve these directories after execution, and save the tests exit code, stdout, and stderr in the scratch directory to `exit`, `stderr`, and `stdout` respectively. This can be useful for debugging/editing a test and updating the expected output.
### Running all the tests
By default the test runner `run.py` will run all the tests, and report the results.
Examples:
```
./run.py
./run.py --preserve
./run.py --zstd ../../build/programs/zstd --datagen ../../build/tests/datagen
```
### Running specific tests
A set of test names can be passed to the test runner `run.py` to only execute those tests.
This can be useful for writing or debugging a test, especially with `--preserve`.
The test name can either be the path to the test file, or the test name, which is the path relative to the test directory.
Examples:
```
./run.py basic/help.sh
./run.py --preserve basic/help.sh basic/version.sh
./run.py --preserve --verbose basic/help.sh
```
### Updating exact output
If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
Example:
```
./run.py --set-exact-output
./run.py basic/help.sh --set-exact-output
```
## Writing a test
Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
After the script executes, the exit code, stderr, and stdout are compared against the expectations.
Each test is run in a clean directory that the test can use for intermediate files. This directory will be cleaned up at the end of the test, unless `--preserve` is passed to the test runner. Additionally, the `setup` script can prepare the directory before the test runs.
### Calling zstd, utilities, and environment variables
The `$PATH` for tests is prepended with the `bin/` sub-directory, which contains helper scripts for ease of testing.
The `zstd` binary will call the zstd binary specified by `run.py` with the correct `$EXEC_PREFIX`.
Similarly, `datagen`, `unzstd`, `zstdgrep`, `zstdcat`, etc, are provided.
Helper utilities like `cmp_size`, `println`, and `die` are provided here too. See their scripts for details.
Common shell script libraries are provided under `common/`, with helper variables and functions. They can be sourced with `source "$COMMON/library.sh`.
Lastly, environment variables are provided for testing, which can be listed when calling `run.py` with `--verbose`.
They are generally used by the helper scripts in `bin/` to coordinate everything.
### Basic test case
When executing your `$TEST` executable, by default the exit code is expected to be `0`. However, you can provide an alternate expected exit code in a `$TEST.exit` file.
When executing your `$TEST` executable, by default the expected stderr and stdout are empty. However, you can override the default by providing one of three files:
* `$TEST.{stdout,stderr}.exact`
* `$TEST.{stdout,stderr}.glob`
* `$TEST.{stdout,stderr}.ignore`
If you provide a `.exact` file, the output is expected to exactly match, byte-for-byte.
If you provide a `.glob` file, the output is expected to match the expected file, where each line is interpreted as a glob syntax. Additionally, a line containing only `...` matches all lines until the next expected line matches.
If you provide a `.ignore` file, the output is ignored.
#### Passing examples
All these examples pass.
Exit 1, and change the expectation to be 1.
```
exit-1.sh
---
#!/bin/sh
exit 1
---
exit-1.sh.exit
---
1
---
```
Check the stdout output exactly matches.
```
echo.sh
---
#!/bin/sh
echo "hello world"
---
echo.sh.stdout.exact
---
hello world
---
```
Check the stderr output using a glob.
```
random.sh
---
#!/bin/sh
head -c 10 < /dev/urandom | xxd >&2
---
random.sh.stderr.glob
---
00000000: * * * * * *
```
Multiple lines can be matched with ...
```
random-num-lines.sh
---
#!/bin/sh
echo hello
seq 0 $RANDOM
echo world
---
random-num-lines.sh.stdout.glob
---
hello
0
...
world
---
```
#### Failing examples
Exit code is expected to be 0, but is 1.
```
exit-1.sh
---
#!/bin/sh
exit 1
---
```
Stdout is expected to be empty, but isn't.
```
echo.sh
---
#!/bin/sh
echo hello world
```
Stderr is expected to be hello but is world.
```
hello.sh
---
#!/bin/sh
echo world >&2
---
hello.sh.stderr.exact
---
hello
---
```
### Setup & teardown scripts
Finally, test writing can be eased with setup and teardown scripts.
Each directory in the test directory is a test-suite consisting of all tests within that directory (but not sub-directories).
This test suite can come with 4 scripts to help test writing:
* `setup_once`
* `teardown_once`
* `setup`
* `teardown`
The `setup_once` and `teardown_once` are run once before and after all the tests in the suite respectively.
They operate in the scratch directory for the test suite, which is the parent directory of each scratch directory for each test case.
They can do work that is shared between tests to improve test efficiency.
For example, the `dictionaries/setup_once` script builds several dictionaries, for use in the `dictionaries` tests.
The `setup` and `teardown` scripts run before and after each test case respectively, in the test case's scratch directory.
These scripts can do work that is shared between test cases to make tests more succinct.
For example, the `dictionaries/setup` script copies the dictionaries built by the `dictionaries/setup_once` script into the test's scratch directory, to make them easier to use, and make sure they aren't accidentally modified.
#### Examples
```
basic/setup
---
#!/bin/sh
# Create some files for testing with
datagen > file
datagen > file0
datagen > file1
---
basic/test.sh
---
#!/bin/sh
zstd file file0 file1
---
dictionaries/setup_once
---
#!/bin/sh
set -e
mkdir files/ dicts/
for i in $(seq 10); do
datagen -g1000 > files/$i
done
zstd --train -r files/ -o dicts/0
---
dictionaries/setup
---
#!/bin/sh
# Runs in the test case's scratch directory.
# The test suite's scratch directory that
# `setup_once` operates in is the parent directory.
cp -r ../files ../dicts .
---
```

10
third_party/zstd/tests/cli-tests/basic/help.sh vendored
View file

@ -1,10 +0,0 @@
#!/bin/sh
set -e
println "+ zstd -h"
zstd -h
println "+ zstd -H"
zstd -H
println "+ zstd --help"
zstd --help

34
third_party/zstd/tests/cli-tests/basic/help.sh.stdout.glob vendored
View file

@ -1,34 +0,0 @@
+ zstd -h
Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
Options:
-o OUTPUT Write output to a single file, OUTPUT.
-k, --keep Preserve INPUT file(s). *Default*
--rm Remove INPUT file(s) after successful (de)compression.
-# Desired compression level, where `#` is a number between 1 and 19;
lower numbers provide faster compression, higher numbers yield
better compression ratios. *Default: 3*
-d, --decompress Perform decompression.
-D DICT Use DICT as the dictionary for compression or decompression.
-f, --force Disable input and output checks. Allows overwriting existing files,
receiving input from the console, printing output to STDOUT, and
operating on links, block devices, etc. Unrecognized formats will be
passed-through through as-is.
-h Display short usage and exit.
-H, --help Display full help and exit.
-V, --version Display the program version and exit.
+ zstd -H
...
Advanced options:
...
+ zstd --help
...
Advanced options:
...

40
third_party/zstd/tests/cli-tests/basic/memlimit.sh vendored
View file

@ -1,40 +0,0 @@
#!/bin/sh
echo "some data" > file
println "+ zstd --memory=32LB file"
zstd --memory=32LB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32LiB file"
zstd --memory=32LiB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32A file"
zstd --memory=32A file && die "Should not allow bogus suffix"
println "+ zstd --memory=32r82347dn83 file"
zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
println "+ zstd --memory=32asbdf file"
zstd --memory=32asbdf file && die "Should not allow bogus suffix"
println "+ zstd --memory=hello file"
zstd --memory=hello file && die "Should not allow non-numeric parameter"
println "+ zstd --memory=1 file"
zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
rm file.zst
println "+ zstd --memory=1K file"
zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KB file"
zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KiB file"
zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1M file"
zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MB file"
zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MiB file"
zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
rm file
exit 0

13
third_party/zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact vendored
View file

@ -1,13 +0,0 @@
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
Should allow numeric parameter without suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix
Should allow numeric parameter with expected suffix

13
third_party/zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact vendored
View file

@ -1,13 +0,0 @@
+ zstd --memory=32LB file
+ zstd --memory=32LiB file
+ zstd --memory=32A file
+ zstd --memory=32r82347dn83 file
+ zstd --memory=32asbdf file
+ zstd --memory=hello file
+ zstd --memory=1 file
+ zstd --memory=1K file
+ zstd --memory=1KB file
+ zstd --memory=1KiB file
+ zstd --memory=1M file
+ zstd --memory=1MB file
+ zstd --memory=1MiB file

7
third_party/zstd/tests/cli-tests/basic/output_dir.sh vendored
View file

@ -1,7 +0,0 @@
#!/bin/sh
println "+ zstd -r * --output-dir-mirror=\"\""
zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!"
println "+ zstd -r * --output-dir-flat=\"\""
zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!"
exit 0

2
third_party/zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact vendored
View file

@ -1,2 +0,0 @@
error: output dir cannot be empty string (did you mean to pass '.' instead?)
error: output dir cannot be empty string (did you mean to pass '.' instead?)

2
third_party/zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact vendored
View file

@ -1,2 +0,0 @@
+ zstd -r * --output-dir-mirror=""
+ zstd -r * --output-dir-flat=""

6
third_party/zstd/tests/cli-tests/basic/version.sh vendored
View file

@ -1,6 +0,0 @@
#!/bin/sh
set -e
zstd -V
zstd --version

2
third_party/zstd/tests/cli-tests/basic/version.sh.stdout.glob vendored
View file

@ -1,2 +0,0 @@
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***

44
third_party/zstd/tests/cli-tests/bin/cmp_size vendored
View file

@ -1,44 +0,0 @@
#!/bin/sh
set -e
usage()
{
printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n"
}
help()
{
printf "Small utility to compare file sizes without printing them with set -x.\n\n"
usage
}
case "$1" in
-h) help; exit 0 ;;
--help) help; exit 0 ;;
esac
if ! test -f $2; then
printf "FILE1='%b' is not a file\n\n" "$2"
usage
exit 1
fi
if ! test -f $3; then
printf "FILE2='%b' is not a file\n\n" "$3"
usage
exit 1
fi
size1=$(wc -c < $2)
size2=$(wc -c < $3)
case "$1" in
-eq) [ "$size1" -eq "$size2" ] ;;
-ne) [ "$size1" -ne "$size2" ] ;;
-lt) [ "$size1" -lt "$size2" ] ;;
-le) [ "$size1" -le "$size2" ] ;;
-gt) [ "$size1" -gt "$size2" ] ;;
-ge) [ "$size1" -ge "$size2" ] ;;
esac

3
third_party/zstd/tests/cli-tests/bin/datagen vendored
View file

@ -1,3 +0,0 @@
#!/bin/sh
"$DATAGEN_BIN" $@

4
third_party/zstd/tests/cli-tests/bin/die vendored
View file

@ -1,4 +0,0 @@
#!/bin/sh
println "${*}" 1>&2
exit 1

2
third_party/zstd/tests/cli-tests/bin/println vendored
View file

@ -1,2 +0,0 @@
#!/bin/sh
printf '%b\n' "${*}"

1
third_party/zstd/tests/cli-tests/bin/unzstd vendored
View file

@ -1 +0,0 @@
zstd

9
third_party/zstd/tests/cli-tests/bin/zstd vendored
View file

@ -1,9 +0,0 @@
#!/bin/sh
zstdname=$(basename $0)
if [ -z "$EXEC_PREFIX" ]; then
"$ZSTD_SYMLINK_DIR/$zstdname" $@
else
$EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@
fi

1
third_party/zstd/tests/cli-tests/bin/zstdcat vendored
View file

@ -1 +0,0 @@
zstd

2
third_party/zstd/tests/cli-tests/bin/zstdgrep vendored
View file

@ -1,2 +0,0 @@
#!/bin/sh
"$ZSTDGREP_BIN" $@

2
third_party/zstd/tests/cli-tests/bin/zstdless vendored
View file

@ -1,2 +0,0 @@
#!/bin/sh
"$ZSTDLESS_BIN" $@

6
third_party/zstd/tests/cli-tests/cltools/setup vendored
View file

@ -1,6 +0,0 @@
#!/bin/sh
set -e
echo "1234" > file
zstd file

8
third_party/zstd/tests/cli-tests/cltools/zstdgrep.sh vendored
View file

@ -1,8 +0,0 @@
#!/bin/sh
set -e
println "+ good path"
zstdgrep "1234" file file.zst
println "+ bad path"
zstdgrep "1234" bad.zst

1
third_party/zstd/tests/cli-tests/cltools/zstdgrep.sh.exit vendored
View file

@ -1 +0,0 @@
1

1
third_party/zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact vendored
View file

@ -1 +0,0 @@
zstd: can't stat bad.zst : No such file or directory -- ignored

4
third_party/zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob vendored
View file

@ -1,4 +0,0 @@
+ good path
file:1234
file.zst:1234
+ bad path

Some files were not shown because too many files have changed in this diff Show more