mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 09:19:51 +00:00
![Nick Terrell](/assets/img/avatar_default.png)
Add zstd compression and decompression kernel modules. zstd offers a wide varity of compression speed and quality trade-offs. It can compress at speeds approaching lz4, and quality approaching lzma. zstd decompressions at speeds more than twice as fast as zlib, and decompression speed remains roughly the same across all compression levels. The code was ported from the upstream zstd source repository. The `linux/zstd.h` header was modified to match linux kernel style. The cross-platform and allocation code was stripped out. Instead zstd requires the caller to pass a preallocated workspace. The source files were clang-formatted [1] to match the Linux Kernel style as much as possible. Otherwise, the code was unmodified. We would like to avoid as much further manual modification to the source code as possible, so it will be easier to keep the kernel zstd up to date. I benchmarked zstd compression as a special character device. I ran zstd and zlib compression at several levels, as well as performing no compression, which measure the time spent copying the data to kernel space. Data is passed to the compresser 4096 B at a time. The benchmark file is located in the upstream zstd source repository under `contrib/linux-kernel/zstd_compress_test.c` [2]. I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor, 16 GB of RAM, and a SSD. I benchmarked using `silesia.tar` [3], which is 211,988,480 B large. Run the following commands for the benchmark: sudo modprobe zstd_compress_test sudo mknod zstd_compress_test c 245 0 sudo cp silesia.tar zstd_compress_test The time is reported by the time of the userland `cp`. The MB/s is computed with 1,536,217,008 B / time(buffer size, hash) which includes the time to copy from userland. The Adjusted MB/s is computed with 1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)). The memory reported is the amount of memory the compressor requests. | Method | Size (B) | Time (s) | Ratio | MB/s | Adj MB/s | Mem (MB) | |----------|----------|----------|-------|---------|----------|----------| | none | 11988480 | 0.100 | 1 | 2119.88 | - | - | | zstd -1 | 73645762 | 1.044 | 2.878 | 203.05 | 224.56 | 1.23 | | zstd -3 | 66988878 | 1.761 | 3.165 | 120.38 | 127.63 | 2.47 | | zstd -5 | 65001259 | 2.563 | 3.261 | 82.71 | 86.07 | 2.86 | | zstd -10 | 60165346 | 13.242 | 3.523 | 16.01 | 16.13 | 13.22 | | zstd -15 | 58009756 | 47.601 | 3.654 | 4.45 | 4.46 | 21.61 | | zstd -19 | 54014593 | 102.835 | 3.925 | 2.06 | 2.06 | 60.15 | | zlib -1 | 77260026 | 2.895 | 2.744 | 73.23 | 75.85 | 0.27 | | zlib -3 | 72972206 | 4.116 | 2.905 | 51.50 | 52.79 | 0.27 | | zlib -6 | 68190360 | 9.633 | 3.109 | 22.01 | 22.24 | 0.27 | | zlib -9 | 67613382 | 22.554 | 3.135 | 9.40 | 9.44 | 0.27 | I benchmarked zstd decompression using the same method on the same machine. The benchmark file is located in the upstream zstd repo under `contrib/linux-kernel/zstd_decompress_test.c` [4]. The memory reported is the amount of memory required to decompress data compressed with the given compression level. If you know the maximum size of your input, you can reduce the memory usage of decompression irrespective of the compression level. | Method | Time (s) | MB/s | Adjusted MB/s | Memory (MB) | |----------|----------|---------|---------------|-------------| | none | 0.025 | 8479.54 | - | - | | zstd -1 | 0.358 | 592.15 | 636.60 | 0.84 | | zstd -3 | 0.396 | 535.32 | 571.40 | 1.46 | | zstd -5 | 0.396 | 535.32 | 571.40 | 1.46 | | zstd -10 | 0.374 | 566.81 | 607.42 | 2.51 | | zstd -15 | 0.379 | 559.34 | 598.84 | 4.61 | | zstd -19 | 0.412 | 514.54 | 547.77 | 8.80 | | zlib -1 | 0.940 | 225.52 | 231.68 | 0.04 | | zlib -3 | 0.883 | 240.08 | 247.07 | 0.04 | | zlib -6 | 0.844 | 251.17 | 258.84 | 0.04 | | zlib -9 | 0.837 | 253.27 | 287.64 | 0.04 | Tested in userland using the test-suite in the zstd repo under `contrib/linux-kernel/test/UserlandTest.cpp` [5] by mocking the kernel functions. Fuzz tested using libfuzzer [6] with the fuzz harnesses under `contrib/linux-kernel/test/{RoundTripCrash.c,DecompressCrash.c}` [7] [8] with ASAN, UBSAN, and MSAN. Additionaly, it was tested while testing the BtrFS and SquashFS patches coming next. [1] https://clang.llvm.org/docs/ClangFormat.html [2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_compress_test.c [3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia [4] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/zstd_decompress_test.c [5] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/UserlandTest.cpp [6] http://llvm.org/docs/LibFuzzer.html [7] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/RoundTripCrash.c [8] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/DecompressCrash.c zstd source repository: https://github.com/facebook/zstd Signed-off-by: Nick Terrell <terrelln@fb.com> Signed-off-by: Chris Mason <clm@fb.com>
332 lines
11 KiB
C
332 lines
11 KiB
C
/*
|
|
* FSE : Finite State Entropy decoder
|
|
* Copyright (C) 2013-2015, Yann Collet.
|
|
*
|
|
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following disclaimer
|
|
* in the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it under
|
|
* the terms of the GNU General Public License version 2 as published by the
|
|
* Free Software Foundation. This program is dual-licensed; you may select
|
|
* either version 2 of the GNU General Public License ("GPL") or BSD license
|
|
* ("BSD").
|
|
*
|
|
* You can contact the author at :
|
|
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
|
|
*/
|
|
|
|
/* **************************************************************
|
|
* Compiler specifics
|
|
****************************************************************/
|
|
#define FORCE_INLINE static __always_inline
|
|
|
|
/* **************************************************************
|
|
* Includes
|
|
****************************************************************/
|
|
#include "bitstream.h"
|
|
#include "fse.h"
|
|
#include <linux/compiler.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/string.h> /* memcpy, memset */
|
|
|
|
/* **************************************************************
|
|
* Error Management
|
|
****************************************************************/
|
|
#define FSE_isError ERR_isError
|
|
#define FSE_STATIC_ASSERT(c) \
|
|
{ \
|
|
enum { FSE_static_assert = 1 / (int)(!!(c)) }; \
|
|
} /* use only *after* variable declarations */
|
|
|
|
/* check and forward error code */
|
|
#define CHECK_F(f) \
|
|
{ \
|
|
size_t const e = f; \
|
|
if (FSE_isError(e)) \
|
|
return e; \
|
|
}
|
|
|
|
/* **************************************************************
|
|
* Templates
|
|
****************************************************************/
|
|
/*
|
|
designed to be included
|
|
for type-specific functions (template emulation in C)
|
|
Objective is to write these functions only once, for improved maintenance
|
|
*/
|
|
|
|
/* safety checks */
|
|
#ifndef FSE_FUNCTION_EXTENSION
|
|
#error "FSE_FUNCTION_EXTENSION must be defined"
|
|
#endif
|
|
#ifndef FSE_FUNCTION_TYPE
|
|
#error "FSE_FUNCTION_TYPE must be defined"
|
|
#endif
|
|
|
|
/* Function names */
|
|
#define FSE_CAT(X, Y) X##Y
|
|
#define FSE_FUNCTION_NAME(X, Y) FSE_CAT(X, Y)
|
|
#define FSE_TYPE_NAME(X, Y) FSE_CAT(X, Y)
|
|
|
|
/* Function templates */
|
|
|
|
size_t FSE_buildDTable_wksp(FSE_DTable *dt, const short *normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void *workspace, size_t workspaceSize)
|
|
{
|
|
void *const tdPtr = dt + 1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
|
|
FSE_DECODE_TYPE *const tableDecode = (FSE_DECODE_TYPE *)(tdPtr);
|
|
U16 *symbolNext = (U16 *)workspace;
|
|
|
|
U32 const maxSV1 = maxSymbolValue + 1;
|
|
U32 const tableSize = 1 << tableLog;
|
|
U32 highThreshold = tableSize - 1;
|
|
|
|
/* Sanity Checks */
|
|
if (workspaceSize < sizeof(U16) * (FSE_MAX_SYMBOL_VALUE + 1))
|
|
return ERROR(tableLog_tooLarge);
|
|
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE)
|
|
return ERROR(maxSymbolValue_tooLarge);
|
|
if (tableLog > FSE_MAX_TABLELOG)
|
|
return ERROR(tableLog_tooLarge);
|
|
|
|
/* Init, lay down lowprob symbols */
|
|
{
|
|
FSE_DTableHeader DTableH;
|
|
DTableH.tableLog = (U16)tableLog;
|
|
DTableH.fastMode = 1;
|
|
{
|
|
S16 const largeLimit = (S16)(1 << (tableLog - 1));
|
|
U32 s;
|
|
for (s = 0; s < maxSV1; s++) {
|
|
if (normalizedCounter[s] == -1) {
|
|
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
|
|
symbolNext[s] = 1;
|
|
} else {
|
|
if (normalizedCounter[s] >= largeLimit)
|
|
DTableH.fastMode = 0;
|
|
symbolNext[s] = normalizedCounter[s];
|
|
}
|
|
}
|
|
}
|
|
memcpy(dt, &DTableH, sizeof(DTableH));
|
|
}
|
|
|
|
/* Spread symbols */
|
|
{
|
|
U32 const tableMask = tableSize - 1;
|
|
U32 const step = FSE_TABLESTEP(tableSize);
|
|
U32 s, position = 0;
|
|
for (s = 0; s < maxSV1; s++) {
|
|
int i;
|
|
for (i = 0; i < normalizedCounter[s]; i++) {
|
|
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
|
|
position = (position + step) & tableMask;
|
|
while (position > highThreshold)
|
|
position = (position + step) & tableMask; /* lowprob area */
|
|
}
|
|
}
|
|
if (position != 0)
|
|
return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
|
|
}
|
|
|
|
/* Build Decoding table */
|
|
{
|
|
U32 u;
|
|
for (u = 0; u < tableSize; u++) {
|
|
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
|
|
U16 nextState = symbolNext[symbol]++;
|
|
tableDecode[u].nbBits = (BYTE)(tableLog - BIT_highbit32((U32)nextState));
|
|
tableDecode[u].newState = (U16)((nextState << tableDecode[u].nbBits) - tableSize);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*-*******************************************************
|
|
* Decompression (Byte symbols)
|
|
*********************************************************/
|
|
size_t FSE_buildDTable_rle(FSE_DTable *dt, BYTE symbolValue)
|
|
{
|
|
void *ptr = dt;
|
|
FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
|
|
void *dPtr = dt + 1;
|
|
FSE_decode_t *const cell = (FSE_decode_t *)dPtr;
|
|
|
|
DTableH->tableLog = 0;
|
|
DTableH->fastMode = 0;
|
|
|
|
cell->newState = 0;
|
|
cell->symbol = symbolValue;
|
|
cell->nbBits = 0;
|
|
|
|
return 0;
|
|
}
|
|
|
|
size_t FSE_buildDTable_raw(FSE_DTable *dt, unsigned nbBits)
|
|
{
|
|
void *ptr = dt;
|
|
FSE_DTableHeader *const DTableH = (FSE_DTableHeader *)ptr;
|
|
void *dPtr = dt + 1;
|
|
FSE_decode_t *const dinfo = (FSE_decode_t *)dPtr;
|
|
const unsigned tableSize = 1 << nbBits;
|
|
const unsigned tableMask = tableSize - 1;
|
|
const unsigned maxSV1 = tableMask + 1;
|
|
unsigned s;
|
|
|
|
/* Sanity checks */
|
|
if (nbBits < 1)
|
|
return ERROR(GENERIC); /* min size */
|
|
|
|
/* Build Decoding Table */
|
|
DTableH->tableLog = (U16)nbBits;
|
|
DTableH->fastMode = 1;
|
|
for (s = 0; s < maxSV1; s++) {
|
|
dinfo[s].newState = 0;
|
|
dinfo[s].symbol = (BYTE)s;
|
|
dinfo[s].nbBits = (BYTE)nbBits;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(void *dst, size_t maxDstSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt,
|
|
const unsigned fast)
|
|
{
|
|
BYTE *const ostart = (BYTE *)dst;
|
|
BYTE *op = ostart;
|
|
BYTE *const omax = op + maxDstSize;
|
|
BYTE *const olimit = omax - 3;
|
|
|
|
BIT_DStream_t bitD;
|
|
FSE_DState_t state1;
|
|
FSE_DState_t state2;
|
|
|
|
/* Init */
|
|
CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
|
|
|
|
FSE_initDState(&state1, &bitD, dt);
|
|
FSE_initDState(&state2, &bitD, dt);
|
|
|
|
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
|
|
|
|
/* 4 symbols per loop */
|
|
for (; (BIT_reloadDStream(&bitD) == BIT_DStream_unfinished) & (op < olimit); op += 4) {
|
|
op[0] = FSE_GETSYMBOL(&state1);
|
|
|
|
if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
|
|
BIT_reloadDStream(&bitD);
|
|
|
|
op[1] = FSE_GETSYMBOL(&state2);
|
|
|
|
if (FSE_MAX_TABLELOG * 4 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
|
|
{
|
|
if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) {
|
|
op += 2;
|
|
break;
|
|
}
|
|
}
|
|
|
|
op[2] = FSE_GETSYMBOL(&state1);
|
|
|
|
if (FSE_MAX_TABLELOG * 2 + 7 > sizeof(bitD.bitContainer) * 8) /* This test must be static */
|
|
BIT_reloadDStream(&bitD);
|
|
|
|
op[3] = FSE_GETSYMBOL(&state2);
|
|
}
|
|
|
|
/* tail */
|
|
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
|
|
while (1) {
|
|
if (op > (omax - 2))
|
|
return ERROR(dstSize_tooSmall);
|
|
*op++ = FSE_GETSYMBOL(&state1);
|
|
if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
|
|
*op++ = FSE_GETSYMBOL(&state2);
|
|
break;
|
|
}
|
|
|
|
if (op > (omax - 2))
|
|
return ERROR(dstSize_tooSmall);
|
|
*op++ = FSE_GETSYMBOL(&state2);
|
|
if (BIT_reloadDStream(&bitD) == BIT_DStream_overflow) {
|
|
*op++ = FSE_GETSYMBOL(&state1);
|
|
break;
|
|
}
|
|
}
|
|
|
|
return op - ostart;
|
|
}
|
|
|
|
size_t FSE_decompress_usingDTable(void *dst, size_t originalSize, const void *cSrc, size_t cSrcSize, const FSE_DTable *dt)
|
|
{
|
|
const void *ptr = dt;
|
|
const FSE_DTableHeader *DTableH = (const FSE_DTableHeader *)ptr;
|
|
const U32 fastMode = DTableH->fastMode;
|
|
|
|
/* select fast mode (static) */
|
|
if (fastMode)
|
|
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
|
|
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
|
|
}
|
|
|
|
size_t FSE_decompress_wksp(void *dst, size_t dstCapacity, const void *cSrc, size_t cSrcSize, unsigned maxLog, void *workspace, size_t workspaceSize)
|
|
{
|
|
const BYTE *const istart = (const BYTE *)cSrc;
|
|
const BYTE *ip = istart;
|
|
unsigned tableLog;
|
|
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
|
|
size_t NCountLength;
|
|
|
|
FSE_DTable *dt;
|
|
short *counting;
|
|
size_t spaceUsed32 = 0;
|
|
|
|
FSE_STATIC_ASSERT(sizeof(FSE_DTable) == sizeof(U32));
|
|
|
|
dt = (FSE_DTable *)((U32 *)workspace + spaceUsed32);
|
|
spaceUsed32 += FSE_DTABLE_SIZE_U32(maxLog);
|
|
counting = (short *)((U32 *)workspace + spaceUsed32);
|
|
spaceUsed32 += ALIGN(sizeof(short) * (FSE_MAX_SYMBOL_VALUE + 1), sizeof(U32)) >> 2;
|
|
|
|
if ((spaceUsed32 << 2) > workspaceSize)
|
|
return ERROR(tableLog_tooLarge);
|
|
workspace = (U32 *)workspace + spaceUsed32;
|
|
workspaceSize -= (spaceUsed32 << 2);
|
|
|
|
/* normal FSE decoding mode */
|
|
NCountLength = FSE_readNCount(counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
|
|
if (FSE_isError(NCountLength))
|
|
return NCountLength;
|
|
// if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining
|
|
// case : NCountLength==cSrcSize */
|
|
if (tableLog > maxLog)
|
|
return ERROR(tableLog_tooLarge);
|
|
ip += NCountLength;
|
|
cSrcSize -= NCountLength;
|
|
|
|
CHECK_F(FSE_buildDTable_wksp(dt, counting, maxSymbolValue, tableLog, workspace, workspaceSize));
|
|
|
|
return FSE_decompress_usingDTable(dst, dstCapacity, ip, cSrcSize, dt); /* always return, even if it is an error code */
|
|
}
|