mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 21:57:43 +00:00
5d2405227a
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
extremely fast non-cryptographic hash algorithm for checksumming.
The zstd compression and decompression modules added in the next patch
require xxhash. I extracted it out from zstd since it is useful on its
own. I copied the code from the upstream XXHash source repository and
translated it into kernel style. I ran benchmarks and tests in the kernel
and tests in userland.
I benchmarked xxhash as a special character device. I ran in four modes,
no-op, xxh32, xxh64, and crc32. The no-op mode simply copies the data to
kernel space and ignores it. The xxh32, xxh64, and crc32 modes compute
hashes on the copied data. I also ran it with four different buffer sizes.
The benchmark file is located in the upstream zstd source repository under
`contrib/linux-kernel/xxhash_test.c` [1].
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
16 GB of RAM, and a SSD. I benchmarked using the file `filesystem.squashfs`
from `ubuntu-16.10-desktop-amd64.iso`, which is 1,536,217,088 B large.
Run the following commands for the benchmark:
modprobe xxhash_test
mknod xxhash_test c 245 0
time cp filesystem.squashfs xxhash_test
The time is reported by the time of the userland `cp`.
The GB/s is computed with
1,536,217,008 B / time(buffer size, hash)
which includes the time to copy from userland.
The Normalized GB/s is computed with
1,536,217,088 B / (time(buffer size, hash) - time(buffer size, none)).
| Buffer Size (B) | Hash | Time (s) | GB/s | Adjusted GB/s |
|-----------------|-------|----------|------|---------------|
| 1024 | none | 0.408 | 3.77 | - |
| 1024 | xxh32 | 0.649 | 2.37 | 6.37 |
| 1024 | xxh64 | 0.542 | 2.83 | 11.46 |
| 1024 | crc32 | 1.290 | 1.19 | 1.74 |
| 4096 | none | 0.380 | 4.04 | - |
| 4096 | xxh32 | 0.645 | 2.38 | 5.79 |
| 4096 | xxh64 | 0.500 | 3.07 | 12.80 |
| 4096 | crc32 | 1.168 | 1.32 | 1.95 |
| 8192 | none | 0.351 | 4.38 | - |
| 8192 | xxh32 | 0.614 | 2.50 | 5.84 |
| 8192 | xxh64 | 0.464 | 3.31 | 13.60 |
| 8192 | crc32 | 1.163 | 1.32 | 1.89 |
| 16384 | none | 0.346 | 4.43 | - |
| 16384 | xxh32 | 0.590 | 2.60 | 6.30 |
| 16384 | xxh64 | 0.466 | 3.30 | 12.80 |
| 16384 | crc32 | 1.183 | 1.30 | 1.84 |
Tested in userland using the test-suite in the zstd repo under
`contrib/linux-kernel/test/XXHashUserlandTest.cpp` [2] by mocking the
kernel functions. A line in each branch of every function in `xxhash.c`
was commented out to ensure that the test-suite fails. Additionally
tested while testing zstd and with SMHasher [3].
[1] https://phabricator.intern.facebook.com/P57526246
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/test/XXHashUserlandTest.cpp
[3] https://github.com/aappleby/smhasher
zstd source repository: https://github.com/facebook/zstd
XXHash source repository: https://github.com/cyan4973/xxhash
Signed-off-by: Nick Terrell <terrelln@fb.com>
Signed-off-by: Chris Mason <clm@fb.com>
246 lines
7.7 KiB
Makefile
246 lines
7.7 KiB
Makefile
#
|
|
# Makefile for some libs needed in the kernel.
|
|
#
|
|
|
|
ifdef CONFIG_FUNCTION_TRACER
|
|
ORIG_CFLAGS := $(KBUILD_CFLAGS)
|
|
KBUILD_CFLAGS = $(subst $(CC_FLAGS_FTRACE),,$(ORIG_CFLAGS))
|
|
endif
|
|
|
|
# These files are disabled because they produce lots of non-interesting and/or
|
|
# flaky coverage that is not a function of syscall inputs. For example,
|
|
# rbtree can be global and individual rotations don't correlate with inputs.
|
|
KCOV_INSTRUMENT_string.o := n
|
|
KCOV_INSTRUMENT_rbtree.o := n
|
|
KCOV_INSTRUMENT_list_debug.o := n
|
|
KCOV_INSTRUMENT_debugobjects.o := n
|
|
KCOV_INSTRUMENT_dynamic_debug.o := n
|
|
|
|
lib-y := ctype.o string.o vsprintf.o cmdline.o \
|
|
rbtree.o radix-tree.o dump_stack.o timerqueue.o\
|
|
idr.o int_sqrt.o extable.o \
|
|
sha1.o chacha20.o irq_regs.o argv_split.o \
|
|
flex_proportions.o ratelimit.o show_mem.o \
|
|
is_single_threaded.o plist.o decompress.o kobject_uevent.o \
|
|
earlycpio.o seq_buf.o siphash.o \
|
|
nmi_backtrace.o nodemask.o win_minmax.o
|
|
|
|
lib-$(CONFIG_MMU) += ioremap.o
|
|
lib-$(CONFIG_SMP) += cpumask.o
|
|
lib-$(CONFIG_DMA_NOOP_OPS) += dma-noop.o
|
|
lib-$(CONFIG_DMA_VIRT_OPS) += dma-virt.o
|
|
|
|
lib-y += kobject.o klist.o
|
|
obj-y += lockref.o
|
|
|
|
obj-y += bcd.o div64.o sort.o parser.o debug_locks.o random32.o \
|
|
bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \
|
|
gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \
|
|
bsearch.o find_bit.o llist.o memweight.o kfifo.o \
|
|
percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o \
|
|
once.o refcount.o usercopy.o errseq.o
|
|
obj-y += string_helpers.o
|
|
obj-$(CONFIG_TEST_STRING_HELPERS) += test-string_helpers.o
|
|
obj-y += hexdump.o
|
|
obj-$(CONFIG_TEST_HEXDUMP) += test_hexdump.o
|
|
obj-y += kstrtox.o
|
|
obj-$(CONFIG_TEST_BPF) += test_bpf.o
|
|
obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o
|
|
obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o
|
|
obj-$(CONFIG_TEST_HASH) += test_hash.o test_siphash.o
|
|
obj-$(CONFIG_TEST_KASAN) += test_kasan.o
|
|
obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
|
|
obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
|
|
obj-$(CONFIG_TEST_LKM) += test_module.o
|
|
obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o
|
|
obj-$(CONFIG_TEST_SORT) += test_sort.o
|
|
obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o
|
|
obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o
|
|
obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o
|
|
obj-$(CONFIG_TEST_PRINTF) += test_printf.o
|
|
obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o
|
|
obj-$(CONFIG_TEST_UUID) += test_uuid.o
|
|
obj-$(CONFIG_TEST_PARMAN) += test_parman.o
|
|
obj-$(CONFIG_TEST_KMOD) += test_kmod.o
|
|
|
|
ifeq ($(CONFIG_DEBUG_KOBJECT),y)
|
|
CFLAGS_kobject.o += -DDEBUG
|
|
CFLAGS_kobject_uevent.o += -DDEBUG
|
|
endif
|
|
|
|
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
|
|
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
|
|
|
|
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
|
|
obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o
|
|
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
|
|
obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o
|
|
obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
|
|
|
|
obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o
|
|
|
|
obj-$(CONFIG_BTREE) += btree.o
|
|
obj-$(CONFIG_INTERVAL_TREE) += interval_tree.o
|
|
obj-$(CONFIG_ASSOCIATIVE_ARRAY) += assoc_array.o
|
|
obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
|
|
obj-$(CONFIG_DEBUG_LIST) += list_debug.o
|
|
obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o
|
|
|
|
ifneq ($(CONFIG_HAVE_DEC_LOCK),y)
|
|
lib-y += dec_and_lock.o
|
|
endif
|
|
|
|
obj-$(CONFIG_BITREVERSE) += bitrev.o
|
|
obj-$(CONFIG_RATIONAL) += rational.o
|
|
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
|
|
obj-$(CONFIG_CRC16) += crc16.o
|
|
obj-$(CONFIG_CRC_T10DIF)+= crc-t10dif.o
|
|
obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o
|
|
obj-$(CONFIG_CRC32) += crc32.o
|
|
obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o
|
|
obj-$(CONFIG_CRC4) += crc4.o
|
|
obj-$(CONFIG_CRC7) += crc7.o
|
|
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
|
|
obj-$(CONFIG_CRC8) += crc8.o
|
|
obj-$(CONFIG_XXHASH) += xxhash.o
|
|
obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
|
|
|
|
obj-$(CONFIG_842_COMPRESS) += 842/
|
|
obj-$(CONFIG_842_DECOMPRESS) += 842/
|
|
obj-$(CONFIG_ZLIB_INFLATE) += zlib_inflate/
|
|
obj-$(CONFIG_ZLIB_DEFLATE) += zlib_deflate/
|
|
obj-$(CONFIG_REED_SOLOMON) += reed_solomon/
|
|
obj-$(CONFIG_BCH) += bch.o
|
|
obj-$(CONFIG_LZO_COMPRESS) += lzo/
|
|
obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
|
|
obj-$(CONFIG_LZ4_COMPRESS) += lz4/
|
|
obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
|
|
obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
|
|
obj-$(CONFIG_XZ_DEC) += xz/
|
|
obj-$(CONFIG_RAID6_PQ) += raid6/
|
|
|
|
lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
|
|
lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
|
|
lib-$(CONFIG_DECOMPRESS_LZMA) += decompress_unlzma.o
|
|
lib-$(CONFIG_DECOMPRESS_XZ) += decompress_unxz.o
|
|
lib-$(CONFIG_DECOMPRESS_LZO) += decompress_unlzo.o
|
|
lib-$(CONFIG_DECOMPRESS_LZ4) += decompress_unlz4.o
|
|
|
|
obj-$(CONFIG_TEXTSEARCH) += textsearch.o
|
|
obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o
|
|
obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o
|
|
obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o
|
|
obj-$(CONFIG_SMP) += percpu_counter.o
|
|
obj-$(CONFIG_AUDIT_GENERIC) += audit.o
|
|
obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
|
|
|
|
obj-$(CONFIG_SWIOTLB) += swiotlb.o
|
|
obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o iommu-common.o
|
|
obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o
|
|
obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o
|
|
obj-$(CONFIG_PM_NOTIFIER_ERROR_INJECT) += pm-notifier-error-inject.o
|
|
obj-$(CONFIG_NETDEV_NOTIFIER_ERROR_INJECT) += netdev-notifier-error-inject.o
|
|
obj-$(CONFIG_MEMORY_NOTIFIER_ERROR_INJECT) += memory-notifier-error-inject.o
|
|
obj-$(CONFIG_OF_RECONFIG_NOTIFIER_ERROR_INJECT) += \
|
|
of-reconfig-notifier-error-inject.o
|
|
|
|
lib-$(CONFIG_GENERIC_BUG) += bug.o
|
|
|
|
obj-$(CONFIG_HAVE_ARCH_TRACEHOOK) += syscall.o
|
|
|
|
obj-$(CONFIG_DYNAMIC_DEBUG) += dynamic_debug.o
|
|
|
|
obj-$(CONFIG_NLATTR) += nlattr.o
|
|
|
|
obj-$(CONFIG_LRU_CACHE) += lru_cache.o
|
|
|
|
obj-$(CONFIG_DMA_API_DEBUG) += dma-debug.o
|
|
|
|
obj-$(CONFIG_GENERIC_CSUM) += checksum.o
|
|
|
|
obj-$(CONFIG_GENERIC_ATOMIC64) += atomic64.o
|
|
|
|
obj-$(CONFIG_ATOMIC64_SELFTEST) += atomic64_test.o
|
|
|
|
obj-$(CONFIG_CPU_RMAP) += cpu_rmap.o
|
|
|
|
obj-$(CONFIG_CORDIC) += cordic.o
|
|
|
|
obj-$(CONFIG_DQL) += dynamic_queue_limits.o
|
|
|
|
obj-$(CONFIG_GLOB) += glob.o
|
|
obj-$(CONFIG_GLOB_SELFTEST) += globtest.o
|
|
|
|
obj-$(CONFIG_MPILIB) += mpi/
|
|
obj-$(CONFIG_SIGNATURE) += digsig.o
|
|
|
|
lib-$(CONFIG_CLZ_TAB) += clz_tab.o
|
|
|
|
obj-$(CONFIG_DDR) += jedec_ddr_data.o
|
|
|
|
obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o
|
|
obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o
|
|
|
|
obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
|
|
|
|
obj-$(CONFIG_SG_SPLIT) += sg_split.o
|
|
obj-$(CONFIG_SG_POOL) += sg_pool.o
|
|
obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
|
|
obj-$(CONFIG_IRQ_POLL) += irq_poll.o
|
|
|
|
obj-$(CONFIG_STACKDEPOT) += stackdepot.o
|
|
KASAN_SANITIZE_stackdepot.o := n
|
|
KCOV_INSTRUMENT_stackdepot.o := n
|
|
|
|
libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
|
|
fdt_empty_tree.o
|
|
$(foreach file, $(libfdt_files), \
|
|
$(eval CFLAGS_$(file) = -I$(src)/../scripts/dtc/libfdt))
|
|
lib-$(CONFIG_LIBFDT) += $(libfdt_files)
|
|
|
|
obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o
|
|
obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o
|
|
|
|
obj-$(CONFIG_PERCPU_TEST) += percpu_test.o
|
|
|
|
obj-$(CONFIG_ASN1) += asn1_decoder.o
|
|
|
|
obj-$(CONFIG_FONT_SUPPORT) += fonts/
|
|
|
|
obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o
|
|
|
|
hostprogs-y := gen_crc32table
|
|
clean-files := crc32table.h
|
|
|
|
$(obj)/crc32.o: $(obj)/crc32table.h
|
|
|
|
quiet_cmd_crc32 = GEN $@
|
|
cmd_crc32 = $< > $@
|
|
|
|
$(obj)/crc32table.h: $(obj)/gen_crc32table
|
|
$(call cmd,crc32)
|
|
|
|
#
|
|
# Build a fast OID lookip registry from include/linux/oid_registry.h
|
|
#
|
|
obj-$(CONFIG_OID_REGISTRY) += oid_registry.o
|
|
|
|
$(obj)/oid_registry.o: $(obj)/oid_registry_data.c
|
|
|
|
$(obj)/oid_registry_data.c: $(srctree)/include/linux/oid_registry.h \
|
|
$(src)/build_OID_registry
|
|
$(call cmd,build_OID_registry)
|
|
|
|
quiet_cmd_build_OID_registry = GEN $@
|
|
cmd_build_OID_registry = perl $(srctree)/$(src)/build_OID_registry $< $@
|
|
|
|
clean-files += oid_registry_data.c
|
|
|
|
obj-$(CONFIG_UCS2_STRING) += ucs2_string.o
|
|
obj-$(CONFIG_UBSAN) += ubsan.o
|
|
|
|
UBSAN_SANITIZE_ubsan.o := n
|
|
|
|
obj-$(CONFIG_SBITMAP) += sbitmap.o
|
|
|
|
obj-$(CONFIG_PARMAN) += parman.o
|