2022-09-02 21:10:43 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
|
|
|
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
|
|
|
|
|
#ifndef _BPF_MEM_ALLOC_H
|
|
|
|
|
#define _BPF_MEM_ALLOC_H
|
|
|
|
|
#include <linux/compiler_types.h>
|
2022-09-02 21:10:58 +00:00
|
|
|
#include <linux/workqueue.h>
|
2022-09-02 21:10:43 +00:00
|
|
|
|
|
|
|
|
struct bpf_mem_cache;
|
|
|
|
|
struct bpf_mem_caches;
|
|
|
|
|
|
|
|
|
|
struct bpf_mem_alloc {
|
|
|
|
|
struct bpf_mem_caches __percpu *caches;
|
|
|
|
|
struct bpf_mem_cache __percpu *cache;
|
2023-12-22 03:17:39 +00:00
|
|
|
struct obj_cgroup *objcg;
|
2023-10-20 13:31:59 +00:00
|
|
|
bool percpu;
|
2022-09-02 21:10:58 +00:00
|
|
|
struct work_struct work;
|
2022-09-02 21:10:43 +00:00
|
|
|
};
|
|
|
|
|
|
2023-02-16 02:48:21 +00:00
|
|
|
/* 'size != 0' is for bpf_mem_alloc which manages fixed-size objects.
|
|
|
|
|
* Alloc and free are done with bpf_mem_cache_{alloc,free}().
|
|
|
|
|
*
|
|
|
|
|
* 'size = 0' is for bpf_mem_alloc which manages many fixed-size objects.
|
|
|
|
|
* Alloc and free are done with bpf_mem_{alloc,free}() and the size of
|
|
|
|
|
* the returned object is given by the size argument of bpf_mem_alloc().
|
bpf: Allow per unit prefill for non-fix-size percpu memory allocator
Commit 41a5db8d8161 ("Add support for non-fix-size percpu mem allocation")
added support for non-fix-size percpu memory allocation.
Such allocation will allocate percpu memory for all buckets on all
cpus and the memory consumption is in the order to quadratic.
For example, let us say, 4 cpus, unit size 16 bytes, so each
cpu has 16 * 4 = 64 bytes, with 4 cpus, total will be 64 * 4 = 256 bytes.
Then let us say, 8 cpus with the same unit size, each cpu
has 16 * 8 = 128 bytes, with 8 cpus, total will be 128 * 8 = 1024 bytes.
So if the number of cpus doubles, the number of memory consumption
will be 4 times. So for a system with large number of cpus, the
memory consumption goes up quickly with quadratic order.
For example, for 4KB percpu allocation, 128 cpus. The total memory
consumption will 4KB * 128 * 128 = 64MB. Things will become
worse if the number of cpus is bigger (e.g., 512, 1024, etc.)
In Commit 41a5db8d8161, the non-fix-size percpu memory allocation is
done in boot time, so for system with large number of cpus, the initial
percpu memory consumption is very visible. For example, for 128 cpu
system, the total percpu memory allocation will be at least
(16 + 32 + 64 + 96 + 128 + 196 + 256 + 512 + 1024 + 2048 + 4096)
* 128 * 128 = ~138MB.
which is pretty big. It will be even bigger for larger number of cpus.
Note that the current prefill also allocates 4 entries if the unit size
is less than 256. So on top of 138MB memory consumption, this will
add more consumption with
3 * (16 + 32 + 64 + 96 + 128 + 196 + 256) * 128 * 128 = ~38MB.
Next patch will try to reduce this memory consumption.
Later on, Commit 1fda5bb66ad8 ("bpf: Do not allocate percpu memory
at init stage") moved the non-fix-size percpu memory allocation
to bpf verificaiton stage. Once a particular bpf_percpu_obj_new()
is called by bpf program, the memory allocator will try to fill in
the cache with all sizes, causing the same amount of percpu memory
consumption as in the boot stage.
To reduce the initial percpu memory consumption for non-fix-size
percpu memory allocation, instead of filling the cache with all
supported allocation sizes, this patch intends to fill the cache
only for the requested size. As typically users will not use large
percpu data structure, this can save memory significantly.
For example, the allocation size is 64 bytes with 128 cpus.
Then total percpu memory amount will be 64 * 128 * 128 = 1MB,
much less than previous 138MB.
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Acked-by: Hou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231222031745.1289082-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-12-22 03:17:45 +00:00
|
|
|
* If percpu equals true, error will be returned in order to avoid
|
|
|
|
|
* large memory consumption and the below bpf_mem_alloc_percpu_unit_init()
|
|
|
|
|
* should be used to do on-demand per-cpu allocation for each size.
|
2023-02-16 02:48:21 +00:00
|
|
|
*/
|
2022-09-02 21:10:52 +00:00
|
|
|
int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu);
|
bpf: Allow per unit prefill for non-fix-size percpu memory allocator
Commit 41a5db8d8161 ("Add support for non-fix-size percpu mem allocation")
added support for non-fix-size percpu memory allocation.
Such allocation will allocate percpu memory for all buckets on all
cpus and the memory consumption is in the order to quadratic.
For example, let us say, 4 cpus, unit size 16 bytes, so each
cpu has 16 * 4 = 64 bytes, with 4 cpus, total will be 64 * 4 = 256 bytes.
Then let us say, 8 cpus with the same unit size, each cpu
has 16 * 8 = 128 bytes, with 8 cpus, total will be 128 * 8 = 1024 bytes.
So if the number of cpus doubles, the number of memory consumption
will be 4 times. So for a system with large number of cpus, the
memory consumption goes up quickly with quadratic order.
For example, for 4KB percpu allocation, 128 cpus. The total memory
consumption will 4KB * 128 * 128 = 64MB. Things will become
worse if the number of cpus is bigger (e.g., 512, 1024, etc.)
In Commit 41a5db8d8161, the non-fix-size percpu memory allocation is
done in boot time, so for system with large number of cpus, the initial
percpu memory consumption is very visible. For example, for 128 cpu
system, the total percpu memory allocation will be at least
(16 + 32 + 64 + 96 + 128 + 196 + 256 + 512 + 1024 + 2048 + 4096)
* 128 * 128 = ~138MB.
which is pretty big. It will be even bigger for larger number of cpus.
Note that the current prefill also allocates 4 entries if the unit size
is less than 256. So on top of 138MB memory consumption, this will
add more consumption with
3 * (16 + 32 + 64 + 96 + 128 + 196 + 256) * 128 * 128 = ~38MB.
Next patch will try to reduce this memory consumption.
Later on, Commit 1fda5bb66ad8 ("bpf: Do not allocate percpu memory
at init stage") moved the non-fix-size percpu memory allocation
to bpf verificaiton stage. Once a particular bpf_percpu_obj_new()
is called by bpf program, the memory allocator will try to fill in
the cache with all sizes, causing the same amount of percpu memory
consumption as in the boot stage.
To reduce the initial percpu memory consumption for non-fix-size
percpu memory allocation, instead of filling the cache with all
supported allocation sizes, this patch intends to fill the cache
only for the requested size. As typically users will not use large
percpu data structure, this can save memory significantly.
For example, the allocation size is 64 bytes with 128 cpus.
Then total percpu memory amount will be 64 * 128 * 128 = 1MB,
much less than previous 138MB.
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Acked-by: Hou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231222031745.1289082-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-12-22 03:17:45 +00:00
|
|
|
/* Initialize a non-fix-size percpu memory allocator */
|
|
|
|
|
int bpf_mem_alloc_percpu_init(struct bpf_mem_alloc *ma, struct obj_cgroup *objcg);
|
|
|
|
|
/* The percpu allocation with a specific unit size. */
|
|
|
|
|
int bpf_mem_alloc_percpu_unit_init(struct bpf_mem_alloc *ma, int size);
|
2022-09-02 21:10:43 +00:00
|
|
|
void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma);
|
|
|
|
|
|
|
|
|
|
/* kmalloc/kfree equivalent: */
|
|
|
|
|
void *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size);
|
|
|
|
|
void bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr);
|
2023-07-06 03:34:45 +00:00
|
|
|
void bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr);
|
2022-09-02 21:10:43 +00:00
|
|
|
|
|
|
|
|
/* kmem_cache_alloc/free equivalent: */
|
|
|
|
|
void *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma);
|
|
|
|
|
void bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr);
|
2023-07-06 03:34:45 +00:00
|
|
|
void bpf_mem_cache_free_rcu(struct bpf_mem_alloc *ma, void *ptr);
|
2023-03-22 21:52:42 +00:00
|
|
|
void bpf_mem_cache_raw_free(void *ptr);
|
|
|
|
|
void *bpf_mem_cache_alloc_flags(struct bpf_mem_alloc *ma, gfp_t flags);
|
2022-09-02 21:10:43 +00:00
|
|
|
|
|
|
|
|
#endif /* _BPF_MEM_ALLOC_H */
|