mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-29 05:44:11 +00:00
bpf: Use pcpu_alloc_size() in bpf_mem_free{_rcu}()
For bpf_global_percpu_ma, the pointer passed to bpf_mem_free_rcu() is allocated by kmalloc() and its size is fixed (16-bytes on x86-64). So no matter which cache allocates the dynamic per-cpu area, on x86-64 cache[2] will always be used to free the per-cpu area. Fix the unbalance by checking whether the bpf memory allocator is per-cpu or not and use pcpu_alloc_size() instead of ksize() to find the correct cache for per-cpu free. Signed-off-by: Hou Tao <houtao1@huawei.com> Link: https://lore.kernel.org/r/20231020133202.4043247-5-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
parent
baa8fdecd8
commit
3f2189e4f7
2 changed files with 15 additions and 2 deletions
|
@ -11,6 +11,7 @@ struct bpf_mem_caches;
|
||||||
struct bpf_mem_alloc {
|
struct bpf_mem_alloc {
|
||||||
struct bpf_mem_caches __percpu *caches;
|
struct bpf_mem_caches __percpu *caches;
|
||||||
struct bpf_mem_cache __percpu *cache;
|
struct bpf_mem_cache __percpu *cache;
|
||||||
|
bool percpu;
|
||||||
struct work_struct work;
|
struct work_struct work;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -525,6 +525,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
|
||||||
/* room for llist_node and per-cpu pointer */
|
/* room for llist_node and per-cpu pointer */
|
||||||
if (percpu)
|
if (percpu)
|
||||||
percpu_size = LLIST_NODE_SZ + sizeof(void *);
|
percpu_size = LLIST_NODE_SZ + sizeof(void *);
|
||||||
|
ma->percpu = percpu;
|
||||||
|
|
||||||
if (size) {
|
if (size) {
|
||||||
pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
|
pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
|
||||||
|
@ -874,6 +875,17 @@ void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
|
||||||
return !ret ? NULL : ret + LLIST_NODE_SZ;
|
return !ret ? NULL : ret + LLIST_NODE_SZ;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static notrace int bpf_mem_free_idx(void *ptr, bool percpu)
|
||||||
|
{
|
||||||
|
size_t size;
|
||||||
|
|
||||||
|
if (percpu)
|
||||||
|
size = pcpu_alloc_size(*((void **)ptr));
|
||||||
|
else
|
||||||
|
size = ksize(ptr - LLIST_NODE_SZ);
|
||||||
|
return bpf_mem_cache_idx(size);
|
||||||
|
}
|
||||||
|
|
||||||
void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
|
void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
|
||||||
{
|
{
|
||||||
int idx;
|
int idx;
|
||||||
|
@ -881,7 +893,7 @@ void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
|
||||||
if (!ptr)
|
if (!ptr)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
|
idx = bpf_mem_free_idx(ptr, ma->percpu);
|
||||||
if (idx < 0)
|
if (idx < 0)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -895,7 +907,7 @@ void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
|
||||||
if (!ptr)
|
if (!ptr)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
|
idx = bpf_mem_free_idx(ptr, ma->percpu);
|
||||||
if (idx < 0)
|
if (idx < 0)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue