2020-08-25 18:29:16 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
/*
|
|
|
|
* Copyright (c) 2019 Facebook
|
|
|
|
* Copyright 2020 Google LLC.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _BPF_LOCAL_STORAGE_H
|
|
|
|
#define _BPF_LOCAL_STORAGE_H
|
|
|
|
|
|
|
|
#include <linux/bpf.h>
|
2021-12-29 00:49:13 +00:00
|
|
|
#include <linux/filter.h>
|
2020-08-25 18:29:16 +00:00
|
|
|
#include <linux/rculist.h>
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/hash.h>
|
|
|
|
#include <linux/types.h>
|
bpf: Use bpf_mem_cache_alloc/free in bpf_local_storage_elem
This patch uses bpf_mem_alloc for the task and cgroup local storage that
the bpf prog can easily get a hold of the storage owner's PTR_TO_BTF_ID.
eg. bpf_get_current_task_btf() can be used in some of the kmalloc code
path which will cause deadlock/recursion. bpf_mem_cache_alloc is
deadlock free and will solve a legit use case in [1].
For sk storage, its batch creation benchmark shows a few percent
regression when the sk create/destroy batch size is larger than 32.
The sk creation/destruction happens much more often and
depends on external traffic. Considering it is hypothetical
to be able to cause deadlock with sk storage, it can cross
the bridge to use bpf_mem_alloc till a legit (ie. useful)
use case comes up.
For inode storage, bpf_local_storage_destroy() is called before
waiting for a rcu gp and its memory cannot be reused immediately.
inode stays with kmalloc/kfree after the rcu [or tasks_trace] gp.
A 'bool bpf_ma' argument is added to bpf_local_storage_map_alloc().
Only task and cgroup storage have 'bpf_ma == true' which
means to use bpf_mem_cache_alloc/free(). This patch only changes
selem to use bpf_mem_alloc for task and cgroup. The next patch
will change the local_storage to use bpf_mem_alloc also for
task and cgroup.
Here is some more details on the changes:
* memory allocation:
After bpf_mem_cache_alloc(), the SDATA(selem)->data is zero-ed because
bpf_mem_cache_alloc() could return a reused selem. It is to keep
the existing bpf_map_kzalloc() behavior. Only SDATA(selem)->data
is zero-ed. SDATA(selem)->data is the visible part to the bpf prog.
No need to use zero_map_value() to do the zeroing because
bpf_selem_free(..., reuse_now = true) ensures no bpf prog is using
the selem before returning the selem through bpf_mem_cache_free().
For the internal fields of selem, they will be initialized when
linking to the new smap and the new local_storage.
When 'bpf_ma == false', nothing changes in this patch. It will
stay with the bpf_map_kzalloc().
* memory free:
The bpf_selem_free() and bpf_selem_free_rcu() are modified to handle
the bpf_ma == true case.
For the common selem free path where its owner is also being destroyed,
the mem is freed in bpf_local_storage_destroy(), the owner (task
and cgroup) has gone through a rcu gp. The memory can be reused
immediately, so bpf_local_storage_destroy() will call
bpf_selem_free(..., reuse_now = true) which will do
bpf_mem_cache_free() for immediate reuse consideration.
An exception is the delete elem code path. The delete elem code path
is called from the helper bpf_*_storage_delete() and the syscall
bpf_map_delete_elem(). This path is an unusual case for local
storage because the common use case is to have the local storage
staying with its owner life time so that the bpf prog and the user
space does not have to monitor the owner's destruction. For the delete
elem path, the selem cannot be reused immediately because there could
be bpf prog using it. It will call bpf_selem_free(..., reuse_now = false)
and it will wait for a rcu tasks trace gp before freeing the elem. The
rcu callback is changed to do bpf_mem_cache_raw_free() instead of kfree().
When 'bpf_ma == false', it should be the same as before.
__bpf_selem_free() is added to do the kfree_rcu and call_tasks_trace_rcu().
A few words on the 'reuse_now == true'. When 'reuse_now == true',
it is still racing with bpf_local_storage_map_free which is under rcu
protection, so it still needs to wait for a rcu gp instead of kfree().
Otherwise, the selem may be reused by slab for a totally different struct
while the bpf_local_storage_map_free() is still using it (as a
rcu reader). For the inode case, there may be other rcu readers also.
In short, when bpf_ma == false and reuse_now == true => vanilla rcu.
[1]: https://lore.kernel.org/bpf/20221118190109.1512674-1-namhyung@kernel.org/
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230322215246.1675516-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 21:52:43 +00:00
|
|
|
#include <linux/bpf_mem_alloc.h>
|
2020-08-25 18:29:16 +00:00
|
|
|
#include <uapi/linux/btf.h>
|
|
|
|
|
|
|
|
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
|
|
|
|
|
2021-12-24 15:29:15 +00:00
|
|
|
#define bpf_rcu_lock_held() \
|
|
|
|
(rcu_read_lock_held() || rcu_read_lock_trace_held() || \
|
|
|
|
rcu_read_lock_bh_held())
|
2020-08-25 18:29:16 +00:00
|
|
|
struct bpf_local_storage_map_bucket {
|
|
|
|
struct hlist_head list;
|
|
|
|
raw_spinlock_t lock;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Thp map is not the primary owner of a bpf_local_storage_elem.
|
|
|
|
* Instead, the container object (eg. sk->sk_bpf_storage) is.
|
|
|
|
*
|
|
|
|
* The map (bpf_local_storage_map) is for two purposes
|
|
|
|
* 1. Define the size of the "local storage". It is
|
|
|
|
* the map's value_size.
|
|
|
|
*
|
|
|
|
* 2. Maintain a list to keep track of all elems such
|
|
|
|
* that they can be cleaned up during the map destruction.
|
|
|
|
*
|
|
|
|
* When a bpf local storage is being looked up for a
|
|
|
|
* particular object, the "bpf_map" pointer is actually used
|
|
|
|
* as the "key" to search in the list of elem in
|
|
|
|
* the respective bpf_local_storage owned by the object.
|
|
|
|
*
|
|
|
|
* e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
|
|
|
|
* as the searching key.
|
|
|
|
*/
|
|
|
|
struct bpf_local_storage_map {
|
|
|
|
struct bpf_map map;
|
|
|
|
/* Lookup elem does not require accessing the map.
|
|
|
|
*
|
|
|
|
* Updating/Deleting requires a bucket lock to
|
|
|
|
* link/unlink the elem from the map. Having
|
|
|
|
* multiple buckets to improve contention.
|
|
|
|
*/
|
|
|
|
struct bpf_local_storage_map_bucket *buckets;
|
|
|
|
u32 bucket_log;
|
|
|
|
u16 elem_size;
|
|
|
|
u16 cache_idx;
|
bpf: Use bpf_mem_cache_alloc/free in bpf_local_storage_elem
This patch uses bpf_mem_alloc for the task and cgroup local storage that
the bpf prog can easily get a hold of the storage owner's PTR_TO_BTF_ID.
eg. bpf_get_current_task_btf() can be used in some of the kmalloc code
path which will cause deadlock/recursion. bpf_mem_cache_alloc is
deadlock free and will solve a legit use case in [1].
For sk storage, its batch creation benchmark shows a few percent
regression when the sk create/destroy batch size is larger than 32.
The sk creation/destruction happens much more often and
depends on external traffic. Considering it is hypothetical
to be able to cause deadlock with sk storage, it can cross
the bridge to use bpf_mem_alloc till a legit (ie. useful)
use case comes up.
For inode storage, bpf_local_storage_destroy() is called before
waiting for a rcu gp and its memory cannot be reused immediately.
inode stays with kmalloc/kfree after the rcu [or tasks_trace] gp.
A 'bool bpf_ma' argument is added to bpf_local_storage_map_alloc().
Only task and cgroup storage have 'bpf_ma == true' which
means to use bpf_mem_cache_alloc/free(). This patch only changes
selem to use bpf_mem_alloc for task and cgroup. The next patch
will change the local_storage to use bpf_mem_alloc also for
task and cgroup.
Here is some more details on the changes:
* memory allocation:
After bpf_mem_cache_alloc(), the SDATA(selem)->data is zero-ed because
bpf_mem_cache_alloc() could return a reused selem. It is to keep
the existing bpf_map_kzalloc() behavior. Only SDATA(selem)->data
is zero-ed. SDATA(selem)->data is the visible part to the bpf prog.
No need to use zero_map_value() to do the zeroing because
bpf_selem_free(..., reuse_now = true) ensures no bpf prog is using
the selem before returning the selem through bpf_mem_cache_free().
For the internal fields of selem, they will be initialized when
linking to the new smap and the new local_storage.
When 'bpf_ma == false', nothing changes in this patch. It will
stay with the bpf_map_kzalloc().
* memory free:
The bpf_selem_free() and bpf_selem_free_rcu() are modified to handle
the bpf_ma == true case.
For the common selem free path where its owner is also being destroyed,
the mem is freed in bpf_local_storage_destroy(), the owner (task
and cgroup) has gone through a rcu gp. The memory can be reused
immediately, so bpf_local_storage_destroy() will call
bpf_selem_free(..., reuse_now = true) which will do
bpf_mem_cache_free() for immediate reuse consideration.
An exception is the delete elem code path. The delete elem code path
is called from the helper bpf_*_storage_delete() and the syscall
bpf_map_delete_elem(). This path is an unusual case for local
storage because the common use case is to have the local storage
staying with its owner life time so that the bpf prog and the user
space does not have to monitor the owner's destruction. For the delete
elem path, the selem cannot be reused immediately because there could
be bpf prog using it. It will call bpf_selem_free(..., reuse_now = false)
and it will wait for a rcu tasks trace gp before freeing the elem. The
rcu callback is changed to do bpf_mem_cache_raw_free() instead of kfree().
When 'bpf_ma == false', it should be the same as before.
__bpf_selem_free() is added to do the kfree_rcu and call_tasks_trace_rcu().
A few words on the 'reuse_now == true'. When 'reuse_now == true',
it is still racing with bpf_local_storage_map_free which is under rcu
protection, so it still needs to wait for a rcu gp instead of kfree().
Otherwise, the selem may be reused by slab for a totally different struct
while the bpf_local_storage_map_free() is still using it (as a
rcu reader). For the inode case, there may be other rcu readers also.
In short, when bpf_ma == false and reuse_now == true => vanilla rcu.
[1]: https://lore.kernel.org/bpf/20221118190109.1512674-1-namhyung@kernel.org/
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230322215246.1675516-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 21:52:43 +00:00
|
|
|
struct bpf_mem_alloc selem_ma;
|
2023-03-22 21:52:44 +00:00
|
|
|
struct bpf_mem_alloc storage_ma;
|
bpf: Use bpf_mem_cache_alloc/free in bpf_local_storage_elem
This patch uses bpf_mem_alloc for the task and cgroup local storage that
the bpf prog can easily get a hold of the storage owner's PTR_TO_BTF_ID.
eg. bpf_get_current_task_btf() can be used in some of the kmalloc code
path which will cause deadlock/recursion. bpf_mem_cache_alloc is
deadlock free and will solve a legit use case in [1].
For sk storage, its batch creation benchmark shows a few percent
regression when the sk create/destroy batch size is larger than 32.
The sk creation/destruction happens much more often and
depends on external traffic. Considering it is hypothetical
to be able to cause deadlock with sk storage, it can cross
the bridge to use bpf_mem_alloc till a legit (ie. useful)
use case comes up.
For inode storage, bpf_local_storage_destroy() is called before
waiting for a rcu gp and its memory cannot be reused immediately.
inode stays with kmalloc/kfree after the rcu [or tasks_trace] gp.
A 'bool bpf_ma' argument is added to bpf_local_storage_map_alloc().
Only task and cgroup storage have 'bpf_ma == true' which
means to use bpf_mem_cache_alloc/free(). This patch only changes
selem to use bpf_mem_alloc for task and cgroup. The next patch
will change the local_storage to use bpf_mem_alloc also for
task and cgroup.
Here is some more details on the changes:
* memory allocation:
After bpf_mem_cache_alloc(), the SDATA(selem)->data is zero-ed because
bpf_mem_cache_alloc() could return a reused selem. It is to keep
the existing bpf_map_kzalloc() behavior. Only SDATA(selem)->data
is zero-ed. SDATA(selem)->data is the visible part to the bpf prog.
No need to use zero_map_value() to do the zeroing because
bpf_selem_free(..., reuse_now = true) ensures no bpf prog is using
the selem before returning the selem through bpf_mem_cache_free().
For the internal fields of selem, they will be initialized when
linking to the new smap and the new local_storage.
When 'bpf_ma == false', nothing changes in this patch. It will
stay with the bpf_map_kzalloc().
* memory free:
The bpf_selem_free() and bpf_selem_free_rcu() are modified to handle
the bpf_ma == true case.
For the common selem free path where its owner is also being destroyed,
the mem is freed in bpf_local_storage_destroy(), the owner (task
and cgroup) has gone through a rcu gp. The memory can be reused
immediately, so bpf_local_storage_destroy() will call
bpf_selem_free(..., reuse_now = true) which will do
bpf_mem_cache_free() for immediate reuse consideration.
An exception is the delete elem code path. The delete elem code path
is called from the helper bpf_*_storage_delete() and the syscall
bpf_map_delete_elem(). This path is an unusual case for local
storage because the common use case is to have the local storage
staying with its owner life time so that the bpf prog and the user
space does not have to monitor the owner's destruction. For the delete
elem path, the selem cannot be reused immediately because there could
be bpf prog using it. It will call bpf_selem_free(..., reuse_now = false)
and it will wait for a rcu tasks trace gp before freeing the elem. The
rcu callback is changed to do bpf_mem_cache_raw_free() instead of kfree().
When 'bpf_ma == false', it should be the same as before.
__bpf_selem_free() is added to do the kfree_rcu and call_tasks_trace_rcu().
A few words on the 'reuse_now == true'. When 'reuse_now == true',
it is still racing with bpf_local_storage_map_free which is under rcu
protection, so it still needs to wait for a rcu gp instead of kfree().
Otherwise, the selem may be reused by slab for a totally different struct
while the bpf_local_storage_map_free() is still using it (as a
rcu reader). For the inode case, there may be other rcu readers also.
In short, when bpf_ma == false and reuse_now == true => vanilla rcu.
[1]: https://lore.kernel.org/bpf/20221118190109.1512674-1-namhyung@kernel.org/
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230322215246.1675516-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 21:52:43 +00:00
|
|
|
bool bpf_ma;
|
2020-08-25 18:29:16 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct bpf_local_storage_data {
|
|
|
|
/* smap is used as the searching key when looking up
|
|
|
|
* from the object's bpf_local_storage.
|
|
|
|
*
|
|
|
|
* Put it in the same cacheline as the data to minimize
|
2021-05-25 02:56:59 +00:00
|
|
|
* the number of cachelines accessed during the cache hit case.
|
2020-08-25 18:29:16 +00:00
|
|
|
*/
|
|
|
|
struct bpf_local_storage_map __rcu *smap;
|
|
|
|
u8 data[] __aligned(8);
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Linked to bpf_local_storage and bpf_local_storage_map */
|
|
|
|
struct bpf_local_storage_elem {
|
|
|
|
struct hlist_node map_node; /* Linked to bpf_local_storage_map */
|
|
|
|
struct hlist_node snode; /* Linked to bpf_local_storage */
|
|
|
|
struct bpf_local_storage __rcu *local_storage;
|
|
|
|
struct rcu_head rcu;
|
|
|
|
/* 8 bytes hole */
|
2021-05-25 02:56:59 +00:00
|
|
|
/* The data is stored in another cacheline to minimize
|
2020-08-25 18:29:16 +00:00
|
|
|
* the number of cachelines access during a cache hit.
|
|
|
|
*/
|
|
|
|
struct bpf_local_storage_data sdata ____cacheline_aligned;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct bpf_local_storage {
|
|
|
|
struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
|
2023-03-08 06:59:24 +00:00
|
|
|
struct bpf_local_storage_map __rcu *smap;
|
2020-08-25 18:29:16 +00:00
|
|
|
struct hlist_head list; /* List of bpf_local_storage_elem */
|
|
|
|
void *owner; /* The object that owns the above "list" of
|
|
|
|
* bpf_local_storage_elem.
|
|
|
|
*/
|
|
|
|
struct rcu_head rcu;
|
|
|
|
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
|
|
|
|
};
|
|
|
|
|
|
|
|
/* U16_MAX is much more than enough for sk local storage
|
|
|
|
* considering a tcp_sock is ~2k.
|
|
|
|
*/
|
|
|
|
#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
|
|
|
|
min_t(u32, \
|
|
|
|
(KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
|
|
|
|
sizeof(struct bpf_local_storage_elem)), \
|
|
|
|
(U16_MAX - sizeof(struct bpf_local_storage_elem)))
|
|
|
|
|
|
|
|
#define SELEM(_SDATA) \
|
|
|
|
container_of((_SDATA), struct bpf_local_storage_elem, sdata)
|
|
|
|
#define SDATA(_SELEM) (&(_SELEM)->sdata)
|
|
|
|
|
|
|
|
#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
|
|
|
|
|
|
|
|
struct bpf_local_storage_cache {
|
|
|
|
spinlock_t idx_lock;
|
|
|
|
u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
#define DEFINE_BPF_STORAGE_CACHE(name) \
|
|
|
|
static struct bpf_local_storage_cache name = { \
|
|
|
|
.idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Helper functions for bpf_local_storage */
|
|
|
|
int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
|
|
|
|
|
2022-10-26 04:28:45 +00:00
|
|
|
struct bpf_map *
|
|
|
|
bpf_local_storage_map_alloc(union bpf_attr *attr,
|
bpf: Use bpf_mem_cache_alloc/free in bpf_local_storage_elem
This patch uses bpf_mem_alloc for the task and cgroup local storage that
the bpf prog can easily get a hold of the storage owner's PTR_TO_BTF_ID.
eg. bpf_get_current_task_btf() can be used in some of the kmalloc code
path which will cause deadlock/recursion. bpf_mem_cache_alloc is
deadlock free and will solve a legit use case in [1].
For sk storage, its batch creation benchmark shows a few percent
regression when the sk create/destroy batch size is larger than 32.
The sk creation/destruction happens much more often and
depends on external traffic. Considering it is hypothetical
to be able to cause deadlock with sk storage, it can cross
the bridge to use bpf_mem_alloc till a legit (ie. useful)
use case comes up.
For inode storage, bpf_local_storage_destroy() is called before
waiting for a rcu gp and its memory cannot be reused immediately.
inode stays with kmalloc/kfree after the rcu [or tasks_trace] gp.
A 'bool bpf_ma' argument is added to bpf_local_storage_map_alloc().
Only task and cgroup storage have 'bpf_ma == true' which
means to use bpf_mem_cache_alloc/free(). This patch only changes
selem to use bpf_mem_alloc for task and cgroup. The next patch
will change the local_storage to use bpf_mem_alloc also for
task and cgroup.
Here is some more details on the changes:
* memory allocation:
After bpf_mem_cache_alloc(), the SDATA(selem)->data is zero-ed because
bpf_mem_cache_alloc() could return a reused selem. It is to keep
the existing bpf_map_kzalloc() behavior. Only SDATA(selem)->data
is zero-ed. SDATA(selem)->data is the visible part to the bpf prog.
No need to use zero_map_value() to do the zeroing because
bpf_selem_free(..., reuse_now = true) ensures no bpf prog is using
the selem before returning the selem through bpf_mem_cache_free().
For the internal fields of selem, they will be initialized when
linking to the new smap and the new local_storage.
When 'bpf_ma == false', nothing changes in this patch. It will
stay with the bpf_map_kzalloc().
* memory free:
The bpf_selem_free() and bpf_selem_free_rcu() are modified to handle
the bpf_ma == true case.
For the common selem free path where its owner is also being destroyed,
the mem is freed in bpf_local_storage_destroy(), the owner (task
and cgroup) has gone through a rcu gp. The memory can be reused
immediately, so bpf_local_storage_destroy() will call
bpf_selem_free(..., reuse_now = true) which will do
bpf_mem_cache_free() for immediate reuse consideration.
An exception is the delete elem code path. The delete elem code path
is called from the helper bpf_*_storage_delete() and the syscall
bpf_map_delete_elem(). This path is an unusual case for local
storage because the common use case is to have the local storage
staying with its owner life time so that the bpf prog and the user
space does not have to monitor the owner's destruction. For the delete
elem path, the selem cannot be reused immediately because there could
be bpf prog using it. It will call bpf_selem_free(..., reuse_now = false)
and it will wait for a rcu tasks trace gp before freeing the elem. The
rcu callback is changed to do bpf_mem_cache_raw_free() instead of kfree().
When 'bpf_ma == false', it should be the same as before.
__bpf_selem_free() is added to do the kfree_rcu and call_tasks_trace_rcu().
A few words on the 'reuse_now == true'. When 'reuse_now == true',
it is still racing with bpf_local_storage_map_free which is under rcu
protection, so it still needs to wait for a rcu gp instead of kfree().
Otherwise, the selem may be reused by slab for a totally different struct
while the bpf_local_storage_map_free() is still using it (as a
rcu reader). For the inode case, there may be other rcu readers also.
In short, when bpf_ma == false and reuse_now == true => vanilla rcu.
[1]: https://lore.kernel.org/bpf/20221118190109.1512674-1-namhyung@kernel.org/
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230322215246.1675516-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 21:52:43 +00:00
|
|
|
struct bpf_local_storage_cache *cache,
|
|
|
|
bool bpf_ma);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
|
|
|
struct bpf_local_storage_data *
|
|
|
|
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
|
|
|
|
struct bpf_local_storage_map *smap,
|
|
|
|
bool cacheit_lockit);
|
|
|
|
|
2023-03-08 06:59:21 +00:00
|
|
|
void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);
|
2022-10-26 04:28:45 +00:00
|
|
|
|
|
|
|
void bpf_local_storage_map_free(struct bpf_map *map,
|
|
|
|
struct bpf_local_storage_cache *cache,
|
2021-02-25 23:43:15 +00:00
|
|
|
int __percpu *busy_counter);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
|
|
|
int bpf_local_storage_map_check_btf(const struct bpf_map *map,
|
|
|
|
const struct btf *btf,
|
|
|
|
const struct btf_type *key_type,
|
|
|
|
const struct btf_type *value_type);
|
|
|
|
|
|
|
|
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
|
|
|
|
struct bpf_local_storage_elem *selem);
|
|
|
|
|
2023-03-08 06:59:25 +00:00
|
|
|
void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
|
|
|
void bpf_selem_link_map(struct bpf_local_storage_map *smap,
|
|
|
|
struct bpf_local_storage_elem *selem);
|
|
|
|
|
|
|
|
struct bpf_local_storage_elem *
|
|
|
|
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
|
2022-03-18 04:55:52 +00:00
|
|
|
bool charge_mem, gfp_t gfp_flags);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
2023-03-08 06:59:28 +00:00
|
|
|
void bpf_selem_free(struct bpf_local_storage_elem *selem,
|
|
|
|
struct bpf_local_storage_map *smap,
|
|
|
|
bool reuse_now);
|
|
|
|
|
2020-08-25 18:29:16 +00:00
|
|
|
int
|
|
|
|
bpf_local_storage_alloc(void *owner,
|
|
|
|
struct bpf_local_storage_map *smap,
|
2022-03-18 04:55:52 +00:00
|
|
|
struct bpf_local_storage_elem *first_selem,
|
|
|
|
gfp_t gfp_flags);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
|
|
|
struct bpf_local_storage_data *
|
|
|
|
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
|
2022-03-18 04:55:52 +00:00
|
|
|
void *value, u64 map_flags, gfp_t gfp_flags);
|
2020-08-25 18:29:16 +00:00
|
|
|
|
2023-03-05 12:46:11 +00:00
|
|
|
u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
|
2021-12-24 15:29:15 +00:00
|
|
|
|
2020-08-25 18:29:16 +00:00
|
|
|
#endif /* _BPF_LOCAL_STORAGE_H */
|