mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-03 17:40:04 +00:00
![Martin KaFai Lau](/assets/img/avatar_default.png)
This patch introduces a new map type BPF_MAP_TYPE_REUSEPORT_SOCKARRAY. To unleash the full potential of a bpf prog, it is essential for the userspace to be capable of directly setting up a bpf map which can then be consumed by the bpf prog to make decision. In this case, decide which SO_REUSEPORT sk to serve the incoming request. By adding BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, the userspace has total control and visibility on where a SO_REUSEPORT sk should be located in a bpf map. The later patch will introduce BPF_PROG_TYPE_SK_REUSEPORT such that the bpf prog can directly select a sk from the bpf map. That will raise the programmability of the bpf prog attached to a reuseport group (a group of sk serving the same IP:PORT). For example, in UDP, the bpf prog can peek into the payload (e.g. through the "data" pointer introduced in the later patch) to learn the application level's connection information and then decide which sk to pick from a bpf map. The userspace can tightly couple the sk's location in a bpf map with the application logic in generating the UDP payload's connection information. This connection info contact/API stays within the userspace. Also, when used with map-in-map, the userspace can switch the old-server-process's inner map to a new-server-process's inner map in one call "bpf_map_update_elem(outer_map, &index, &new_reuseport_array)". The bpf prog will then direct incoming requests to the new process instead of the old process. The old process can finish draining the pending requests (e.g. by "accept()") before closing the old-fds. [Note that deleting a fd from a bpf map does not necessary mean the fd is closed] During map_update_elem(), Only SO_REUSEPORT sk (i.e. which has already been added to a reuse->socks[]) can be used. That means a SO_REUSEPORT sk that is "bind()" for UDP or "bind()+listen()" for TCP. These conditions are ensured in "reuseport_array_update_check()". A SO_REUSEPORT sk can only be added once to a map (i.e. the same sk cannot be added twice even to the same map). SO_REUSEPORT already allows another sk to be created for the same IP:PORT. There is no need to re-create a similar usage in the BPF side. When a SO_REUSEPORT is deleted from the "reuse->socks[]" (e.g. "close()"), it will notify the bpf map to remove it from the map also. It is done through "bpf_sk_reuseport_detach()" and it will only be called if >=1 of the "reuse->sock[]" has ever been added to a bpf map. The map_update()/map_delete() has to be in-sync with the "reuse->socks[]". Hence, the same "reuseport_lock" used by "reuse->socks[]" has to be used here also. Care has been taken to ensure the lock is only acquired when the adding sk passes some strict tests. and freeing the map does not require the reuseport_lock. The reuseport_array will also support lookup from the syscall side. It will return a sock_gen_cookie(). The sock_gen_cookie() is on-demand (i.e. a sk's cookie is not generated until the very first map_lookup_elem()). The lookup cookie is 64bits but it goes against the logical userspace expectation on 32bits sizeof(fd) (and as other fd based bpf maps do also). It may catch user in surprise if we enforce value_size=8 while userspace still pass a 32bits fd during update. Supporting different value_size between lookup and update seems unintuitive also. We also need to consider what if other existing fd based maps want to return 64bits value from syscall's lookup in the future. Hence, reuseport_array supports both value_size 4 and 8, and assuming user will usually use value_size=4. The syscall's lookup will return ENOSPC on value_size=4. It will will only return 64bits value from sock_gen_cookie() when user consciously choose value_size=8 (as a signal that lookup is desired) which then requires a 64bits value in both lookup and update. Signed-off-by: Martin KaFai Lau <kafai@fb.com> Acked-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
363 lines
8.6 KiB
C
363 lines
8.6 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (c) 2018 Facebook
|
|
*/
|
|
#include <linux/bpf.h>
|
|
#include <linux/err.h>
|
|
#include <linux/sock_diag.h>
|
|
#include <net/sock_reuseport.h>
|
|
|
|
struct reuseport_array {
|
|
struct bpf_map map;
|
|
struct sock __rcu *ptrs[];
|
|
};
|
|
|
|
static struct reuseport_array *reuseport_array(struct bpf_map *map)
|
|
{
|
|
return (struct reuseport_array *)map;
|
|
}
|
|
|
|
/* The caller must hold the reuseport_lock */
|
|
void bpf_sk_reuseport_detach(struct sock *sk)
|
|
{
|
|
struct sock __rcu **socks;
|
|
|
|
write_lock_bh(&sk->sk_callback_lock);
|
|
socks = sk->sk_user_data;
|
|
if (socks) {
|
|
WRITE_ONCE(sk->sk_user_data, NULL);
|
|
/*
|
|
* Do not move this NULL assignment outside of
|
|
* sk->sk_callback_lock because there is
|
|
* a race with reuseport_array_free()
|
|
* which does not hold the reuseport_lock.
|
|
*/
|
|
RCU_INIT_POINTER(*socks, NULL);
|
|
}
|
|
write_unlock_bh(&sk->sk_callback_lock);
|
|
}
|
|
|
|
static int reuseport_array_alloc_check(union bpf_attr *attr)
|
|
{
|
|
if (attr->value_size != sizeof(u32) &&
|
|
attr->value_size != sizeof(u64))
|
|
return -EINVAL;
|
|
|
|
return array_map_alloc_check(attr);
|
|
}
|
|
|
|
static void *reuseport_array_lookup_elem(struct bpf_map *map, void *key)
|
|
{
|
|
struct reuseport_array *array = reuseport_array(map);
|
|
u32 index = *(u32 *)key;
|
|
|
|
if (unlikely(index >= array->map.max_entries))
|
|
return NULL;
|
|
|
|
return rcu_dereference(array->ptrs[index]);
|
|
}
|
|
|
|
/* Called from syscall only */
|
|
static int reuseport_array_delete_elem(struct bpf_map *map, void *key)
|
|
{
|
|
struct reuseport_array *array = reuseport_array(map);
|
|
u32 index = *(u32 *)key;
|
|
struct sock *sk;
|
|
int err;
|
|
|
|
if (index >= map->max_entries)
|
|
return -E2BIG;
|
|
|
|
if (!rcu_access_pointer(array->ptrs[index]))
|
|
return -ENOENT;
|
|
|
|
spin_lock_bh(&reuseport_lock);
|
|
|
|
sk = rcu_dereference_protected(array->ptrs[index],
|
|
lockdep_is_held(&reuseport_lock));
|
|
if (sk) {
|
|
write_lock_bh(&sk->sk_callback_lock);
|
|
WRITE_ONCE(sk->sk_user_data, NULL);
|
|
RCU_INIT_POINTER(array->ptrs[index], NULL);
|
|
write_unlock_bh(&sk->sk_callback_lock);
|
|
err = 0;
|
|
} else {
|
|
err = -ENOENT;
|
|
}
|
|
|
|
spin_unlock_bh(&reuseport_lock);
|
|
|
|
return err;
|
|
}
|
|
|
|
static void reuseport_array_free(struct bpf_map *map)
|
|
{
|
|
struct reuseport_array *array = reuseport_array(map);
|
|
struct sock *sk;
|
|
u32 i;
|
|
|
|
synchronize_rcu();
|
|
|
|
/*
|
|
* ops->map_*_elem() will not be able to access this
|
|
* array now. Hence, this function only races with
|
|
* bpf_sk_reuseport_detach() which was triggerred by
|
|
* close() or disconnect().
|
|
*
|
|
* This function and bpf_sk_reuseport_detach() are
|
|
* both removing sk from "array". Who removes it
|
|
* first does not matter.
|
|
*
|
|
* The only concern here is bpf_sk_reuseport_detach()
|
|
* may access "array" which is being freed here.
|
|
* bpf_sk_reuseport_detach() access this "array"
|
|
* through sk->sk_user_data _and_ with sk->sk_callback_lock
|
|
* held which is enough because this "array" is not freed
|
|
* until all sk->sk_user_data has stopped referencing this "array".
|
|
*
|
|
* Hence, due to the above, taking "reuseport_lock" is not
|
|
* needed here.
|
|
*/
|
|
|
|
/*
|
|
* Since reuseport_lock is not taken, sk is accessed under
|
|
* rcu_read_lock()
|
|
*/
|
|
rcu_read_lock();
|
|
for (i = 0; i < map->max_entries; i++) {
|
|
sk = rcu_dereference(array->ptrs[i]);
|
|
if (sk) {
|
|
write_lock_bh(&sk->sk_callback_lock);
|
|
/*
|
|
* No need for WRITE_ONCE(). At this point,
|
|
* no one is reading it without taking the
|
|
* sk->sk_callback_lock.
|
|
*/
|
|
sk->sk_user_data = NULL;
|
|
write_unlock_bh(&sk->sk_callback_lock);
|
|
RCU_INIT_POINTER(array->ptrs[i], NULL);
|
|
}
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
/*
|
|
* Once reaching here, all sk->sk_user_data is not
|
|
* referenceing this "array". "array" can be freed now.
|
|
*/
|
|
bpf_map_area_free(array);
|
|
}
|
|
|
|
static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
|
|
{
|
|
int err, numa_node = bpf_map_attr_numa_node(attr);
|
|
struct reuseport_array *array;
|
|
u64 cost, array_size;
|
|
|
|
if (!capable(CAP_SYS_ADMIN))
|
|
return ERR_PTR(-EPERM);
|
|
|
|
array_size = sizeof(*array);
|
|
array_size += (u64)attr->max_entries * sizeof(struct sock *);
|
|
|
|
/* make sure there is no u32 overflow later in round_up() */
|
|
cost = array_size;
|
|
if (cost >= U32_MAX - PAGE_SIZE)
|
|
return ERR_PTR(-ENOMEM);
|
|
cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
|
|
|
|
err = bpf_map_precharge_memlock(cost);
|
|
if (err)
|
|
return ERR_PTR(err);
|
|
|
|
/* allocate all map elements and zero-initialize them */
|
|
array = bpf_map_area_alloc(array_size, numa_node);
|
|
if (!array)
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
/* copy mandatory map attributes */
|
|
bpf_map_init_from_attr(&array->map, attr);
|
|
array->map.pages = cost;
|
|
|
|
return &array->map;
|
|
}
|
|
|
|
int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
|
|
void *value)
|
|
{
|
|
struct sock *sk;
|
|
int err;
|
|
|
|
if (map->value_size != sizeof(u64))
|
|
return -ENOSPC;
|
|
|
|
rcu_read_lock();
|
|
sk = reuseport_array_lookup_elem(map, key);
|
|
if (sk) {
|
|
*(u64 *)value = sock_gen_cookie(sk);
|
|
err = 0;
|
|
} else {
|
|
err = -ENOENT;
|
|
}
|
|
rcu_read_unlock();
|
|
|
|
return err;
|
|
}
|
|
|
|
static int
|
|
reuseport_array_update_check(const struct reuseport_array *array,
|
|
const struct sock *nsk,
|
|
const struct sock *osk,
|
|
const struct sock_reuseport *nsk_reuse,
|
|
u32 map_flags)
|
|
{
|
|
if (osk && map_flags == BPF_NOEXIST)
|
|
return -EEXIST;
|
|
|
|
if (!osk && map_flags == BPF_EXIST)
|
|
return -ENOENT;
|
|
|
|
if (nsk->sk_protocol != IPPROTO_UDP && nsk->sk_protocol != IPPROTO_TCP)
|
|
return -ENOTSUPP;
|
|
|
|
if (nsk->sk_family != AF_INET && nsk->sk_family != AF_INET6)
|
|
return -ENOTSUPP;
|
|
|
|
if (nsk->sk_type != SOCK_STREAM && nsk->sk_type != SOCK_DGRAM)
|
|
return -ENOTSUPP;
|
|
|
|
/*
|
|
* sk must be hashed (i.e. listening in the TCP case or binded
|
|
* in the UDP case) and
|
|
* it must also be a SO_REUSEPORT sk (i.e. reuse cannot be NULL).
|
|
*
|
|
* Also, sk will be used in bpf helper that is protected by
|
|
* rcu_read_lock().
|
|
*/
|
|
if (!sock_flag(nsk, SOCK_RCU_FREE) || !sk_hashed(nsk) || !nsk_reuse)
|
|
return -EINVAL;
|
|
|
|
/* READ_ONCE because the sk->sk_callback_lock may not be held here */
|
|
if (READ_ONCE(nsk->sk_user_data))
|
|
return -EBUSY;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Called from syscall only.
|
|
* The "nsk" in the fd refcnt.
|
|
* The "osk" and "reuse" are protected by reuseport_lock.
|
|
*/
|
|
int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
|
|
void *value, u64 map_flags)
|
|
{
|
|
struct reuseport_array *array = reuseport_array(map);
|
|
struct sock *free_osk = NULL, *osk, *nsk;
|
|
struct sock_reuseport *reuse;
|
|
u32 index = *(u32 *)key;
|
|
struct socket *socket;
|
|
int err, fd;
|
|
|
|
if (map_flags > BPF_EXIST)
|
|
return -EINVAL;
|
|
|
|
if (index >= map->max_entries)
|
|
return -E2BIG;
|
|
|
|
if (map->value_size == sizeof(u64)) {
|
|
u64 fd64 = *(u64 *)value;
|
|
|
|
if (fd64 > S32_MAX)
|
|
return -EINVAL;
|
|
fd = fd64;
|
|
} else {
|
|
fd = *(int *)value;
|
|
}
|
|
|
|
socket = sockfd_lookup(fd, &err);
|
|
if (!socket)
|
|
return err;
|
|
|
|
nsk = socket->sk;
|
|
if (!nsk) {
|
|
err = -EINVAL;
|
|
goto put_file;
|
|
}
|
|
|
|
/* Quick checks before taking reuseport_lock */
|
|
err = reuseport_array_update_check(array, nsk,
|
|
rcu_access_pointer(array->ptrs[index]),
|
|
rcu_access_pointer(nsk->sk_reuseport_cb),
|
|
map_flags);
|
|
if (err)
|
|
goto put_file;
|
|
|
|
spin_lock_bh(&reuseport_lock);
|
|
/*
|
|
* Some of the checks only need reuseport_lock
|
|
* but it is done under sk_callback_lock also
|
|
* for simplicity reason.
|
|
*/
|
|
write_lock_bh(&nsk->sk_callback_lock);
|
|
|
|
osk = rcu_dereference_protected(array->ptrs[index],
|
|
lockdep_is_held(&reuseport_lock));
|
|
reuse = rcu_dereference_protected(nsk->sk_reuseport_cb,
|
|
lockdep_is_held(&reuseport_lock));
|
|
err = reuseport_array_update_check(array, nsk, osk, reuse, map_flags);
|
|
if (err)
|
|
goto put_file_unlock;
|
|
|
|
/* Ensure reuse->reuseport_id is set */
|
|
err = reuseport_get_id(reuse);
|
|
if (err < 0)
|
|
goto put_file_unlock;
|
|
|
|
WRITE_ONCE(nsk->sk_user_data, &array->ptrs[index]);
|
|
rcu_assign_pointer(array->ptrs[index], nsk);
|
|
free_osk = osk;
|
|
err = 0;
|
|
|
|
put_file_unlock:
|
|
write_unlock_bh(&nsk->sk_callback_lock);
|
|
|
|
if (free_osk) {
|
|
write_lock_bh(&free_osk->sk_callback_lock);
|
|
WRITE_ONCE(free_osk->sk_user_data, NULL);
|
|
write_unlock_bh(&free_osk->sk_callback_lock);
|
|
}
|
|
|
|
spin_unlock_bh(&reuseport_lock);
|
|
put_file:
|
|
fput(socket->file);
|
|
return err;
|
|
}
|
|
|
|
/* Called from syscall */
|
|
static int reuseport_array_get_next_key(struct bpf_map *map, void *key,
|
|
void *next_key)
|
|
{
|
|
struct reuseport_array *array = reuseport_array(map);
|
|
u32 index = key ? *(u32 *)key : U32_MAX;
|
|
u32 *next = (u32 *)next_key;
|
|
|
|
if (index >= array->map.max_entries) {
|
|
*next = 0;
|
|
return 0;
|
|
}
|
|
|
|
if (index == array->map.max_entries - 1)
|
|
return -ENOENT;
|
|
|
|
*next = index + 1;
|
|
return 0;
|
|
}
|
|
|
|
const struct bpf_map_ops reuseport_array_ops = {
|
|
.map_alloc_check = reuseport_array_alloc_check,
|
|
.map_alloc = reuseport_array_alloc,
|
|
.map_free = reuseport_array_free,
|
|
.map_lookup_elem = reuseport_array_lookup_elem,
|
|
.map_get_next_key = reuseport_array_get_next_key,
|
|
.map_delete_elem = reuseport_array_delete_elem,
|
|
};
|