mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 11:19:37 +00:00
![Guo Zhengkui](/assets/img/avatar_default.png)
Fix the following coccicheck warning: tools/lib/bpf/bpf.c:114:31-32: WARNING: Use ARRAY_SIZE tools/lib/bpf/xsk.c:484:34-35: WARNING: Use ARRAY_SIZE tools/lib/bpf/xsk.c:485:35-36: WARNING: Use ARRAY_SIZE It has been tested with gcc (Debian 8.3.0-6) 8.3.0 on x86_64. Signed-off-by: Guo Zhengkui <guozhengkui@vivo.com> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20220306023426.19324-1-guozhengkui@vivo.com
1260 lines
28 KiB
C
1260 lines
28 KiB
C
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
|
|
|
|
/*
|
|
* AF_XDP user-space access library.
|
|
*
|
|
* Copyright(c) 2018 - 2019 Intel Corporation.
|
|
*
|
|
* Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
|
|
*/
|
|
|
|
#include <errno.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
#include <arpa/inet.h>
|
|
#include <asm/barrier.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/ethtool.h>
|
|
#include <linux/filter.h>
|
|
#include <linux/if_ether.h>
|
|
#include <linux/if_packet.h>
|
|
#include <linux/if_xdp.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/list.h>
|
|
#include <linux/sockios.h>
|
|
#include <net/if.h>
|
|
#include <sys/ioctl.h>
|
|
#include <sys/mman.h>
|
|
#include <sys/socket.h>
|
|
#include <sys/types.h>
|
|
#include <linux/if_link.h>
|
|
|
|
#include "bpf.h"
|
|
#include "libbpf.h"
|
|
#include "libbpf_internal.h"
|
|
#include "xsk.h"
|
|
|
|
/* entire xsk.h and xsk.c is going away in libbpf 1.0, so ignore all internal
|
|
* uses of deprecated APIs
|
|
*/
|
|
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
|
|
|
|
#ifndef SOL_XDP
|
|
#define SOL_XDP 283
|
|
#endif
|
|
|
|
#ifndef AF_XDP
|
|
#define AF_XDP 44
|
|
#endif
|
|
|
|
#ifndef PF_XDP
|
|
#define PF_XDP AF_XDP
|
|
#endif
|
|
|
|
enum xsk_prog {
|
|
XSK_PROG_FALLBACK,
|
|
XSK_PROG_REDIRECT_FLAGS,
|
|
};
|
|
|
|
struct xsk_umem {
|
|
struct xsk_ring_prod *fill_save;
|
|
struct xsk_ring_cons *comp_save;
|
|
char *umem_area;
|
|
struct xsk_umem_config config;
|
|
int fd;
|
|
int refcount;
|
|
struct list_head ctx_list;
|
|
bool rx_ring_setup_done;
|
|
bool tx_ring_setup_done;
|
|
};
|
|
|
|
struct xsk_ctx {
|
|
struct xsk_ring_prod *fill;
|
|
struct xsk_ring_cons *comp;
|
|
__u32 queue_id;
|
|
struct xsk_umem *umem;
|
|
int refcount;
|
|
int ifindex;
|
|
struct list_head list;
|
|
int prog_fd;
|
|
int link_fd;
|
|
int xsks_map_fd;
|
|
char ifname[IFNAMSIZ];
|
|
bool has_bpf_link;
|
|
};
|
|
|
|
struct xsk_socket {
|
|
struct xsk_ring_cons *rx;
|
|
struct xsk_ring_prod *tx;
|
|
__u64 outstanding_tx;
|
|
struct xsk_ctx *ctx;
|
|
struct xsk_socket_config config;
|
|
int fd;
|
|
};
|
|
|
|
struct xsk_nl_info {
|
|
bool xdp_prog_attached;
|
|
int ifindex;
|
|
int fd;
|
|
};
|
|
|
|
/* Up until and including Linux 5.3 */
|
|
struct xdp_ring_offset_v1 {
|
|
__u64 producer;
|
|
__u64 consumer;
|
|
__u64 desc;
|
|
};
|
|
|
|
/* Up until and including Linux 5.3 */
|
|
struct xdp_mmap_offsets_v1 {
|
|
struct xdp_ring_offset_v1 rx;
|
|
struct xdp_ring_offset_v1 tx;
|
|
struct xdp_ring_offset_v1 fr;
|
|
struct xdp_ring_offset_v1 cr;
|
|
};
|
|
|
|
int xsk_umem__fd(const struct xsk_umem *umem)
|
|
{
|
|
return umem ? umem->fd : -EINVAL;
|
|
}
|
|
|
|
int xsk_socket__fd(const struct xsk_socket *xsk)
|
|
{
|
|
return xsk ? xsk->fd : -EINVAL;
|
|
}
|
|
|
|
static bool xsk_page_aligned(void *buffer)
|
|
{
|
|
unsigned long addr = (unsigned long)buffer;
|
|
|
|
return !(addr & (getpagesize() - 1));
|
|
}
|
|
|
|
static void xsk_set_umem_config(struct xsk_umem_config *cfg,
|
|
const struct xsk_umem_config *usr_cfg)
|
|
{
|
|
if (!usr_cfg) {
|
|
cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
|
|
cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
|
|
cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
|
|
cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
|
|
cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
|
|
return;
|
|
}
|
|
|
|
cfg->fill_size = usr_cfg->fill_size;
|
|
cfg->comp_size = usr_cfg->comp_size;
|
|
cfg->frame_size = usr_cfg->frame_size;
|
|
cfg->frame_headroom = usr_cfg->frame_headroom;
|
|
cfg->flags = usr_cfg->flags;
|
|
}
|
|
|
|
static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
|
|
const struct xsk_socket_config *usr_cfg)
|
|
{
|
|
if (!usr_cfg) {
|
|
cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
|
|
cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
|
|
cfg->libbpf_flags = 0;
|
|
cfg->xdp_flags = 0;
|
|
cfg->bind_flags = 0;
|
|
return 0;
|
|
}
|
|
|
|
if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
|
|
return -EINVAL;
|
|
|
|
cfg->rx_size = usr_cfg->rx_size;
|
|
cfg->tx_size = usr_cfg->tx_size;
|
|
cfg->libbpf_flags = usr_cfg->libbpf_flags;
|
|
cfg->xdp_flags = usr_cfg->xdp_flags;
|
|
cfg->bind_flags = usr_cfg->bind_flags;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
|
|
{
|
|
struct xdp_mmap_offsets_v1 off_v1;
|
|
|
|
/* getsockopt on a kernel <= 5.3 has no flags fields.
|
|
* Copy over the offsets to the correct places in the >=5.4 format
|
|
* and put the flags where they would have been on that kernel.
|
|
*/
|
|
memcpy(&off_v1, off, sizeof(off_v1));
|
|
|
|
off->rx.producer = off_v1.rx.producer;
|
|
off->rx.consumer = off_v1.rx.consumer;
|
|
off->rx.desc = off_v1.rx.desc;
|
|
off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
|
|
|
|
off->tx.producer = off_v1.tx.producer;
|
|
off->tx.consumer = off_v1.tx.consumer;
|
|
off->tx.desc = off_v1.tx.desc;
|
|
off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
|
|
|
|
off->fr.producer = off_v1.fr.producer;
|
|
off->fr.consumer = off_v1.fr.consumer;
|
|
off->fr.desc = off_v1.fr.desc;
|
|
off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
|
|
|
|
off->cr.producer = off_v1.cr.producer;
|
|
off->cr.consumer = off_v1.cr.consumer;
|
|
off->cr.desc = off_v1.cr.desc;
|
|
off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
|
|
}
|
|
|
|
static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
|
|
{
|
|
socklen_t optlen;
|
|
int err;
|
|
|
|
optlen = sizeof(*off);
|
|
err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
|
|
if (err)
|
|
return err;
|
|
|
|
if (optlen == sizeof(*off))
|
|
return 0;
|
|
|
|
if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
|
|
xsk_mmap_offsets_v1(off);
|
|
return 0;
|
|
}
|
|
|
|
return -EINVAL;
|
|
}
|
|
|
|
static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
|
|
struct xsk_ring_prod *fill,
|
|
struct xsk_ring_cons *comp)
|
|
{
|
|
struct xdp_mmap_offsets off;
|
|
void *map;
|
|
int err;
|
|
|
|
err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
|
|
&umem->config.fill_size,
|
|
sizeof(umem->config.fill_size));
|
|
if (err)
|
|
return -errno;
|
|
|
|
err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
|
|
&umem->config.comp_size,
|
|
sizeof(umem->config.comp_size));
|
|
if (err)
|
|
return -errno;
|
|
|
|
err = xsk_get_mmap_offsets(fd, &off);
|
|
if (err)
|
|
return -errno;
|
|
|
|
map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
|
|
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
|
|
XDP_UMEM_PGOFF_FILL_RING);
|
|
if (map == MAP_FAILED)
|
|
return -errno;
|
|
|
|
fill->mask = umem->config.fill_size - 1;
|
|
fill->size = umem->config.fill_size;
|
|
fill->producer = map + off.fr.producer;
|
|
fill->consumer = map + off.fr.consumer;
|
|
fill->flags = map + off.fr.flags;
|
|
fill->ring = map + off.fr.desc;
|
|
fill->cached_cons = umem->config.fill_size;
|
|
|
|
map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
|
|
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
|
|
XDP_UMEM_PGOFF_COMPLETION_RING);
|
|
if (map == MAP_FAILED) {
|
|
err = -errno;
|
|
goto out_mmap;
|
|
}
|
|
|
|
comp->mask = umem->config.comp_size - 1;
|
|
comp->size = umem->config.comp_size;
|
|
comp->producer = map + off.cr.producer;
|
|
comp->consumer = map + off.cr.consumer;
|
|
comp->flags = map + off.cr.flags;
|
|
comp->ring = map + off.cr.desc;
|
|
|
|
return 0;
|
|
|
|
out_mmap:
|
|
munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
|
|
return err;
|
|
}
|
|
|
|
DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
|
|
int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
|
|
__u64 size, struct xsk_ring_prod *fill,
|
|
struct xsk_ring_cons *comp,
|
|
const struct xsk_umem_config *usr_config)
|
|
{
|
|
struct xdp_umem_reg mr;
|
|
struct xsk_umem *umem;
|
|
int err;
|
|
|
|
if (!umem_area || !umem_ptr || !fill || !comp)
|
|
return -EFAULT;
|
|
if (!size && !xsk_page_aligned(umem_area))
|
|
return -EINVAL;
|
|
|
|
umem = calloc(1, sizeof(*umem));
|
|
if (!umem)
|
|
return -ENOMEM;
|
|
|
|
umem->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
|
|
if (umem->fd < 0) {
|
|
err = -errno;
|
|
goto out_umem_alloc;
|
|
}
|
|
|
|
umem->umem_area = umem_area;
|
|
INIT_LIST_HEAD(&umem->ctx_list);
|
|
xsk_set_umem_config(&umem->config, usr_config);
|
|
|
|
memset(&mr, 0, sizeof(mr));
|
|
mr.addr = (uintptr_t)umem_area;
|
|
mr.len = size;
|
|
mr.chunk_size = umem->config.frame_size;
|
|
mr.headroom = umem->config.frame_headroom;
|
|
mr.flags = umem->config.flags;
|
|
|
|
err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
|
|
if (err) {
|
|
err = -errno;
|
|
goto out_socket;
|
|
}
|
|
|
|
err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
|
|
if (err)
|
|
goto out_socket;
|
|
|
|
umem->fill_save = fill;
|
|
umem->comp_save = comp;
|
|
*umem_ptr = umem;
|
|
return 0;
|
|
|
|
out_socket:
|
|
close(umem->fd);
|
|
out_umem_alloc:
|
|
free(umem);
|
|
return err;
|
|
}
|
|
|
|
struct xsk_umem_config_v1 {
|
|
__u32 fill_size;
|
|
__u32 comp_size;
|
|
__u32 frame_size;
|
|
__u32 frame_headroom;
|
|
};
|
|
|
|
COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
|
|
int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,
|
|
__u64 size, struct xsk_ring_prod *fill,
|
|
struct xsk_ring_cons *comp,
|
|
const struct xsk_umem_config *usr_config)
|
|
{
|
|
struct xsk_umem_config config;
|
|
|
|
memcpy(&config, usr_config, sizeof(struct xsk_umem_config_v1));
|
|
config.flags = 0;
|
|
|
|
return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,
|
|
&config);
|
|
}
|
|
|
|
static enum xsk_prog get_xsk_prog(void)
|
|
{
|
|
enum xsk_prog detected = XSK_PROG_FALLBACK;
|
|
__u32 size_out, retval, duration;
|
|
char data_in = 0, data_out;
|
|
struct bpf_insn insns[] = {
|
|
BPF_LD_MAP_FD(BPF_REG_1, 0),
|
|
BPF_MOV64_IMM(BPF_REG_2, 0),
|
|
BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
|
|
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
|
BPF_EXIT_INSN(),
|
|
};
|
|
int prog_fd, map_fd, ret, insn_cnt = ARRAY_SIZE(insns);
|
|
|
|
map_fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, NULL, sizeof(int), sizeof(int), 1, NULL);
|
|
if (map_fd < 0)
|
|
return detected;
|
|
|
|
insns[0].imm = map_fd;
|
|
|
|
prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
|
|
if (prog_fd < 0) {
|
|
close(map_fd);
|
|
return detected;
|
|
}
|
|
|
|
ret = bpf_prog_test_run(prog_fd, 0, &data_in, 1, &data_out, &size_out, &retval, &duration);
|
|
if (!ret && retval == XDP_PASS)
|
|
detected = XSK_PROG_REDIRECT_FLAGS;
|
|
close(prog_fd);
|
|
close(map_fd);
|
|
return detected;
|
|
}
|
|
|
|
static int xsk_load_xdp_prog(struct xsk_socket *xsk)
|
|
{
|
|
static const int log_buf_size = 16 * 1024;
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
char log_buf[log_buf_size];
|
|
int prog_fd;
|
|
|
|
/* This is the fallback C-program:
|
|
* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
|
|
* {
|
|
* int ret, index = ctx->rx_queue_index;
|
|
*
|
|
* // A set entry here means that the correspnding queue_id
|
|
* // has an active AF_XDP socket bound to it.
|
|
* ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
|
|
* if (ret > 0)
|
|
* return ret;
|
|
*
|
|
* // Fallback for pre-5.3 kernels, not supporting default
|
|
* // action in the flags parameter.
|
|
* if (bpf_map_lookup_elem(&xsks_map, &index))
|
|
* return bpf_redirect_map(&xsks_map, index, 0);
|
|
* return XDP_PASS;
|
|
* }
|
|
*/
|
|
struct bpf_insn prog[] = {
|
|
/* r2 = *(u32 *)(r1 + 16) */
|
|
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
|
|
/* *(u32 *)(r10 - 4) = r2 */
|
|
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
|
|
/* r1 = xskmap[] */
|
|
BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
|
|
/* r3 = XDP_PASS */
|
|
BPF_MOV64_IMM(BPF_REG_3, 2),
|
|
/* call bpf_redirect_map */
|
|
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
|
/* if w0 != 0 goto pc+13 */
|
|
BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
|
|
/* r2 = r10 */
|
|
BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
|
|
/* r2 += -4 */
|
|
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
|
|
/* r1 = xskmap[] */
|
|
BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
|
|
/* call bpf_map_lookup_elem */
|
|
BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
|
|
/* r1 = r0 */
|
|
BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
|
|
/* r0 = XDP_PASS */
|
|
BPF_MOV64_IMM(BPF_REG_0, 2),
|
|
/* if r1 == 0 goto pc+5 */
|
|
BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
|
|
/* r2 = *(u32 *)(r10 - 4) */
|
|
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
|
|
/* r1 = xskmap[] */
|
|
BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
|
|
/* r3 = 0 */
|
|
BPF_MOV64_IMM(BPF_REG_3, 0),
|
|
/* call bpf_redirect_map */
|
|
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
|
/* The jumps are to this instruction */
|
|
BPF_EXIT_INSN(),
|
|
};
|
|
|
|
/* This is the post-5.3 kernel C-program:
|
|
* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
|
|
* {
|
|
* return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
|
|
* }
|
|
*/
|
|
struct bpf_insn prog_redirect_flags[] = {
|
|
/* r2 = *(u32 *)(r1 + 16) */
|
|
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
|
|
/* r1 = xskmap[] */
|
|
BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
|
|
/* r3 = XDP_PASS */
|
|
BPF_MOV64_IMM(BPF_REG_3, 2),
|
|
/* call bpf_redirect_map */
|
|
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
|
|
BPF_EXIT_INSN(),
|
|
};
|
|
size_t insns_cnt[] = {ARRAY_SIZE(prog),
|
|
ARRAY_SIZE(prog_redirect_flags),
|
|
};
|
|
struct bpf_insn *progs[] = {prog, prog_redirect_flags};
|
|
enum xsk_prog option = get_xsk_prog();
|
|
LIBBPF_OPTS(bpf_prog_load_opts, opts,
|
|
.log_buf = log_buf,
|
|
.log_size = log_buf_size,
|
|
);
|
|
|
|
prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "LGPL-2.1 or BSD-2-Clause",
|
|
progs[option], insns_cnt[option], &opts);
|
|
if (prog_fd < 0) {
|
|
pr_warn("BPF log buffer:\n%s", log_buf);
|
|
return prog_fd;
|
|
}
|
|
|
|
ctx->prog_fd = prog_fd;
|
|
return 0;
|
|
}
|
|
|
|
static int xsk_create_bpf_link(struct xsk_socket *xsk)
|
|
{
|
|
DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
__u32 prog_id = 0;
|
|
int link_fd;
|
|
int err;
|
|
|
|
err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
|
|
if (err) {
|
|
pr_warn("getting XDP prog id failed\n");
|
|
return err;
|
|
}
|
|
|
|
/* if there's a netlink-based XDP prog loaded on interface, bail out
|
|
* and ask user to do the removal by himself
|
|
*/
|
|
if (prog_id) {
|
|
pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_REPLACE);
|
|
|
|
link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
|
|
if (link_fd < 0) {
|
|
pr_warn("bpf_link_create failed: %s\n", strerror(errno));
|
|
return link_fd;
|
|
}
|
|
|
|
ctx->link_fd = link_fd;
|
|
return 0;
|
|
}
|
|
|
|
static int xsk_get_max_queues(struct xsk_socket *xsk)
|
|
{
|
|
struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
struct ifreq ifr = {};
|
|
int fd, err, ret;
|
|
|
|
fd = socket(AF_LOCAL, SOCK_DGRAM | SOCK_CLOEXEC, 0);
|
|
if (fd < 0)
|
|
return -errno;
|
|
|
|
ifr.ifr_data = (void *)&channels;
|
|
libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ);
|
|
err = ioctl(fd, SIOCETHTOOL, &ifr);
|
|
if (err && errno != EOPNOTSUPP) {
|
|
ret = -errno;
|
|
goto out;
|
|
}
|
|
|
|
if (err) {
|
|
/* If the device says it has no channels, then all traffic
|
|
* is sent to a single stream, so max queues = 1.
|
|
*/
|
|
ret = 1;
|
|
} else {
|
|
/* Take the max of rx, tx, combined. Drivers return
|
|
* the number of channels in different ways.
|
|
*/
|
|
ret = max(channels.max_rx, channels.max_tx);
|
|
ret = max(ret, (int)channels.max_combined);
|
|
}
|
|
|
|
out:
|
|
close(fd);
|
|
return ret;
|
|
}
|
|
|
|
static int xsk_create_bpf_maps(struct xsk_socket *xsk)
|
|
{
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
int max_queues;
|
|
int fd;
|
|
|
|
max_queues = xsk_get_max_queues(xsk);
|
|
if (max_queues < 0)
|
|
return max_queues;
|
|
|
|
fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, "xsks_map",
|
|
sizeof(int), sizeof(int), max_queues, NULL);
|
|
if (fd < 0)
|
|
return fd;
|
|
|
|
ctx->xsks_map_fd = fd;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
|
|
{
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
|
|
bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
|
|
close(ctx->xsks_map_fd);
|
|
}
|
|
|
|
static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
|
|
{
|
|
__u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
|
|
__u32 map_len = sizeof(struct bpf_map_info);
|
|
struct bpf_prog_info prog_info = {};
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
struct bpf_map_info map_info;
|
|
int fd, err;
|
|
|
|
err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
|
|
if (err)
|
|
return err;
|
|
|
|
num_maps = prog_info.nr_map_ids;
|
|
|
|
map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
|
|
if (!map_ids)
|
|
return -ENOMEM;
|
|
|
|
memset(&prog_info, 0, prog_len);
|
|
prog_info.nr_map_ids = num_maps;
|
|
prog_info.map_ids = (__u64)(unsigned long)map_ids;
|
|
|
|
err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
|
|
if (err)
|
|
goto out_map_ids;
|
|
|
|
ctx->xsks_map_fd = -1;
|
|
|
|
for (i = 0; i < prog_info.nr_map_ids; i++) {
|
|
fd = bpf_map_get_fd_by_id(map_ids[i]);
|
|
if (fd < 0)
|
|
continue;
|
|
|
|
memset(&map_info, 0, map_len);
|
|
err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
|
|
if (err) {
|
|
close(fd);
|
|
continue;
|
|
}
|
|
|
|
if (!strncmp(map_info.name, "xsks_map", sizeof(map_info.name))) {
|
|
ctx->xsks_map_fd = fd;
|
|
break;
|
|
}
|
|
|
|
close(fd);
|
|
}
|
|
|
|
if (ctx->xsks_map_fd == -1)
|
|
err = -ENOENT;
|
|
|
|
out_map_ids:
|
|
free(map_ids);
|
|
return err;
|
|
}
|
|
|
|
static int xsk_set_bpf_maps(struct xsk_socket *xsk)
|
|
{
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
|
|
return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
|
|
&xsk->fd, 0);
|
|
}
|
|
|
|
static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd)
|
|
{
|
|
struct bpf_link_info link_info;
|
|
__u32 link_len;
|
|
__u32 id = 0;
|
|
int err;
|
|
int fd;
|
|
|
|
while (true) {
|
|
err = bpf_link_get_next_id(id, &id);
|
|
if (err) {
|
|
if (errno == ENOENT) {
|
|
err = 0;
|
|
break;
|
|
}
|
|
pr_warn("can't get next link: %s\n", strerror(errno));
|
|
break;
|
|
}
|
|
|
|
fd = bpf_link_get_fd_by_id(id);
|
|
if (fd < 0) {
|
|
if (errno == ENOENT)
|
|
continue;
|
|
pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
|
|
err = -errno;
|
|
break;
|
|
}
|
|
|
|
link_len = sizeof(struct bpf_link_info);
|
|
memset(&link_info, 0, link_len);
|
|
err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
|
|
if (err) {
|
|
pr_warn("can't get link info: %s\n", strerror(errno));
|
|
close(fd);
|
|
break;
|
|
}
|
|
if (link_info.type == BPF_LINK_TYPE_XDP) {
|
|
if (link_info.xdp.ifindex == ifindex) {
|
|
*link_fd = fd;
|
|
if (prog_id)
|
|
*prog_id = link_info.prog_id;
|
|
break;
|
|
}
|
|
}
|
|
close(fd);
|
|
}
|
|
|
|
return err;
|
|
}
|
|
|
|
static bool xsk_probe_bpf_link(void)
|
|
{
|
|
LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = XDP_FLAGS_SKB_MODE);
|
|
struct bpf_insn insns[2] = {
|
|
BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
|
|
BPF_EXIT_INSN()
|
|
};
|
|
int prog_fd, link_fd = -1, insn_cnt = ARRAY_SIZE(insns);
|
|
int ifindex_lo = 1;
|
|
bool ret = false;
|
|
int err;
|
|
|
|
err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
|
|
if (err)
|
|
return ret;
|
|
|
|
if (link_fd >= 0)
|
|
return true;
|
|
|
|
prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
|
|
if (prog_fd < 0)
|
|
return ret;
|
|
|
|
link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
|
|
close(prog_fd);
|
|
|
|
if (link_fd >= 0) {
|
|
ret = true;
|
|
close(link_fd);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
|
|
{
|
|
char ifname[IFNAMSIZ];
|
|
struct xsk_ctx *ctx;
|
|
char *interface;
|
|
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
if (!ctx)
|
|
return -ENOMEM;
|
|
|
|
interface = if_indextoname(ifindex, &ifname[0]);
|
|
if (!interface) {
|
|
free(ctx);
|
|
return -errno;
|
|
}
|
|
|
|
ctx->ifindex = ifindex;
|
|
libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
|
|
|
|
xsk->ctx = ctx;
|
|
xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int xsk_init_xdp_res(struct xsk_socket *xsk,
|
|
int *xsks_map_fd)
|
|
{
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
int err;
|
|
|
|
err = xsk_create_bpf_maps(xsk);
|
|
if (err)
|
|
return err;
|
|
|
|
err = xsk_load_xdp_prog(xsk);
|
|
if (err)
|
|
goto err_load_xdp_prog;
|
|
|
|
if (ctx->has_bpf_link)
|
|
err = xsk_create_bpf_link(xsk);
|
|
else
|
|
err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, ctx->prog_fd,
|
|
xsk->config.xdp_flags);
|
|
|
|
if (err)
|
|
goto err_attach_xdp_prog;
|
|
|
|
if (!xsk->rx)
|
|
return err;
|
|
|
|
err = xsk_set_bpf_maps(xsk);
|
|
if (err)
|
|
goto err_set_bpf_maps;
|
|
|
|
return err;
|
|
|
|
err_set_bpf_maps:
|
|
if (ctx->has_bpf_link)
|
|
close(ctx->link_fd);
|
|
else
|
|
bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
|
|
err_attach_xdp_prog:
|
|
close(ctx->prog_fd);
|
|
err_load_xdp_prog:
|
|
xsk_delete_bpf_maps(xsk);
|
|
return err;
|
|
}
|
|
|
|
static int xsk_lookup_xdp_res(struct xsk_socket *xsk, int *xsks_map_fd, int prog_id)
|
|
{
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
int err;
|
|
|
|
ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
|
|
if (ctx->prog_fd < 0) {
|
|
err = -errno;
|
|
goto err_prog_fd;
|
|
}
|
|
err = xsk_lookup_bpf_maps(xsk);
|
|
if (err)
|
|
goto err_lookup_maps;
|
|
|
|
if (!xsk->rx)
|
|
return err;
|
|
|
|
err = xsk_set_bpf_maps(xsk);
|
|
if (err)
|
|
goto err_set_maps;
|
|
|
|
return err;
|
|
|
|
err_set_maps:
|
|
close(ctx->xsks_map_fd);
|
|
err_lookup_maps:
|
|
close(ctx->prog_fd);
|
|
err_prog_fd:
|
|
if (ctx->has_bpf_link)
|
|
close(ctx->link_fd);
|
|
return err;
|
|
}
|
|
|
|
static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp, int *xsks_map_fd)
|
|
{
|
|
struct xsk_socket *xsk = _xdp;
|
|
struct xsk_ctx *ctx = xsk->ctx;
|
|
__u32 prog_id = 0;
|
|
int err;
|
|
|
|
if (ctx->has_bpf_link)
|
|
err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
|
|
else
|
|
err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
|
|
|
|
if (err)
|
|
return err;
|
|
|
|
err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
|
|
xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
|
|
|
|
if (!err && xsks_map_fd)
|
|
*xsks_map_fd = ctx->xsks_map_fd;
|
|
|
|
return err;
|
|
}
|
|
|
|
static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
|
|
__u32 queue_id)
|
|
{
|
|
struct xsk_ctx *ctx;
|
|
|
|
if (list_empty(&umem->ctx_list))
|
|
return NULL;
|
|
|
|
list_for_each_entry(ctx, &umem->ctx_list, list) {
|
|
if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
|
|
ctx->refcount++;
|
|
return ctx;
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
|
|
{
|
|
struct xsk_umem *umem = ctx->umem;
|
|
struct xdp_mmap_offsets off;
|
|
int err;
|
|
|
|
if (--ctx->refcount)
|
|
return;
|
|
|
|
if (!unmap)
|
|
goto out_free;
|
|
|
|
err = xsk_get_mmap_offsets(umem->fd, &off);
|
|
if (err)
|
|
goto out_free;
|
|
|
|
munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
|
|
sizeof(__u64));
|
|
munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
|
|
sizeof(__u64));
|
|
|
|
out_free:
|
|
list_del(&ctx->list);
|
|
free(ctx);
|
|
}
|
|
|
|
static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
|
|
struct xsk_umem *umem, int ifindex,
|
|
const char *ifname, __u32 queue_id,
|
|
struct xsk_ring_prod *fill,
|
|
struct xsk_ring_cons *comp)
|
|
{
|
|
struct xsk_ctx *ctx;
|
|
int err;
|
|
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
if (!ctx)
|
|
return NULL;
|
|
|
|
if (!umem->fill_save) {
|
|
err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
|
|
if (err) {
|
|
free(ctx);
|
|
return NULL;
|
|
}
|
|
} else if (umem->fill_save != fill || umem->comp_save != comp) {
|
|
/* Copy over rings to new structs. */
|
|
memcpy(fill, umem->fill_save, sizeof(*fill));
|
|
memcpy(comp, umem->comp_save, sizeof(*comp));
|
|
}
|
|
|
|
ctx->ifindex = ifindex;
|
|
ctx->refcount = 1;
|
|
ctx->umem = umem;
|
|
ctx->queue_id = queue_id;
|
|
libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
|
|
|
|
ctx->fill = fill;
|
|
ctx->comp = comp;
|
|
list_add(&ctx->list, &umem->ctx_list);
|
|
return ctx;
|
|
}
|
|
|
|
static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
|
|
{
|
|
free(xsk->ctx);
|
|
free(xsk);
|
|
}
|
|
|
|
int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
|
|
{
|
|
xsk->ctx->xsks_map_fd = fd;
|
|
return xsk_set_bpf_maps(xsk);
|
|
}
|
|
|
|
int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
|
|
{
|
|
struct xsk_socket *xsk;
|
|
int res;
|
|
|
|
xsk = calloc(1, sizeof(*xsk));
|
|
if (!xsk)
|
|
return -ENOMEM;
|
|
|
|
res = xsk_create_xsk_struct(ifindex, xsk);
|
|
if (res) {
|
|
free(xsk);
|
|
return -EINVAL;
|
|
}
|
|
|
|
res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
|
|
|
|
xsk_destroy_xsk_struct(xsk);
|
|
|
|
return res;
|
|
}
|
|
|
|
int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
|
|
const char *ifname,
|
|
__u32 queue_id, struct xsk_umem *umem,
|
|
struct xsk_ring_cons *rx,
|
|
struct xsk_ring_prod *tx,
|
|
struct xsk_ring_prod *fill,
|
|
struct xsk_ring_cons *comp,
|
|
const struct xsk_socket_config *usr_config)
|
|
{
|
|
bool unmap, rx_setup_done = false, tx_setup_done = false;
|
|
void *rx_map = NULL, *tx_map = NULL;
|
|
struct sockaddr_xdp sxdp = {};
|
|
struct xdp_mmap_offsets off;
|
|
struct xsk_socket *xsk;
|
|
struct xsk_ctx *ctx;
|
|
int err, ifindex;
|
|
|
|
if (!umem || !xsk_ptr || !(rx || tx))
|
|
return -EFAULT;
|
|
|
|
unmap = umem->fill_save != fill;
|
|
|
|
xsk = calloc(1, sizeof(*xsk));
|
|
if (!xsk)
|
|
return -ENOMEM;
|
|
|
|
err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
|
|
if (err)
|
|
goto out_xsk_alloc;
|
|
|
|
xsk->outstanding_tx = 0;
|
|
ifindex = if_nametoindex(ifname);
|
|
if (!ifindex) {
|
|
err = -errno;
|
|
goto out_xsk_alloc;
|
|
}
|
|
|
|
if (umem->refcount++ > 0) {
|
|
xsk->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
|
|
if (xsk->fd < 0) {
|
|
err = -errno;
|
|
goto out_xsk_alloc;
|
|
}
|
|
} else {
|
|
xsk->fd = umem->fd;
|
|
rx_setup_done = umem->rx_ring_setup_done;
|
|
tx_setup_done = umem->tx_ring_setup_done;
|
|
}
|
|
|
|
ctx = xsk_get_ctx(umem, ifindex, queue_id);
|
|
if (!ctx) {
|
|
if (!fill || !comp) {
|
|
err = -EFAULT;
|
|
goto out_socket;
|
|
}
|
|
|
|
ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
|
|
fill, comp);
|
|
if (!ctx) {
|
|
err = -ENOMEM;
|
|
goto out_socket;
|
|
}
|
|
}
|
|
xsk->ctx = ctx;
|
|
xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
|
|
|
|
if (rx && !rx_setup_done) {
|
|
err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
|
|
&xsk->config.rx_size,
|
|
sizeof(xsk->config.rx_size));
|
|
if (err) {
|
|
err = -errno;
|
|
goto out_put_ctx;
|
|
}
|
|
if (xsk->fd == umem->fd)
|
|
umem->rx_ring_setup_done = true;
|
|
}
|
|
if (tx && !tx_setup_done) {
|
|
err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
|
|
&xsk->config.tx_size,
|
|
sizeof(xsk->config.tx_size));
|
|
if (err) {
|
|
err = -errno;
|
|
goto out_put_ctx;
|
|
}
|
|
if (xsk->fd == umem->fd)
|
|
umem->tx_ring_setup_done = true;
|
|
}
|
|
|
|
err = xsk_get_mmap_offsets(xsk->fd, &off);
|
|
if (err) {
|
|
err = -errno;
|
|
goto out_put_ctx;
|
|
}
|
|
|
|
if (rx) {
|
|
rx_map = mmap(NULL, off.rx.desc +
|
|
xsk->config.rx_size * sizeof(struct xdp_desc),
|
|
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
|
|
xsk->fd, XDP_PGOFF_RX_RING);
|
|
if (rx_map == MAP_FAILED) {
|
|
err = -errno;
|
|
goto out_put_ctx;
|
|
}
|
|
|
|
rx->mask = xsk->config.rx_size - 1;
|
|
rx->size = xsk->config.rx_size;
|
|
rx->producer = rx_map + off.rx.producer;
|
|
rx->consumer = rx_map + off.rx.consumer;
|
|
rx->flags = rx_map + off.rx.flags;
|
|
rx->ring = rx_map + off.rx.desc;
|
|
rx->cached_prod = *rx->producer;
|
|
rx->cached_cons = *rx->consumer;
|
|
}
|
|
xsk->rx = rx;
|
|
|
|
if (tx) {
|
|
tx_map = mmap(NULL, off.tx.desc +
|
|
xsk->config.tx_size * sizeof(struct xdp_desc),
|
|
PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
|
|
xsk->fd, XDP_PGOFF_TX_RING);
|
|
if (tx_map == MAP_FAILED) {
|
|
err = -errno;
|
|
goto out_mmap_rx;
|
|
}
|
|
|
|
tx->mask = xsk->config.tx_size - 1;
|
|
tx->size = xsk->config.tx_size;
|
|
tx->producer = tx_map + off.tx.producer;
|
|
tx->consumer = tx_map + off.tx.consumer;
|
|
tx->flags = tx_map + off.tx.flags;
|
|
tx->ring = tx_map + off.tx.desc;
|
|
tx->cached_prod = *tx->producer;
|
|
/* cached_cons is r->size bigger than the real consumer pointer
|
|
* See xsk_prod_nb_free
|
|
*/
|
|
tx->cached_cons = *tx->consumer + xsk->config.tx_size;
|
|
}
|
|
xsk->tx = tx;
|
|
|
|
sxdp.sxdp_family = PF_XDP;
|
|
sxdp.sxdp_ifindex = ctx->ifindex;
|
|
sxdp.sxdp_queue_id = ctx->queue_id;
|
|
if (umem->refcount > 1) {
|
|
sxdp.sxdp_flags |= XDP_SHARED_UMEM;
|
|
sxdp.sxdp_shared_umem_fd = umem->fd;
|
|
} else {
|
|
sxdp.sxdp_flags = xsk->config.bind_flags;
|
|
}
|
|
|
|
err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
|
|
if (err) {
|
|
err = -errno;
|
|
goto out_mmap_tx;
|
|
}
|
|
|
|
ctx->prog_fd = -1;
|
|
|
|
if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
|
|
err = __xsk_setup_xdp_prog(xsk, NULL);
|
|
if (err)
|
|
goto out_mmap_tx;
|
|
}
|
|
|
|
*xsk_ptr = xsk;
|
|
umem->fill_save = NULL;
|
|
umem->comp_save = NULL;
|
|
return 0;
|
|
|
|
out_mmap_tx:
|
|
if (tx)
|
|
munmap(tx_map, off.tx.desc +
|
|
xsk->config.tx_size * sizeof(struct xdp_desc));
|
|
out_mmap_rx:
|
|
if (rx)
|
|
munmap(rx_map, off.rx.desc +
|
|
xsk->config.rx_size * sizeof(struct xdp_desc));
|
|
out_put_ctx:
|
|
xsk_put_ctx(ctx, unmap);
|
|
out_socket:
|
|
if (--umem->refcount)
|
|
close(xsk->fd);
|
|
out_xsk_alloc:
|
|
free(xsk);
|
|
return err;
|
|
}
|
|
|
|
int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
|
|
__u32 queue_id, struct xsk_umem *umem,
|
|
struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
|
|
const struct xsk_socket_config *usr_config)
|
|
{
|
|
if (!umem)
|
|
return -EFAULT;
|
|
|
|
return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
|
|
rx, tx, umem->fill_save,
|
|
umem->comp_save, usr_config);
|
|
}
|
|
|
|
int xsk_umem__delete(struct xsk_umem *umem)
|
|
{
|
|
struct xdp_mmap_offsets off;
|
|
int err;
|
|
|
|
if (!umem)
|
|
return 0;
|
|
|
|
if (umem->refcount)
|
|
return -EBUSY;
|
|
|
|
err = xsk_get_mmap_offsets(umem->fd, &off);
|
|
if (!err && umem->fill_save && umem->comp_save) {
|
|
munmap(umem->fill_save->ring - off.fr.desc,
|
|
off.fr.desc + umem->config.fill_size * sizeof(__u64));
|
|
munmap(umem->comp_save->ring - off.cr.desc,
|
|
off.cr.desc + umem->config.comp_size * sizeof(__u64));
|
|
}
|
|
|
|
close(umem->fd);
|
|
free(umem);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void xsk_socket__delete(struct xsk_socket *xsk)
|
|
{
|
|
size_t desc_sz = sizeof(struct xdp_desc);
|
|
struct xdp_mmap_offsets off;
|
|
struct xsk_umem *umem;
|
|
struct xsk_ctx *ctx;
|
|
int err;
|
|
|
|
if (!xsk)
|
|
return;
|
|
|
|
ctx = xsk->ctx;
|
|
umem = ctx->umem;
|
|
if (ctx->prog_fd != -1) {
|
|
xsk_delete_bpf_maps(xsk);
|
|
close(ctx->prog_fd);
|
|
if (ctx->has_bpf_link)
|
|
close(ctx->link_fd);
|
|
}
|
|
|
|
err = xsk_get_mmap_offsets(xsk->fd, &off);
|
|
if (!err) {
|
|
if (xsk->rx) {
|
|
munmap(xsk->rx->ring - off.rx.desc,
|
|
off.rx.desc + xsk->config.rx_size * desc_sz);
|
|
}
|
|
if (xsk->tx) {
|
|
munmap(xsk->tx->ring - off.tx.desc,
|
|
off.tx.desc + xsk->config.tx_size * desc_sz);
|
|
}
|
|
}
|
|
|
|
xsk_put_ctx(ctx, true);
|
|
|
|
umem->refcount--;
|
|
/* Do not close an fd that also has an associated umem connected
|
|
* to it.
|
|
*/
|
|
if (xsk->fd != umem->fd)
|
|
close(xsk->fd);
|
|
free(xsk);
|
|
}
|