bpf-next-for-netdev
-----BEGIN PGP SIGNATURE----- iHUEABYIAB0WIQTFp0I1jqZrAX+hPRXbK58LschIgwUCZgHylwAKCRDbK58LschI gzmaAPwKhDFFSU/DU08k22muJxLIXVR7Xx04baJ9mPiFrqZyyAEA8RFNamC7wZIB AnfwwoDjfDTP60rlXFaEf8UT5PpA7Ao= =/KF6 -----END PGP SIGNATURE----- Merge tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next Daniel Borkmann says: ==================== pull-request: bpf-next 2024-03-25 We've added 38 non-merge commits during the last 13 day(s) which contain a total of 50 files changed, 867 insertions(+), 274 deletions(-). The main changes are: 1) Add the ability to specify and retrieve BPF cookie also for raw tracepoint programs in order to ease migration from classic to raw tracepoints, from Andrii Nakryiko. 2) Allow the use of bpf_get_{ns_,}current_pid_tgid() helper for all program types and add additional BPF selftests, from Yonghong Song. 3) Several improvements to bpftool and its build, for example, enabling libbpf logs when loading pid_iter in debug mode, from Quentin Monnet. 4) Check the return code of all BPF-related set_memory_*() functions during load and bail out in case they fail, from Christophe Leroy. 5) Avoid a goto in regs_refine_cond_op() such that the verifier can be better integrated into Agni tool which doesn't support backedges yet, from Harishankar Vishwanathan. 6) Add a small BPF trie perf improvement by always inlining longest_prefix_match, from Jesper Dangaard Brouer. 7) Small BPF selftest refactor in bpf_tcp_ca.c to utilize start_server() helper instead of open-coding it, from Geliang Tang. 8) Improve test_tc_tunnel.sh BPF selftest to prevent client connect before the server bind, from Alessandro Carminati. 9) Fix BPF selftest benchmark for older glibc and use syscall(SYS_gettid) instead of gettid(), from Alan Maguire. 10) Implement a backward-compatible method for struct_ops types with additional fields which are not present in older kernels, from Kui-Feng Lee. 11) Add a small helper to check if an instruction is addr_space_cast from as(0) to as(1) and utilize it in x86-64 JIT, from Puranjay Mohan. 12) Small cleanup to remove unnecessary error check in bpf_struct_ops_map_update_elem, from Martin KaFai Lau. 13) Improvements to libbpf fd validity checks for BPF map/programs, from Mykyta Yatsenko. * tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (38 commits) selftests/bpf: Fix flaky test btf_map_in_map/lookup_update bpf: implement insn_is_cast_user() helper for JITs bpf: Avoid get_kernel_nofault() to fetch kprobe entry IP selftests/bpf: Use start_server in bpf_tcp_ca bpf: Sync uapi bpf.h to tools directory libbpf: Add new sec_def "sk_skb/verdict" selftests/bpf: Mark uprobe trigger functions with nocf_check attribute selftests/bpf: Use syscall(SYS_gettid) instead of gettid() wrapper in bench bpf-next: Avoid goto in regs_refine_cond_op() bpftool: Clean up HOST_CFLAGS, HOST_LDFLAGS for bootstrap bpftool selftests/bpf: scale benchmark counting by using per-CPU counters bpftool: Remove unnecessary source files from bootstrap version bpftool: Enable libbpf logs when loading pid_iter in debug mode selftests/bpf: add raw_tp/tp_btf BPF cookie subtests libbpf: add support for BPF cookie for raw_tp/tp_btf programs bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs bpf: pass whole link instead of prog when triggering raw tracepoint bpf: flatten bpf_probe_register call chain selftests/bpf: Prevent client connect before server bind in test_tc_tunnel.sh selftests/bpf: Add a sk_msg prog bpf_get_ns_current_pid_tgid() test ... ==================== Link: https://lore.kernel.org/r/20240325233940.7154-1-daniel@iogearbox.net Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski
commit
2a702c2e57
|
|
@ -2222,28 +2222,21 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
/* If building the body of the JITed code fails somehow,
|
||||
* we fall back to the interpretation.
|
||||
*/
|
||||
if (build_body(&ctx) < 0) {
|
||||
image_ptr = NULL;
|
||||
bpf_jit_binary_free(header);
|
||||
prog = orig_prog;
|
||||
goto out_imms;
|
||||
}
|
||||
if (build_body(&ctx) < 0)
|
||||
goto out_free;
|
||||
build_epilogue(&ctx);
|
||||
|
||||
/* 3.) Extra pass to validate JITed Code */
|
||||
if (validate_code(&ctx)) {
|
||||
image_ptr = NULL;
|
||||
bpf_jit_binary_free(header);
|
||||
prog = orig_prog;
|
||||
goto out_imms;
|
||||
}
|
||||
if (validate_code(&ctx))
|
||||
goto out_free;
|
||||
flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
|
||||
|
||||
if (bpf_jit_enable > 1)
|
||||
/* there are 2 passes here */
|
||||
bpf_jit_dump(prog->len, image_size, 2, ctx.target);
|
||||
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
if (bpf_jit_binary_lock_ro(header))
|
||||
goto out_free;
|
||||
prog->bpf_func = (void *)ctx.target;
|
||||
prog->jited = 1;
|
||||
prog->jited_len = image_size;
|
||||
|
|
@ -2260,5 +2253,11 @@ out:
|
|||
bpf_jit_prog_release_other(prog, prog == orig_prog ?
|
||||
tmp : orig_prog);
|
||||
return prog;
|
||||
|
||||
out_free:
|
||||
image_ptr = NULL;
|
||||
bpf_jit_binary_free(header);
|
||||
prog = orig_prog;
|
||||
goto out_imms;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2176,12 +2176,9 @@ void arch_free_bpf_trampoline(void *image, unsigned int size)
|
|||
bpf_prog_pack_free(image, size);
|
||||
}
|
||||
|
||||
void arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
}
|
||||
|
||||
void arch_unprotect_bpf_trampoline(void *image, unsigned int size)
|
||||
int arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,
|
||||
|
|
|
|||
|
|
@ -1294,16 +1294,19 @@ skip_init_ctx:
|
|||
flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
|
||||
|
||||
if (!prog->is_func || extra_pass) {
|
||||
int err;
|
||||
|
||||
if (extra_pass && ctx.idx != jit_data->ctx.idx) {
|
||||
pr_err_once("multi-func JIT bug %d != %d\n",
|
||||
ctx.idx, jit_data->ctx.idx);
|
||||
bpf_jit_binary_free(header);
|
||||
prog->bpf_func = NULL;
|
||||
prog->jited = 0;
|
||||
prog->jited_len = 0;
|
||||
goto out_offset;
|
||||
goto out_free;
|
||||
}
|
||||
err = bpf_jit_binary_lock_ro(header);
|
||||
if (err) {
|
||||
pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
|
||||
err);
|
||||
goto out_free;
|
||||
}
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
} else {
|
||||
jit_data->ctx = ctx;
|
||||
jit_data->image = image_ptr;
|
||||
|
|
@ -1334,6 +1337,13 @@ out:
|
|||
out_offset = -1;
|
||||
|
||||
return prog;
|
||||
|
||||
out_free:
|
||||
bpf_jit_binary_free(header);
|
||||
prog->bpf_func = NULL;
|
||||
prog->jited = 0;
|
||||
prog->jited_len = 0;
|
||||
goto out_offset;
|
||||
}
|
||||
|
||||
/* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */
|
||||
|
|
|
|||
|
|
@ -1012,7 +1012,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|||
bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]);
|
||||
|
||||
/* Set as read-only exec and flush instruction cache */
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
if (bpf_jit_binary_lock_ro(header))
|
||||
goto out_err;
|
||||
flush_icache_range((unsigned long)header,
|
||||
(unsigned long)&ctx.target[ctx.jit_index]);
|
||||
|
||||
|
|
|
|||
|
|
@ -167,7 +167,13 @@ skip_init_ctx:
|
|||
bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
|
||||
|
||||
if (!prog->is_func || extra_pass) {
|
||||
bpf_jit_binary_lock_ro(jit_data->header);
|
||||
if (bpf_jit_binary_lock_ro(jit_data->header)) {
|
||||
bpf_jit_binary_free(jit_data->header);
|
||||
prog->bpf_func = NULL;
|
||||
prog->jited = 0;
|
||||
prog->jited_len = 0;
|
||||
goto out_offset;
|
||||
}
|
||||
prologue_len = ctx->epilogue_offset - ctx->body_len;
|
||||
for (i = 0; i < prog->len; i++)
|
||||
ctx->offset[i] += prologue_len;
|
||||
|
|
|
|||
|
|
@ -2111,7 +2111,11 @@ skip_init_ctx:
|
|||
print_fn_code(jit.prg_buf, jit.size_prg);
|
||||
}
|
||||
if (!fp->is_func || extra_pass) {
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
if (bpf_jit_binary_lock_ro(header)) {
|
||||
bpf_jit_binary_free(header);
|
||||
fp = orig_fp;
|
||||
goto free_addrs;
|
||||
}
|
||||
} else {
|
||||
jit_data->header = header;
|
||||
jit_data->ctx = jit;
|
||||
|
|
|
|||
|
|
@ -1602,7 +1602,11 @@ skip_init_ctx:
|
|||
bpf_flush_icache(header, (u8 *)header + header->size);
|
||||
|
||||
if (!prog->is_func || extra_pass) {
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
if (bpf_jit_binary_lock_ro(header)) {
|
||||
bpf_jit_binary_free(header);
|
||||
prog = orig_prog;
|
||||
goto out_off;
|
||||
}
|
||||
} else {
|
||||
jit_data->ctx = ctx;
|
||||
jit_data->image = image_ptr;
|
||||
|
|
|
|||
|
|
@ -1351,8 +1351,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
|
|||
break;
|
||||
|
||||
case BPF_ALU64 | BPF_MOV | BPF_X:
|
||||
if (insn->off == BPF_ADDR_SPACE_CAST &&
|
||||
insn->imm == 1U << 16) {
|
||||
if (insn_is_cast_user(insn)) {
|
||||
if (dst_reg != src_reg)
|
||||
/* 32-bit mov */
|
||||
emit_mov_reg(&prog, false, dst_reg, src_reg);
|
||||
|
|
@ -3004,12 +3003,9 @@ void arch_free_bpf_trampoline(void *image, unsigned int size)
|
|||
bpf_prog_pack_free(image, size);
|
||||
}
|
||||
|
||||
void arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
}
|
||||
|
||||
void arch_unprotect_bpf_trampoline(void *image, unsigned int size)
|
||||
int arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
|
||||
|
|
|
|||
|
|
@ -2600,8 +2600,7 @@ out_image:
|
|||
if (bpf_jit_enable > 1)
|
||||
bpf_jit_dump(prog->len, proglen, pass + 1, image);
|
||||
|
||||
if (image) {
|
||||
bpf_jit_binary_lock_ro(header);
|
||||
if (image && !bpf_jit_binary_lock_ro(header)) {
|
||||
prog->bpf_func = (void *)image;
|
||||
prog->jited = 1;
|
||||
prog->jited_len = proglen;
|
||||
|
|
|
|||
|
|
@ -1116,8 +1116,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
|
|||
void *func_addr);
|
||||
void *arch_alloc_bpf_trampoline(unsigned int size);
|
||||
void arch_free_bpf_trampoline(void *image, unsigned int size);
|
||||
void arch_protect_bpf_trampoline(void *image, unsigned int size);
|
||||
void arch_unprotect_bpf_trampoline(void *image, unsigned int size);
|
||||
int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
|
||||
int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
struct bpf_tramp_links *tlinks, void *func_addr);
|
||||
|
||||
|
|
@ -1608,6 +1607,12 @@ struct bpf_tracing_link {
|
|||
struct bpf_prog *tgt_prog;
|
||||
};
|
||||
|
||||
struct bpf_raw_tp_link {
|
||||
struct bpf_link link;
|
||||
struct bpf_raw_event_map *btp;
|
||||
u64 cookie;
|
||||
};
|
||||
|
||||
struct bpf_link_primer {
|
||||
struct bpf_link *link;
|
||||
struct file *file;
|
||||
|
|
|
|||
|
|
@ -228,6 +228,16 @@ static inline bool insn_is_zext(const struct bpf_insn *insn)
|
|||
return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1;
|
||||
}
|
||||
|
||||
/* addr_space_cast from as(0) to as(1) is for converting bpf arena pointers
|
||||
* to pointers in user vma.
|
||||
*/
|
||||
static inline bool insn_is_cast_user(const struct bpf_insn *insn)
|
||||
{
|
||||
return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
|
||||
insn->off == BPF_ADDR_SPACE_CAST &&
|
||||
insn->imm == 1U << 16;
|
||||
}
|
||||
|
||||
/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
|
||||
#define BPF_LD_IMM64(DST, IMM) \
|
||||
BPF_LD_IMM64_RAW(DST, 0, IMM)
|
||||
|
|
@ -887,20 +897,22 @@ bpf_ctx_narrow_access_offset(u32 off, u32 size, u32 size_default)
|
|||
|
||||
#define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
|
||||
|
||||
static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
|
||||
static inline int __must_check bpf_prog_lock_ro(struct bpf_prog *fp)
|
||||
{
|
||||
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
|
||||
if (!fp->jited) {
|
||||
set_vm_flush_reset_perms(fp);
|
||||
set_memory_ro((unsigned long)fp, fp->pages);
|
||||
return set_memory_ro((unsigned long)fp, fp->pages);
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
|
||||
static inline int __must_check
|
||||
bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
|
||||
{
|
||||
set_vm_flush_reset_perms(hdr);
|
||||
set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
|
||||
return set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
|
||||
}
|
||||
|
||||
int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
|
||||
|
|
|
|||
|
|
@ -765,8 +765,11 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
|
|||
int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
|
||||
void perf_event_detach_bpf_prog(struct perf_event *event);
|
||||
int perf_event_query_prog_array(struct perf_event *event, void __user *info);
|
||||
int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog);
|
||||
int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog);
|
||||
|
||||
struct bpf_raw_tp_link;
|
||||
int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
|
||||
int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
|
||||
|
||||
struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name);
|
||||
void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
|
||||
int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
|
||||
|
|
@ -794,11 +797,12 @@ perf_event_query_prog_array(struct perf_event *event, void __user *info)
|
|||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *p)
|
||||
struct bpf_raw_tp_link;
|
||||
static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *p)
|
||||
static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
|
@ -909,31 +913,31 @@ void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
|
|||
int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
|
||||
void perf_event_free_bpf_prog(struct perf_event *event);
|
||||
|
||||
void bpf_trace_run1(struct bpf_prog *prog, u64 arg1);
|
||||
void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2);
|
||||
void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1);
|
||||
void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2);
|
||||
void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3);
|
||||
void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4);
|
||||
void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5);
|
||||
void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6);
|
||||
void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);
|
||||
void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
|
||||
u64 arg8);
|
||||
void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
|
||||
u64 arg8, u64 arg9);
|
||||
void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
|
||||
u64 arg8, u64 arg9, u64 arg10);
|
||||
void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
|
||||
u64 arg8, u64 arg9, u64 arg10, u64 arg11);
|
||||
void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2,
|
||||
void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
|
||||
u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
|
||||
u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);
|
||||
void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
|
||||
|
|
|
|||
|
|
@ -46,8 +46,7 @@
|
|||
static notrace void \
|
||||
__bpf_trace_##call(void *__data, proto) \
|
||||
{ \
|
||||
struct bpf_prog *prog = __data; \
|
||||
CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \
|
||||
CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(__data, CAST_TO_U64(args)); \
|
||||
}
|
||||
|
||||
#undef DECLARE_EVENT_CLASS
|
||||
|
|
|
|||
|
|
@ -1662,8 +1662,10 @@ union bpf_attr {
|
|||
} query;
|
||||
|
||||
struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
|
||||
__u64 name;
|
||||
__u32 prog_fd;
|
||||
__u64 name;
|
||||
__u32 prog_fd;
|
||||
__u32 :32;
|
||||
__aligned_u64 cookie;
|
||||
} raw_tracepoint;
|
||||
|
||||
struct { /* anonymous struct for BPF_BTF_LOAD */
|
||||
|
|
|
|||
|
|
@ -728,8 +728,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
|
|||
cur_image = image;
|
||||
trampoline_start = 0;
|
||||
}
|
||||
if (err < 0)
|
||||
goto reset_unlock;
|
||||
|
||||
*(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
|
||||
|
||||
|
|
@ -742,8 +740,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
|
|||
if (err)
|
||||
goto reset_unlock;
|
||||
}
|
||||
for (i = 0; i < st_map->image_pages_cnt; i++)
|
||||
arch_protect_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE);
|
||||
for (i = 0; i < st_map->image_pages_cnt; i++) {
|
||||
err = arch_protect_bpf_trampoline(st_map->image_pages[i],
|
||||
PAGE_SIZE);
|
||||
if (err)
|
||||
goto reset_unlock;
|
||||
}
|
||||
|
||||
if (st_map->map.map_flags & BPF_F_LINK) {
|
||||
err = 0;
|
||||
|
|
|
|||
|
|
@ -2575,8 +2575,6 @@ cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
switch (func_id) {
|
||||
case BPF_FUNC_get_current_uid_gid:
|
||||
return &bpf_get_current_uid_gid_proto;
|
||||
case BPF_FUNC_get_current_pid_tgid:
|
||||
return &bpf_get_current_pid_tgid_proto;
|
||||
case BPF_FUNC_get_current_comm:
|
||||
return &bpf_get_current_comm_proto;
|
||||
#ifdef CONFIG_CGROUP_NET_CLASSID
|
||||
|
|
|
|||
|
|
@ -908,23 +908,30 @@ static LIST_HEAD(pack_list);
|
|||
static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
|
||||
{
|
||||
struct bpf_prog_pack *pack;
|
||||
int err;
|
||||
|
||||
pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
|
||||
GFP_KERNEL);
|
||||
if (!pack)
|
||||
return NULL;
|
||||
pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
|
||||
if (!pack->ptr) {
|
||||
kfree(pack);
|
||||
return NULL;
|
||||
}
|
||||
if (!pack->ptr)
|
||||
goto out;
|
||||
bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
|
||||
bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
|
||||
list_add_tail(&pack->list, &pack_list);
|
||||
|
||||
set_vm_flush_reset_perms(pack->ptr);
|
||||
set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
|
||||
err = set_memory_rox((unsigned long)pack->ptr,
|
||||
BPF_PROG_PACK_SIZE / PAGE_SIZE);
|
||||
if (err)
|
||||
goto out;
|
||||
list_add_tail(&pack->list, &pack_list);
|
||||
return pack;
|
||||
|
||||
out:
|
||||
bpf_jit_free_exec(pack->ptr);
|
||||
kfree(pack);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
|
||||
|
|
@ -939,9 +946,16 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
|
|||
size = round_up(size, PAGE_SIZE);
|
||||
ptr = bpf_jit_alloc_exec(size);
|
||||
if (ptr) {
|
||||
int err;
|
||||
|
||||
bpf_fill_ill_insns(ptr, size);
|
||||
set_vm_flush_reset_perms(ptr);
|
||||
set_memory_rox((unsigned long)ptr, size / PAGE_SIZE);
|
||||
err = set_memory_rox((unsigned long)ptr,
|
||||
size / PAGE_SIZE);
|
||||
if (err) {
|
||||
bpf_jit_free_exec(ptr);
|
||||
ptr = NULL;
|
||||
}
|
||||
}
|
||||
goto out;
|
||||
}
|
||||
|
|
@ -2403,7 +2417,9 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
|
|||
}
|
||||
|
||||
finalize:
|
||||
bpf_prog_lock_ro(fp);
|
||||
*err = bpf_prog_lock_ro(fp);
|
||||
if (*err)
|
||||
return fp;
|
||||
|
||||
/* The tail call compatibility check can only be done at
|
||||
* this late stage as we need to determine, if we deal
|
||||
|
|
|
|||
|
|
@ -1730,6 +1730,10 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_strtol_proto;
|
||||
case BPF_FUNC_strtoul:
|
||||
return &bpf_strtoul_proto;
|
||||
case BPF_FUNC_get_current_pid_tgid:
|
||||
return &bpf_get_current_pid_tgid_proto;
|
||||
case BPF_FUNC_get_ns_current_pid_tgid:
|
||||
return &bpf_get_ns_current_pid_tgid_proto;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -155,16 +155,17 @@ static inline int extract_bit(const u8 *data, size_t index)
|
|||
}
|
||||
|
||||
/**
|
||||
* longest_prefix_match() - determine the longest prefix
|
||||
* __longest_prefix_match() - determine the longest prefix
|
||||
* @trie: The trie to get internal sizes from
|
||||
* @node: The node to operate on
|
||||
* @key: The key to compare to @node
|
||||
*
|
||||
* Determine the longest prefix of @node that matches the bits in @key.
|
||||
*/
|
||||
static size_t longest_prefix_match(const struct lpm_trie *trie,
|
||||
const struct lpm_trie_node *node,
|
||||
const struct bpf_lpm_trie_key_u8 *key)
|
||||
static __always_inline
|
||||
size_t __longest_prefix_match(const struct lpm_trie *trie,
|
||||
const struct lpm_trie_node *node,
|
||||
const struct bpf_lpm_trie_key_u8 *key)
|
||||
{
|
||||
u32 limit = min(node->prefixlen, key->prefixlen);
|
||||
u32 prefixlen = 0, i = 0;
|
||||
|
|
@ -224,6 +225,13 @@ static size_t longest_prefix_match(const struct lpm_trie *trie,
|
|||
return prefixlen;
|
||||
}
|
||||
|
||||
static size_t longest_prefix_match(const struct lpm_trie *trie,
|
||||
const struct lpm_trie_node *node,
|
||||
const struct bpf_lpm_trie_key_u8 *key)
|
||||
{
|
||||
return __longest_prefix_match(trie, node, key);
|
||||
}
|
||||
|
||||
/* Called from syscall or from eBPF program */
|
||||
static void *trie_lookup_elem(struct bpf_map *map, void *_key)
|
||||
{
|
||||
|
|
@ -245,7 +253,7 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key)
|
|||
* If it's the maximum possible prefix for this trie, we have
|
||||
* an exact match and can return it directly.
|
||||
*/
|
||||
matchlen = longest_prefix_match(trie, node, key);
|
||||
matchlen = __longest_prefix_match(trie, node, key);
|
||||
if (matchlen == trie->max_prefixlen) {
|
||||
found = node;
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -3469,17 +3469,12 @@ out_put_prog:
|
|||
return err;
|
||||
}
|
||||
|
||||
struct bpf_raw_tp_link {
|
||||
struct bpf_link link;
|
||||
struct bpf_raw_event_map *btp;
|
||||
};
|
||||
|
||||
static void bpf_raw_tp_link_release(struct bpf_link *link)
|
||||
{
|
||||
struct bpf_raw_tp_link *raw_tp =
|
||||
container_of(link, struct bpf_raw_tp_link, link);
|
||||
|
||||
bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
|
||||
bpf_probe_unregister(raw_tp->btp, raw_tp);
|
||||
bpf_put_raw_tracepoint(raw_tp->btp);
|
||||
}
|
||||
|
||||
|
|
@ -3779,7 +3774,7 @@ static int bpf_perf_link_attach(const union bpf_attr *attr, struct bpf_prog *pro
|
|||
#endif /* CONFIG_PERF_EVENTS */
|
||||
|
||||
static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
|
||||
const char __user *user_tp_name)
|
||||
const char __user *user_tp_name, u64 cookie)
|
||||
{
|
||||
struct bpf_link_primer link_primer;
|
||||
struct bpf_raw_tp_link *link;
|
||||
|
|
@ -3826,6 +3821,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
|
|||
bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
|
||||
&bpf_raw_tp_link_lops, prog);
|
||||
link->btp = btp;
|
||||
link->cookie = cookie;
|
||||
|
||||
err = bpf_link_prime(&link->link, &link_primer);
|
||||
if (err) {
|
||||
|
|
@ -3833,7 +3829,7 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
|
|||
goto out_put_btp;
|
||||
}
|
||||
|
||||
err = bpf_probe_register(link->btp, prog);
|
||||
err = bpf_probe_register(link->btp, link);
|
||||
if (err) {
|
||||
bpf_link_cleanup(&link_primer);
|
||||
goto out_put_btp;
|
||||
|
|
@ -3846,11 +3842,13 @@ out_put_btp:
|
|||
return err;
|
||||
}
|
||||
|
||||
#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
|
||||
#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie
|
||||
|
||||
static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
|
||||
{
|
||||
struct bpf_prog *prog;
|
||||
void __user *tp_name;
|
||||
__u64 cookie;
|
||||
int fd;
|
||||
|
||||
if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
|
||||
|
|
@ -3860,7 +3858,9 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
|
|||
if (IS_ERR(prog))
|
||||
return PTR_ERR(prog);
|
||||
|
||||
fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name));
|
||||
tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
|
||||
cookie = attr->raw_tracepoint.cookie;
|
||||
fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);
|
||||
if (fd < 0)
|
||||
bpf_prog_put(prog);
|
||||
return fd;
|
||||
|
|
@ -5198,7 +5198,7 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr)
|
|||
goto out;
|
||||
}
|
||||
if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
|
||||
ret = bpf_raw_tp_link_attach(prog, NULL);
|
||||
ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);
|
||||
else if (prog->expected_attach_type == BPF_TRACE_ITER)
|
||||
ret = bpf_iter_link_attach(attr, uattr, prog);
|
||||
else if (prog->expected_attach_type == BPF_LSM_CGROUP)
|
||||
|
|
|
|||
|
|
@ -456,7 +456,9 @@ again:
|
|||
if (err < 0)
|
||||
goto out_free;
|
||||
|
||||
arch_protect_bpf_trampoline(im->image, im->size);
|
||||
err = arch_protect_bpf_trampoline(im->image, im->size);
|
||||
if (err)
|
||||
goto out_free;
|
||||
|
||||
WARN_ON(tr->cur_image && total == 0);
|
||||
if (tr->cur_image)
|
||||
|
|
@ -1072,17 +1074,10 @@ void __weak arch_free_bpf_trampoline(void *image, unsigned int size)
|
|||
bpf_jit_free_exec(image);
|
||||
}
|
||||
|
||||
void __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
int __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
WARN_ON_ONCE(size > PAGE_SIZE);
|
||||
set_memory_rox((long)image, 1);
|
||||
}
|
||||
|
||||
void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size)
|
||||
{
|
||||
WARN_ON_ONCE(size > PAGE_SIZE);
|
||||
set_memory_nx((long)image, 1);
|
||||
set_memory_rw((long)image, 1);
|
||||
return set_memory_rox((long)image, 1);
|
||||
}
|
||||
|
||||
int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
|
||||
|
|
|
|||
|
|
@ -14544,7 +14544,19 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state
|
|||
struct tnum t;
|
||||
u64 val;
|
||||
|
||||
again:
|
||||
/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
|
||||
switch (opcode) {
|
||||
case BPF_JGE:
|
||||
case BPF_JGT:
|
||||
case BPF_JSGE:
|
||||
case BPF_JSGT:
|
||||
opcode = flip_opcode(opcode);
|
||||
swap(reg1, reg2);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
switch (opcode) {
|
||||
case BPF_JEQ:
|
||||
if (is_jmp32) {
|
||||
|
|
@ -14687,14 +14699,6 @@ again:
|
|||
reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
|
||||
}
|
||||
break;
|
||||
case BPF_JGE:
|
||||
case BPF_JGT:
|
||||
case BPF_JSGE:
|
||||
case BPF_JSGT:
|
||||
/* just reuse LE/LT logic above */
|
||||
opcode = flip_opcode(opcode);
|
||||
swap(reg1, reg2);
|
||||
goto again;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
|
@ -19158,6 +19162,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
|
|||
if (bpf_prog_calc_tag(func[i]))
|
||||
goto out_free;
|
||||
func[i]->is_func = 1;
|
||||
func[i]->sleepable = prog->sleepable;
|
||||
func[i]->aux->func_idx = i;
|
||||
/* Below members will be freed only at prog->aux */
|
||||
func[i]->aux->btf = prog->aux->btf;
|
||||
|
|
@ -19262,10 +19267,14 @@ static int jit_subprogs(struct bpf_verifier_env *env)
|
|||
* bpf_prog_load will add the kallsyms for the main program.
|
||||
*/
|
||||
for (i = 1; i < env->subprog_cnt; i++) {
|
||||
bpf_prog_lock_ro(func[i]);
|
||||
bpf_prog_kallsyms_add(func[i]);
|
||||
err = bpf_prog_lock_ro(func[i]);
|
||||
if (err)
|
||||
goto out_free;
|
||||
}
|
||||
|
||||
for (i = 1; i < env->subprog_cnt; i++)
|
||||
bpf_prog_kallsyms_add(func[i]);
|
||||
|
||||
/* Last step: make now unused interpreter insns from main
|
||||
* prog consistent for later dump requests, so they can
|
||||
* later look the same as if they were interpreted only.
|
||||
|
|
|
|||
|
|
@ -1053,9 +1053,15 @@ static unsigned long get_entry_ip(unsigned long fentry_ip)
|
|||
{
|
||||
u32 instr;
|
||||
|
||||
/* Being extra safe in here in case entry ip is on the page-edge. */
|
||||
if (get_kernel_nofault(instr, (u32 *) fentry_ip - 1))
|
||||
return fentry_ip;
|
||||
/* We want to be extra safe in case entry ip is on the page edge,
|
||||
* but otherwise we need to avoid get_kernel_nofault()'s overhead.
|
||||
*/
|
||||
if ((fentry_ip & ~PAGE_MASK) < ENDBR_INSN_SIZE) {
|
||||
if (get_kernel_nofault(instr, (u32 *)(fentry_ip - ENDBR_INSN_SIZE)))
|
||||
return fentry_ip;
|
||||
} else {
|
||||
instr = *(u32 *)(fentry_ip - ENDBR_INSN_SIZE);
|
||||
}
|
||||
if (is_endbr(instr))
|
||||
fentry_ip -= ENDBR_INSN_SIZE;
|
||||
return fentry_ip;
|
||||
|
|
@ -1525,8 +1531,6 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_ktime_get_boot_ns_proto;
|
||||
case BPF_FUNC_tail_call:
|
||||
return &bpf_tail_call_proto;
|
||||
case BPF_FUNC_get_current_pid_tgid:
|
||||
return &bpf_get_current_pid_tgid_proto;
|
||||
case BPF_FUNC_get_current_task:
|
||||
return &bpf_get_current_task_proto;
|
||||
case BPF_FUNC_get_current_task_btf:
|
||||
|
|
@ -1582,8 +1586,6 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_send_signal_thread_proto;
|
||||
case BPF_FUNC_perf_event_read_value:
|
||||
return &bpf_perf_event_read_value_proto;
|
||||
case BPF_FUNC_get_ns_current_pid_tgid:
|
||||
return &bpf_get_ns_current_pid_tgid_proto;
|
||||
case BPF_FUNC_ringbuf_output:
|
||||
return &bpf_ringbuf_output_proto;
|
||||
case BPF_FUNC_ringbuf_reserve:
|
||||
|
|
@ -2008,6 +2010,8 @@ raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_get_stackid_proto_raw_tp;
|
||||
case BPF_FUNC_get_stack:
|
||||
return &bpf_get_stack_proto_raw_tp;
|
||||
case BPF_FUNC_get_attach_cookie:
|
||||
return &bpf_get_attach_cookie_proto_tracing;
|
||||
default:
|
||||
return bpf_tracing_func_proto(func_id, prog);
|
||||
}
|
||||
|
|
@ -2070,6 +2074,9 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
case BPF_FUNC_get_func_arg_cnt:
|
||||
return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
|
||||
case BPF_FUNC_get_attach_cookie:
|
||||
if (prog->type == BPF_PROG_TYPE_TRACING &&
|
||||
prog->expected_attach_type == BPF_TRACE_RAW_TP)
|
||||
return &bpf_get_attach_cookie_proto_tracing;
|
||||
return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
|
||||
default:
|
||||
fn = raw_tp_prog_func_proto(func_id, prog);
|
||||
|
|
@ -2370,16 +2377,26 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
|
|||
}
|
||||
|
||||
static __always_inline
|
||||
void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
|
||||
void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
|
||||
{
|
||||
struct bpf_prog *prog = link->link.prog;
|
||||
struct bpf_run_ctx *old_run_ctx;
|
||||
struct bpf_trace_run_ctx run_ctx;
|
||||
|
||||
cant_sleep();
|
||||
if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
|
||||
bpf_prog_inc_misses_counter(prog);
|
||||
goto out;
|
||||
}
|
||||
|
||||
run_ctx.bpf_cookie = link->cookie;
|
||||
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
|
||||
|
||||
rcu_read_lock();
|
||||
(void) bpf_prog_run(prog, args);
|
||||
rcu_read_unlock();
|
||||
|
||||
bpf_reset_run_ctx(old_run_ctx);
|
||||
out:
|
||||
this_cpu_dec(*(prog->active));
|
||||
}
|
||||
|
|
@ -2408,12 +2425,12 @@ out:
|
|||
#define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
|
||||
|
||||
#define BPF_TRACE_DEFN_x(x) \
|
||||
void bpf_trace_run##x(struct bpf_prog *prog, \
|
||||
void bpf_trace_run##x(struct bpf_raw_tp_link *link, \
|
||||
REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \
|
||||
{ \
|
||||
u64 args[x]; \
|
||||
REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \
|
||||
__bpf_trace_run(prog, args); \
|
||||
__bpf_trace_run(link, args); \
|
||||
} \
|
||||
EXPORT_SYMBOL_GPL(bpf_trace_run##x)
|
||||
BPF_TRACE_DEFN_x(1);
|
||||
|
|
@ -2429,9 +2446,10 @@ BPF_TRACE_DEFN_x(10);
|
|||
BPF_TRACE_DEFN_x(11);
|
||||
BPF_TRACE_DEFN_x(12);
|
||||
|
||||
static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
|
||||
int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
|
||||
{
|
||||
struct tracepoint *tp = btp->tp;
|
||||
struct bpf_prog *prog = link->link.prog;
|
||||
|
||||
/*
|
||||
* check that program doesn't access arguments beyond what's
|
||||
|
|
@ -2443,18 +2461,12 @@ static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *
|
|||
if (prog->aux->max_tp_access > btp->writable_size)
|
||||
return -EINVAL;
|
||||
|
||||
return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func,
|
||||
prog);
|
||||
return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link);
|
||||
}
|
||||
|
||||
int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
|
||||
int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
|
||||
{
|
||||
return __bpf_probe_register(btp, prog);
|
||||
}
|
||||
|
||||
int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
|
||||
{
|
||||
return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog);
|
||||
return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link);
|
||||
}
|
||||
|
||||
int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
|
||||
|
|
|
|||
|
|
@ -133,7 +133,9 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
|
|||
if (err < 0)
|
||||
goto out;
|
||||
|
||||
arch_protect_bpf_trampoline(image, PAGE_SIZE);
|
||||
err = arch_protect_bpf_trampoline(image, PAGE_SIZE);
|
||||
if (err)
|
||||
goto out;
|
||||
prog_ret = dummy_ops_call_op(image, args);
|
||||
|
||||
err = dummy_ops_copy_args(args);
|
||||
|
|
|
|||
|
|
@ -8342,8 +8342,6 @@ sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
|
|||
return &bpf_event_output_data_proto;
|
||||
case BPF_FUNC_get_current_uid_gid:
|
||||
return &bpf_get_current_uid_gid_proto;
|
||||
case BPF_FUNC_get_current_pid_tgid:
|
||||
return &bpf_get_current_pid_tgid_proto;
|
||||
case BPF_FUNC_sk_storage_get:
|
||||
return &bpf_sk_storage_get_proto;
|
||||
case BPF_FUNC_sk_storage_delete:
|
||||
|
|
|
|||
|
|
@ -89,6 +89,10 @@ ifneq ($(EXTRA_LDFLAGS),)
|
|||
LDFLAGS += $(EXTRA_LDFLAGS)
|
||||
endif
|
||||
|
||||
HOST_CFLAGS := $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\
|
||||
$(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS)))
|
||||
HOST_LDFLAGS := $(LDFLAGS)
|
||||
|
||||
INSTALL ?= install
|
||||
RM ?= rm -f
|
||||
|
||||
|
|
@ -178,12 +182,9 @@ ifeq ($(filter -DHAVE_LLVM_SUPPORT -DHAVE_LIBBFD_SUPPORT,$(CFLAGS)),)
|
|||
SRCS := $(filter-out jit_disasm.c,$(SRCS))
|
||||
endif
|
||||
|
||||
HOST_CFLAGS = $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\
|
||||
$(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS)))
|
||||
|
||||
BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
|
||||
|
||||
BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o xlated_dumper.o btf_dumper.o disasm.o)
|
||||
BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o)
|
||||
$(BOOTSTRAP_OBJS): $(LIBBPF_BOOTSTRAP)
|
||||
|
||||
OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
|
||||
|
|
@ -231,14 +232,11 @@ endif
|
|||
|
||||
CFLAGS += $(if $(BUILD_BPF_SKELS),,-DBPFTOOL_WITHOUT_SKELETONS)
|
||||
|
||||
$(BOOTSTRAP_OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
|
||||
$(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
|
||||
|
||||
$(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
|
||||
$(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@
|
||||
|
||||
$(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
|
||||
$(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
|
||||
$(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(HOST_LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
|
||||
|
||||
$(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
|
||||
$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@
|
||||
|
|
|
|||
|
|
@ -1131,7 +1131,8 @@ static void gen_st_ops_shadow_init(struct btf *btf, struct bpf_object *obj)
|
|||
continue;
|
||||
codegen("\
|
||||
\n\
|
||||
obj->struct_ops.%1$s = bpf_map__initial_value(obj->maps.%1$s, NULL);\n\
|
||||
obj->struct_ops.%1$s = (typeof(obj->struct_ops.%1$s))\n\
|
||||
bpf_map__initial_value(obj->maps.%1$s, NULL);\n\
|
||||
\n\
|
||||
", ident);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -101,7 +101,6 @@ int build_obj_refs_table(struct hashmap **map, enum bpf_obj_type type)
|
|||
char buf[4096 / sizeof(*e) * sizeof(*e)];
|
||||
struct pid_iter_bpf *skel;
|
||||
int err, ret, fd = -1, i;
|
||||
libbpf_print_fn_t default_print;
|
||||
|
||||
*map = hashmap__new(hash_fn_for_key_as_id, equal_fn_for_key_as_id, NULL);
|
||||
if (IS_ERR(*map)) {
|
||||
|
|
@ -118,12 +117,18 @@ int build_obj_refs_table(struct hashmap **map, enum bpf_obj_type type)
|
|||
|
||||
skel->rodata->obj_type = type;
|
||||
|
||||
/* we don't want output polluted with libbpf errors if bpf_iter is not
|
||||
* supported
|
||||
*/
|
||||
default_print = libbpf_set_print(libbpf_print_none);
|
||||
err = pid_iter_bpf__load(skel);
|
||||
libbpf_set_print(default_print);
|
||||
if (!verifier_logs) {
|
||||
libbpf_print_fn_t default_print;
|
||||
|
||||
/* Unless debug information is on, we don't want the output to
|
||||
* be polluted with libbpf errors if bpf_iter is not supported.
|
||||
*/
|
||||
default_print = libbpf_set_print(libbpf_print_none);
|
||||
err = pid_iter_bpf__load(skel);
|
||||
libbpf_set_print(default_print);
|
||||
} else {
|
||||
err = pid_iter_bpf__load(skel);
|
||||
}
|
||||
if (err) {
|
||||
/* too bad, kernel doesn't support BPF iterators yet */
|
||||
err = 0;
|
||||
|
|
|
|||
|
|
@ -102,8 +102,8 @@ int iter(struct bpf_iter__task_file *ctx)
|
|||
BPF_LINK_TYPE_PERF_EVENT___local)) {
|
||||
struct bpf_link *link = (struct bpf_link *) file->private_data;
|
||||
|
||||
if (link->type == bpf_core_enum_value(enum bpf_link_type___local,
|
||||
BPF_LINK_TYPE_PERF_EVENT___local)) {
|
||||
if (BPF_CORE_READ(link, type) == bpf_core_enum_value(enum bpf_link_type___local,
|
||||
BPF_LINK_TYPE_PERF_EVENT___local)) {
|
||||
e.has_bpf_cookie = true;
|
||||
e.bpf_cookie = get_bpf_cookie(link);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,10 @@
|
|||
#define noinline
|
||||
#endif
|
||||
|
||||
#ifndef __nocf_check
|
||||
#define __nocf_check __attribute__((nocf_check))
|
||||
#endif
|
||||
|
||||
/* Are two types/vars the same type (ignoring qualifiers)? */
|
||||
#ifndef __same_type
|
||||
# define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
|
||||
|
|
|
|||
|
|
@ -1662,8 +1662,10 @@ union bpf_attr {
|
|||
} query;
|
||||
|
||||
struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
|
||||
__u64 name;
|
||||
__u32 prog_fd;
|
||||
__u64 name;
|
||||
__u32 prog_fd;
|
||||
__u32 :32;
|
||||
__aligned_u64 cookie;
|
||||
} raw_tracepoint;
|
||||
|
||||
struct { /* anonymous struct for BPF_BTF_LOAD */
|
||||
|
|
|
|||
|
|
@ -785,6 +785,7 @@ int bpf_link_create(int prog_fd, int target_fd,
|
|||
if (!OPTS_ZEROED(opts, uprobe_multi))
|
||||
return libbpf_err(-EINVAL);
|
||||
break;
|
||||
case BPF_TRACE_RAW_TP:
|
||||
case BPF_TRACE_FENTRY:
|
||||
case BPF_TRACE_FEXIT:
|
||||
case BPF_MODIFY_RETURN:
|
||||
|
|
@ -1173,20 +1174,31 @@ int bpf_link_get_info_by_fd(int link_fd, struct bpf_link_info *info, __u32 *info
|
|||
return bpf_obj_get_info_by_fd(link_fd, info, info_len);
|
||||
}
|
||||
|
||||
int bpf_raw_tracepoint_open(const char *name, int prog_fd)
|
||||
int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts)
|
||||
{
|
||||
const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);
|
||||
union bpf_attr attr;
|
||||
int fd;
|
||||
|
||||
if (!OPTS_VALID(opts, bpf_raw_tp_opts))
|
||||
return libbpf_err(-EINVAL);
|
||||
|
||||
memset(&attr, 0, attr_sz);
|
||||
attr.raw_tracepoint.name = ptr_to_u64(name);
|
||||
attr.raw_tracepoint.prog_fd = prog_fd;
|
||||
attr.raw_tracepoint.name = ptr_to_u64(OPTS_GET(opts, tp_name, NULL));
|
||||
attr.raw_tracepoint.cookie = OPTS_GET(opts, cookie, 0);
|
||||
|
||||
fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
|
||||
return libbpf_err_errno(fd);
|
||||
}
|
||||
|
||||
int bpf_raw_tracepoint_open(const char *name, int prog_fd)
|
||||
{
|
||||
LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name);
|
||||
|
||||
return bpf_raw_tracepoint_open_opts(prog_fd, &opts);
|
||||
}
|
||||
|
||||
int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts)
|
||||
{
|
||||
const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd);
|
||||
|
|
|
|||
|
|
@ -617,6 +617,15 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type,
|
|||
__u32 query_flags, __u32 *attach_flags,
|
||||
__u32 *prog_ids, __u32 *prog_cnt);
|
||||
|
||||
struct bpf_raw_tp_opts {
|
||||
size_t sz; /* size of this struct for forward/backward compatibility */
|
||||
const char *tp_name;
|
||||
__u64 cookie;
|
||||
size_t :0;
|
||||
};
|
||||
#define bpf_raw_tp_opts__last_field cookie
|
||||
|
||||
LIBBPF_API int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts);
|
||||
LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);
|
||||
LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,
|
||||
__u32 *buf_len, __u32 *prog_id, __u32 *fd_type,
|
||||
|
|
|
|||
|
|
@ -1132,8 +1132,26 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
|
|||
const char *mname;
|
||||
|
||||
mname = btf__name_by_offset(btf, member->name_off);
|
||||
moff = member->offset / 8;
|
||||
mdata = data + moff;
|
||||
msize = btf__resolve_size(btf, member->type);
|
||||
if (msize < 0) {
|
||||
pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
|
||||
map->name, mname);
|
||||
return msize;
|
||||
}
|
||||
|
||||
kern_member = find_member_by_name(kern_btf, kern_type, mname);
|
||||
if (!kern_member) {
|
||||
/* Skip all zeros or null fields if they are not
|
||||
* presented in the kernel BTF.
|
||||
*/
|
||||
if (libbpf_is_mem_zeroed(mdata, msize)) {
|
||||
pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
|
||||
map->name, mname);
|
||||
continue;
|
||||
}
|
||||
|
||||
pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
|
||||
map->name, mname);
|
||||
return -ENOTSUP;
|
||||
|
|
@ -1147,10 +1165,7 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
|
|||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
moff = member->offset / 8;
|
||||
kern_moff = kern_member->offset / 8;
|
||||
|
||||
mdata = data + moff;
|
||||
kern_mdata = kern_data + kern_moff;
|
||||
|
||||
mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
|
||||
|
|
@ -1230,9 +1245,8 @@ static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
|
|||
continue;
|
||||
}
|
||||
|
||||
msize = btf__resolve_size(btf, mtype_id);
|
||||
kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
|
||||
if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
|
||||
if (kern_msize < 0 || msize != kern_msize) {
|
||||
pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
|
||||
map->name, mname, (ssize_t)msize,
|
||||
(ssize_t)kern_msize);
|
||||
|
|
@ -7317,9 +7331,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
|
|||
char *cp, errmsg[STRERR_BUFSIZE];
|
||||
size_t log_buf_size = 0;
|
||||
char *log_buf = NULL, *tmp;
|
||||
int btf_fd, ret, err;
|
||||
bool own_log_buf = true;
|
||||
__u32 log_level = prog->log_level;
|
||||
int ret, err;
|
||||
|
||||
if (prog->type == BPF_PROG_TYPE_UNSPEC) {
|
||||
/*
|
||||
|
|
@ -7343,9 +7357,8 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
|
|||
load_attr.prog_ifindex = prog->prog_ifindex;
|
||||
|
||||
/* specify func_info/line_info only if kernel supports them */
|
||||
btf_fd = btf__fd(obj->btf);
|
||||
if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
|
||||
load_attr.prog_btf_fd = btf_fd;
|
||||
if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
|
||||
load_attr.prog_btf_fd = btf__fd(obj->btf);
|
||||
load_attr.func_info = prog->func_info;
|
||||
load_attr.func_info_rec_size = prog->func_info_rec_size;
|
||||
load_attr.func_info_cnt = prog->func_info_cnt;
|
||||
|
|
@ -8559,6 +8572,11 @@ int bpf_map__pin(struct bpf_map *map, const char *path)
|
|||
return libbpf_err(-EINVAL);
|
||||
}
|
||||
|
||||
if (map->fd < 0) {
|
||||
pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
|
||||
return libbpf_err(-EINVAL);
|
||||
}
|
||||
|
||||
if (map->pin_path) {
|
||||
if (path && strcmp(path, map->pin_path)) {
|
||||
pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
|
||||
|
|
@ -9294,6 +9312,7 @@ static const struct bpf_sec_def section_defs[] = {
|
|||
SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
|
||||
SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
|
||||
SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
|
||||
SEC_DEF("sk_skb/verdict", SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),
|
||||
SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
|
||||
SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
|
||||
SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
|
||||
|
|
@ -10303,6 +10322,11 @@ static int validate_map_op(const struct bpf_map *map, size_t key_sz,
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (map->fd < 0) {
|
||||
pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (!check_value_sz)
|
||||
return 0;
|
||||
|
||||
|
|
@ -10415,8 +10439,15 @@ long libbpf_get_error(const void *ptr)
|
|||
int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
|
||||
{
|
||||
int ret;
|
||||
int prog_fd = bpf_program__fd(prog);
|
||||
|
||||
ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
|
||||
if (prog_fd < 0) {
|
||||
pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err(-EINVAL);
|
||||
}
|
||||
|
||||
ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);
|
||||
return libbpf_err_errno(ret);
|
||||
}
|
||||
|
||||
|
|
@ -10610,7 +10641,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p
|
|||
}
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
if (prog_fd < 0) {
|
||||
pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
|
||||
pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
|
@ -11334,6 +11365,13 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
|
|||
if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
if (prog_fd < 0) {
|
||||
pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
||||
syms = OPTS_GET(opts, syms, false);
|
||||
addrs = OPTS_GET(opts, addrs, false);
|
||||
cnt = OPTS_GET(opts, cnt, false);
|
||||
|
|
@ -11374,7 +11412,6 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
|
|||
}
|
||||
link->detach = &bpf_link__detach_fd;
|
||||
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
|
||||
if (link_fd < 0) {
|
||||
err = -errno;
|
||||
|
|
@ -11757,6 +11794,13 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
|
|||
if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
if (prog_fd < 0) {
|
||||
pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
||||
syms = OPTS_GET(opts, syms, NULL);
|
||||
offsets = OPTS_GET(opts, offsets, NULL);
|
||||
ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
|
||||
|
|
@ -11832,7 +11876,6 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
|
|||
}
|
||||
link->detach = &bpf_link__detach_fd;
|
||||
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
|
||||
if (link_fd < 0) {
|
||||
err = -errno;
|
||||
|
|
@ -12076,7 +12119,7 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
|
|||
return libbpf_err_ptr(-EINVAL);
|
||||
|
||||
if (bpf_program__fd(prog) < 0) {
|
||||
pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
|
||||
pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
|
@ -12267,13 +12310,19 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_lin
|
|||
return libbpf_get_error(*link);
|
||||
}
|
||||
|
||||
struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
|
||||
const char *tp_name)
|
||||
struct bpf_link *
|
||||
bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
|
||||
const char *tp_name,
|
||||
struct bpf_raw_tracepoint_opts *opts)
|
||||
{
|
||||
LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
|
||||
char errmsg[STRERR_BUFSIZE];
|
||||
struct bpf_link *link;
|
||||
int prog_fd, pfd;
|
||||
|
||||
if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
|
||||
prog_fd = bpf_program__fd(prog);
|
||||
if (prog_fd < 0) {
|
||||
pr_warn("prog '%s': can't attach before loaded\n", prog->name);
|
||||
|
|
@ -12285,7 +12334,9 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr
|
|||
return libbpf_err_ptr(-ENOMEM);
|
||||
link->detach = &bpf_link__detach_fd;
|
||||
|
||||
pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
|
||||
raw_opts.tp_name = tp_name;
|
||||
raw_opts.cookie = OPTS_GET(opts, cookie, 0);
|
||||
pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);
|
||||
if (pfd < 0) {
|
||||
pfd = -errno;
|
||||
free(link);
|
||||
|
|
@ -12297,6 +12348,12 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr
|
|||
return link;
|
||||
}
|
||||
|
||||
struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
|
||||
const char *tp_name)
|
||||
{
|
||||
return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
|
||||
}
|
||||
|
||||
static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
|
||||
{
|
||||
static const char *const prefixes[] = {
|
||||
|
|
@ -12658,6 +12715,12 @@ struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
|
|||
if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
|
||||
return libbpf_err_ptr(-EOPNOTSUPP);
|
||||
|
||||
if (bpf_program__fd(prog) < 0) {
|
||||
pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
|
||||
prog->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
||||
err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
|
||||
if (err)
|
||||
return libbpf_err_ptr(err);
|
||||
|
|
@ -12698,9 +12761,14 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
|
|||
__u32 zero = 0;
|
||||
int err, fd;
|
||||
|
||||
if (!bpf_map__is_struct_ops(map) || map->fd == -1)
|
||||
if (!bpf_map__is_struct_ops(map))
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
|
||||
if (map->fd < 0) {
|
||||
pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
}
|
||||
|
||||
link = calloc(1, sizeof(*link));
|
||||
if (!link)
|
||||
return libbpf_err_ptr(-EINVAL);
|
||||
|
|
@ -12747,9 +12815,14 @@ int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map)
|
|||
__u32 zero = 0;
|
||||
int err;
|
||||
|
||||
if (!bpf_map__is_struct_ops(map) || !map_is_created(map))
|
||||
if (!bpf_map__is_struct_ops(map))
|
||||
return -EINVAL;
|
||||
|
||||
if (map->fd < 0) {
|
||||
pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
|
||||
/* Ensure the type of a link is correct */
|
||||
if (st_ops_link->map_fd < 0)
|
||||
|
|
|
|||
|
|
@ -760,9 +760,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
|
|||
const char *tp_name,
|
||||
const struct bpf_tracepoint_opts *opts);
|
||||
|
||||
struct bpf_raw_tracepoint_opts {
|
||||
size_t sz; /* size of this struct for forward/backward compatibility */
|
||||
__u64 cookie;
|
||||
size_t :0;
|
||||
};
|
||||
#define bpf_raw_tracepoint_opts__last_field cookie
|
||||
|
||||
LIBBPF_API struct bpf_link *
|
||||
bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
|
||||
const char *tp_name);
|
||||
LIBBPF_API struct bpf_link *
|
||||
bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
|
||||
const char *tp_name,
|
||||
struct bpf_raw_tracepoint_opts *opts);
|
||||
|
||||
struct bpf_trace_opts {
|
||||
/* size of this struct, for forward/backward compatibility */
|
||||
|
|
|
|||
|
|
@ -410,6 +410,8 @@ LIBBPF_1.3.0 {
|
|||
|
||||
LIBBPF_1.4.0 {
|
||||
global:
|
||||
bpf_program__attach_raw_tracepoint_opts;
|
||||
bpf_raw_tracepoint_open_opts;
|
||||
bpf_token_create;
|
||||
btf__new_split;
|
||||
btf_ext__raw_data;
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ static void *task_producer(void *input)
|
|||
|
||||
for (i = 0; i < batch_sz; i++) {
|
||||
if (!pthd_results[i])
|
||||
pthread_join(pthds[i], NULL);;
|
||||
pthread_join(pthds[i], NULL);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,47 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2020 Facebook */
|
||||
#define _GNU_SOURCE
|
||||
#include <unistd.h>
|
||||
#include "bench.h"
|
||||
#include "trigger_bench.skel.h"
|
||||
#include "trace_helpers.h"
|
||||
|
||||
/* adjust slot shift in inc_hits() if changing */
|
||||
#define MAX_BUCKETS 256
|
||||
|
||||
#pragma GCC diagnostic ignored "-Wattributes"
|
||||
|
||||
/* BPF triggering benchmarks */
|
||||
static struct trigger_ctx {
|
||||
struct trigger_bench *skel;
|
||||
} ctx;
|
||||
|
||||
static struct counter base_hits;
|
||||
static struct counter base_hits[MAX_BUCKETS];
|
||||
|
||||
static __always_inline void inc_counter(struct counter *counters)
|
||||
{
|
||||
static __thread int tid = 0;
|
||||
unsigned slot;
|
||||
|
||||
if (unlikely(tid == 0))
|
||||
tid = syscall(SYS_gettid);
|
||||
|
||||
/* multiplicative hashing, it's fast */
|
||||
slot = 2654435769U * tid;
|
||||
slot >>= 24;
|
||||
|
||||
atomic_inc(&base_hits[slot].value); /* use highest byte as an index */
|
||||
}
|
||||
|
||||
static long sum_and_reset_counters(struct counter *counters)
|
||||
{
|
||||
int i;
|
||||
long sum = 0;
|
||||
|
||||
for (i = 0; i < MAX_BUCKETS; i++)
|
||||
sum += atomic_swap(&counters[i].value, 0);
|
||||
return sum;
|
||||
}
|
||||
|
||||
static void trigger_validate(void)
|
||||
{
|
||||
|
|
@ -23,14 +55,14 @@ static void *trigger_base_producer(void *input)
|
|||
{
|
||||
while (true) {
|
||||
(void)syscall(__NR_getpgid);
|
||||
atomic_inc(&base_hits.value);
|
||||
inc_counter(base_hits);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void trigger_base_measure(struct bench_res *res)
|
||||
{
|
||||
res->hits = atomic_swap(&base_hits.value, 0);
|
||||
res->hits = sum_and_reset_counters(base_hits);
|
||||
}
|
||||
|
||||
static void *trigger_producer(void *input)
|
||||
|
|
@ -42,7 +74,7 @@ static void *trigger_producer(void *input)
|
|||
|
||||
static void trigger_measure(struct bench_res *res)
|
||||
{
|
||||
res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
|
||||
res->hits = sum_and_reset_counters(ctx.skel->bss->hits);
|
||||
}
|
||||
|
||||
static void setup_ctx(void)
|
||||
|
|
@ -137,7 +169,7 @@ static void trigger_fmodret_setup(void)
|
|||
* GCC doesn't generate stack setup preample for these functions due to them
|
||||
* having no input arguments and doing nothing in the body.
|
||||
*/
|
||||
__weak void uprobe_target_nop(void)
|
||||
__nocf_check __weak void uprobe_target_nop(void)
|
||||
{
|
||||
asm volatile ("nop");
|
||||
}
|
||||
|
|
@ -146,7 +178,7 @@ __weak void opaque_noop_func(void)
|
|||
{
|
||||
}
|
||||
|
||||
__weak int uprobe_target_push(void)
|
||||
__nocf_check __weak int uprobe_target_push(void)
|
||||
{
|
||||
/* overhead of function call is negligible compared to uprobe
|
||||
* triggering, so this shouldn't affect benchmark results much
|
||||
|
|
@ -155,7 +187,7 @@ __weak int uprobe_target_push(void)
|
|||
return 1;
|
||||
}
|
||||
|
||||
__weak void uprobe_target_ret(void)
|
||||
__nocf_check __weak void uprobe_target_ret(void)
|
||||
{
|
||||
asm volatile ("");
|
||||
}
|
||||
|
|
@ -164,7 +196,7 @@ static void *uprobe_base_producer(void *input)
|
|||
{
|
||||
while (true) {
|
||||
uprobe_target_nop();
|
||||
atomic_inc(&base_hits.value);
|
||||
inc_counter(base_hits);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -573,6 +573,115 @@ cleanup:
|
|||
close(lsm_fd);
|
||||
}
|
||||
|
||||
static void tp_btf_subtest(struct test_bpf_cookie *skel)
|
||||
{
|
||||
__u64 cookie;
|
||||
int prog_fd, link_fd = -1;
|
||||
struct bpf_link *link = NULL;
|
||||
LIBBPF_OPTS(bpf_link_create_opts, link_opts);
|
||||
LIBBPF_OPTS(bpf_raw_tp_opts, raw_tp_opts);
|
||||
LIBBPF_OPTS(bpf_trace_opts, trace_opts);
|
||||
|
||||
/* There are three different ways to attach tp_btf (BTF-aware raw
|
||||
* tracepoint) programs. Let's test all of them.
|
||||
*/
|
||||
prog_fd = bpf_program__fd(skel->progs.handle_tp_btf);
|
||||
|
||||
/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */
|
||||
skel->bss->tp_btf_res = 0;
|
||||
|
||||
raw_tp_opts.cookie = cookie = 0x11000000000000L;
|
||||
link_fd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_tp_opts);
|
||||
if (!ASSERT_GE(link_fd, 0, "bpf_raw_tracepoint_open_opts"))
|
||||
goto cleanup;
|
||||
|
||||
usleep(1); /* trigger */
|
||||
close(link_fd); /* detach */
|
||||
link_fd = -1;
|
||||
|
||||
ASSERT_EQ(skel->bss->tp_btf_res, cookie, "raw_tp_open_res");
|
||||
|
||||
/* low-level generic bpf_link_create() API */
|
||||
skel->bss->tp_btf_res = 0;
|
||||
|
||||
link_opts.tracing.cookie = cookie = 0x22000000000000L;
|
||||
link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_RAW_TP, &link_opts);
|
||||
if (!ASSERT_GE(link_fd, 0, "bpf_link_create"))
|
||||
goto cleanup;
|
||||
|
||||
usleep(1); /* trigger */
|
||||
close(link_fd); /* detach */
|
||||
link_fd = -1;
|
||||
|
||||
ASSERT_EQ(skel->bss->tp_btf_res, cookie, "link_create_res");
|
||||
|
||||
/* high-level bpf_link-based bpf_program__attach_trace_opts() API */
|
||||
skel->bss->tp_btf_res = 0;
|
||||
|
||||
trace_opts.cookie = cookie = 0x33000000000000L;
|
||||
link = bpf_program__attach_trace_opts(skel->progs.handle_tp_btf, &trace_opts);
|
||||
if (!ASSERT_OK_PTR(link, "attach_trace_opts"))
|
||||
goto cleanup;
|
||||
|
||||
usleep(1); /* trigger */
|
||||
bpf_link__destroy(link); /* detach */
|
||||
link = NULL;
|
||||
|
||||
ASSERT_EQ(skel->bss->tp_btf_res, cookie, "attach_trace_opts_res");
|
||||
|
||||
cleanup:
|
||||
if (link_fd >= 0)
|
||||
close(link_fd);
|
||||
bpf_link__destroy(link);
|
||||
}
|
||||
|
||||
static void raw_tp_subtest(struct test_bpf_cookie *skel)
|
||||
{
|
||||
__u64 cookie;
|
||||
int prog_fd, link_fd = -1;
|
||||
struct bpf_link *link = NULL;
|
||||
LIBBPF_OPTS(bpf_raw_tp_opts, raw_tp_opts);
|
||||
LIBBPF_OPTS(bpf_raw_tracepoint_opts, opts);
|
||||
|
||||
/* There are two different ways to attach raw_tp programs */
|
||||
prog_fd = bpf_program__fd(skel->progs.handle_raw_tp);
|
||||
|
||||
/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */
|
||||
skel->bss->raw_tp_res = 0;
|
||||
|
||||
raw_tp_opts.tp_name = "sys_enter";
|
||||
raw_tp_opts.cookie = cookie = 0x55000000000000L;
|
||||
link_fd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_tp_opts);
|
||||
if (!ASSERT_GE(link_fd, 0, "bpf_raw_tracepoint_open_opts"))
|
||||
goto cleanup;
|
||||
|
||||
usleep(1); /* trigger */
|
||||
close(link_fd); /* detach */
|
||||
link_fd = -1;
|
||||
|
||||
ASSERT_EQ(skel->bss->raw_tp_res, cookie, "raw_tp_open_res");
|
||||
|
||||
/* high-level bpf_link-based bpf_program__attach_raw_tracepoint_opts() API */
|
||||
skel->bss->raw_tp_res = 0;
|
||||
|
||||
opts.cookie = cookie = 0x66000000000000L;
|
||||
link = bpf_program__attach_raw_tracepoint_opts(skel->progs.handle_raw_tp,
|
||||
"sys_enter", &opts);
|
||||
if (!ASSERT_OK_PTR(link, "attach_raw_tp_opts"))
|
||||
goto cleanup;
|
||||
|
||||
usleep(1); /* trigger */
|
||||
bpf_link__destroy(link); /* detach */
|
||||
link = NULL;
|
||||
|
||||
ASSERT_EQ(skel->bss->raw_tp_res, cookie, "attach_raw_tp_opts_res");
|
||||
|
||||
cleanup:
|
||||
if (link_fd >= 0)
|
||||
close(link_fd);
|
||||
bpf_link__destroy(link);
|
||||
}
|
||||
|
||||
void test_bpf_cookie(void)
|
||||
{
|
||||
struct test_bpf_cookie *skel;
|
||||
|
|
@ -601,6 +710,9 @@ void test_bpf_cookie(void)
|
|||
tracing_subtest(skel);
|
||||
if (test__start_subtest("lsm"))
|
||||
lsm_subtest(skel);
|
||||
|
||||
if (test__start_subtest("tp_btf"))
|
||||
tp_btf_subtest(skel);
|
||||
if (test__start_subtest("raw_tp"))
|
||||
raw_tp_subtest(skel);
|
||||
test_bpf_cookie__destroy(skel);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
|
|||
|
||||
WRITE_ONCE(stop, 0);
|
||||
|
||||
lfd = socket(AF_INET6, SOCK_STREAM, 0);
|
||||
lfd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0);
|
||||
if (!ASSERT_NEQ(lfd, -1, "socket"))
|
||||
return;
|
||||
|
||||
|
|
@ -103,21 +103,10 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
|
|||
settimeo(lfd, 0) || settimeo(fd, 0))
|
||||
goto done;
|
||||
|
||||
/* bind, listen and start server thread to accept */
|
||||
sa6.sin6_family = AF_INET6;
|
||||
sa6.sin6_addr = in6addr_loopback;
|
||||
err = bind(lfd, (struct sockaddr *)&sa6, addrlen);
|
||||
if (!ASSERT_NEQ(err, -1, "bind"))
|
||||
goto done;
|
||||
|
||||
err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen);
|
||||
if (!ASSERT_NEQ(err, -1, "getsockname"))
|
||||
goto done;
|
||||
|
||||
err = listen(lfd, 1);
|
||||
if (!ASSERT_NEQ(err, -1, "listen"))
|
||||
goto done;
|
||||
|
||||
if (sk_stg_map) {
|
||||
err = bpf_map_update_elem(bpf_map__fd(sk_stg_map), &fd,
|
||||
&expected_stg, BPF_NOEXIST);
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ static void test_lookup_update(void)
|
|||
int map1_fd, map2_fd, map3_fd, map4_fd, map5_fd, map1_id, map2_id;
|
||||
int outer_arr_fd, outer_hash_fd, outer_arr_dyn_fd;
|
||||
struct test_btf_map_in_map *skel;
|
||||
int err, key = 0, val, i, fd;
|
||||
int err, key = 0, val, i;
|
||||
|
||||
skel = test_btf_map_in_map__open_and_load();
|
||||
if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))
|
||||
|
|
@ -102,30 +102,6 @@ static void test_lookup_update(void)
|
|||
CHECK(map1_id == 0, "map1_id", "failed to get ID 1\n");
|
||||
CHECK(map2_id == 0, "map2_id", "failed to get ID 2\n");
|
||||
|
||||
test_btf_map_in_map__destroy(skel);
|
||||
skel = NULL;
|
||||
|
||||
/* we need to either wait for or force synchronize_rcu(), before
|
||||
* checking for "still exists" condition, otherwise map could still be
|
||||
* resolvable by ID, causing false positives.
|
||||
*
|
||||
* Older kernels (5.8 and earlier) freed map only after two
|
||||
* synchronize_rcu()s, so trigger two, to be entirely sure.
|
||||
*/
|
||||
CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
|
||||
CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
|
||||
|
||||
fd = bpf_map_get_fd_by_id(map1_id);
|
||||
if (CHECK(fd >= 0, "map1_leak", "inner_map1 leaked!\n")) {
|
||||
close(fd);
|
||||
goto cleanup;
|
||||
}
|
||||
fd = bpf_map_get_fd_by_id(map2_id);
|
||||
if (CHECK(fd >= 0, "map2_leak", "inner_map2 leaked!\n")) {
|
||||
close(fd);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
cleanup:
|
||||
test_btf_map_in_map__destroy(skel);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,77 +12,229 @@
|
|||
#include <sys/wait.h>
|
||||
#include <sys/mount.h>
|
||||
#include <sys/fcntl.h>
|
||||
#include "network_helpers.h"
|
||||
|
||||
#define STACK_SIZE (1024 * 1024)
|
||||
static char child_stack[STACK_SIZE];
|
||||
|
||||
static int test_current_pid_tgid(void *args)
|
||||
static int get_pid_tgid(pid_t *pid, pid_t *tgid,
|
||||
struct test_ns_current_pid_tgid__bss *bss)
|
||||
{
|
||||
struct test_ns_current_pid_tgid__bss *bss;
|
||||
struct test_ns_current_pid_tgid *skel;
|
||||
int err = -1, duration = 0;
|
||||
pid_t tgid, pid;
|
||||
struct stat st;
|
||||
int err;
|
||||
|
||||
skel = test_ns_current_pid_tgid__open_and_load();
|
||||
if (CHECK(!skel, "skel_open_load", "failed to load skeleton\n"))
|
||||
goto cleanup;
|
||||
|
||||
pid = syscall(SYS_gettid);
|
||||
tgid = getpid();
|
||||
*pid = syscall(SYS_gettid);
|
||||
*tgid = getpid();
|
||||
|
||||
err = stat("/proc/self/ns/pid", &st);
|
||||
if (CHECK(err, "stat", "failed /proc/self/ns/pid: %d\n", err))
|
||||
goto cleanup;
|
||||
if (!ASSERT_OK(err, "stat /proc/self/ns/pid"))
|
||||
return err;
|
||||
|
||||
bss = skel->bss;
|
||||
bss->dev = st.st_dev;
|
||||
bss->ino = st.st_ino;
|
||||
bss->user_pid = 0;
|
||||
bss->user_tgid = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int test_current_pid_tgid_tp(void *args)
|
||||
{
|
||||
struct test_ns_current_pid_tgid__bss *bss;
|
||||
struct test_ns_current_pid_tgid *skel;
|
||||
int ret = -1, err;
|
||||
pid_t tgid, pid;
|
||||
|
||||
skel = test_ns_current_pid_tgid__open();
|
||||
if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
|
||||
return ret;
|
||||
|
||||
bpf_program__set_autoload(skel->progs.tp_handler, true);
|
||||
|
||||
err = test_ns_current_pid_tgid__load(skel);
|
||||
if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
|
||||
goto cleanup;
|
||||
|
||||
bss = skel->bss;
|
||||
if (get_pid_tgid(&pid, &tgid, bss))
|
||||
goto cleanup;
|
||||
|
||||
err = test_ns_current_pid_tgid__attach(skel);
|
||||
if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
|
||||
if (!ASSERT_OK(err, "test_ns_current_pid_tgid__attach"))
|
||||
goto cleanup;
|
||||
|
||||
/* trigger tracepoint */
|
||||
usleep(1);
|
||||
ASSERT_EQ(bss->user_pid, pid, "pid");
|
||||
ASSERT_EQ(bss->user_tgid, tgid, "tgid");
|
||||
err = 0;
|
||||
if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
|
||||
goto cleanup;
|
||||
if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
|
||||
goto cleanup;
|
||||
ret = 0;
|
||||
|
||||
cleanup:
|
||||
test_ns_current_pid_tgid__destroy(skel);
|
||||
|
||||
return err;
|
||||
test_ns_current_pid_tgid__destroy(skel);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void test_ns_current_pid_tgid_new_ns(void)
|
||||
static int test_current_pid_tgid_cgrp(void *args)
|
||||
{
|
||||
int wstatus, duration = 0;
|
||||
struct test_ns_current_pid_tgid__bss *bss;
|
||||
struct test_ns_current_pid_tgid *skel;
|
||||
int server_fd = -1, ret = -1, err;
|
||||
int cgroup_fd = *(int *)args;
|
||||
pid_t tgid, pid;
|
||||
|
||||
skel = test_ns_current_pid_tgid__open();
|
||||
if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
|
||||
return ret;
|
||||
|
||||
bpf_program__set_autoload(skel->progs.cgroup_bind4, true);
|
||||
|
||||
err = test_ns_current_pid_tgid__load(skel);
|
||||
if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
|
||||
goto cleanup;
|
||||
|
||||
bss = skel->bss;
|
||||
if (get_pid_tgid(&pid, &tgid, bss))
|
||||
goto cleanup;
|
||||
|
||||
skel->links.cgroup_bind4 = bpf_program__attach_cgroup(
|
||||
skel->progs.cgroup_bind4, cgroup_fd);
|
||||
if (!ASSERT_OK_PTR(skel->links.cgroup_bind4, "bpf_program__attach_cgroup"))
|
||||
goto cleanup;
|
||||
|
||||
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
|
||||
if (!ASSERT_GE(server_fd, 0, "start_server"))
|
||||
goto cleanup;
|
||||
|
||||
if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
|
||||
goto cleanup;
|
||||
if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
|
||||
goto cleanup;
|
||||
ret = 0;
|
||||
|
||||
cleanup:
|
||||
if (server_fd >= 0)
|
||||
close(server_fd);
|
||||
test_ns_current_pid_tgid__destroy(skel);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int test_current_pid_tgid_sk_msg(void *args)
|
||||
{
|
||||
int verdict, map, server_fd = -1, client_fd = -1;
|
||||
struct test_ns_current_pid_tgid__bss *bss;
|
||||
static const char send_msg[] = "message";
|
||||
struct test_ns_current_pid_tgid *skel;
|
||||
int ret = -1, err, key = 0;
|
||||
pid_t tgid, pid;
|
||||
|
||||
skel = test_ns_current_pid_tgid__open();
|
||||
if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
|
||||
return ret;
|
||||
|
||||
bpf_program__set_autoload(skel->progs.sk_msg, true);
|
||||
|
||||
err = test_ns_current_pid_tgid__load(skel);
|
||||
if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
|
||||
goto cleanup;
|
||||
|
||||
bss = skel->bss;
|
||||
if (get_pid_tgid(&pid, &tgid, skel->bss))
|
||||
goto cleanup;
|
||||
|
||||
verdict = bpf_program__fd(skel->progs.sk_msg);
|
||||
map = bpf_map__fd(skel->maps.sock_map);
|
||||
err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
|
||||
if (!ASSERT_OK(err, "prog_attach"))
|
||||
goto cleanup;
|
||||
|
||||
server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
|
||||
if (!ASSERT_GE(server_fd, 0, "start_server"))
|
||||
goto cleanup;
|
||||
|
||||
client_fd = connect_to_fd(server_fd, 0);
|
||||
if (!ASSERT_GE(client_fd, 0, "connect_to_fd"))
|
||||
goto cleanup;
|
||||
|
||||
err = bpf_map_update_elem(map, &key, &client_fd, BPF_ANY);
|
||||
if (!ASSERT_OK(err, "bpf_map_update_elem"))
|
||||
goto cleanup;
|
||||
|
||||
err = send(client_fd, send_msg, sizeof(send_msg), 0);
|
||||
if (!ASSERT_EQ(err, sizeof(send_msg), "send(msg)"))
|
||||
goto cleanup;
|
||||
|
||||
if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
|
||||
goto cleanup;
|
||||
if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
|
||||
goto cleanup;
|
||||
ret = 0;
|
||||
|
||||
cleanup:
|
||||
if (server_fd >= 0)
|
||||
close(server_fd);
|
||||
if (client_fd >= 0)
|
||||
close(client_fd);
|
||||
test_ns_current_pid_tgid__destroy(skel);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void test_ns_current_pid_tgid_new_ns(int (*fn)(void *), void *arg)
|
||||
{
|
||||
int wstatus;
|
||||
pid_t cpid;
|
||||
|
||||
/* Create a process in a new namespace, this process
|
||||
* will be the init process of this new namespace hence will be pid 1.
|
||||
*/
|
||||
cpid = clone(test_current_pid_tgid, child_stack + STACK_SIZE,
|
||||
CLONE_NEWPID | SIGCHLD, NULL);
|
||||
cpid = clone(fn, child_stack + STACK_SIZE,
|
||||
CLONE_NEWPID | SIGCHLD, arg);
|
||||
|
||||
if (CHECK(cpid == -1, "clone", "%s\n", strerror(errno)))
|
||||
if (!ASSERT_NEQ(cpid, -1, "clone"))
|
||||
return;
|
||||
|
||||
if (CHECK(waitpid(cpid, &wstatus, 0) == -1, "waitpid", "%s\n", strerror(errno)))
|
||||
if (!ASSERT_NEQ(waitpid(cpid, &wstatus, 0), -1, "waitpid"))
|
||||
return;
|
||||
|
||||
if (CHECK(WEXITSTATUS(wstatus) != 0, "newns_pidtgid", "failed"))
|
||||
if (!ASSERT_OK(WEXITSTATUS(wstatus), "newns_pidtgid"))
|
||||
return;
|
||||
}
|
||||
|
||||
static void test_in_netns(int (*fn)(void *), void *arg)
|
||||
{
|
||||
struct nstoken *nstoken = NULL;
|
||||
|
||||
SYS(cleanup, "ip netns add ns_current_pid_tgid");
|
||||
SYS(cleanup, "ip -net ns_current_pid_tgid link set dev lo up");
|
||||
|
||||
nstoken = open_netns("ns_current_pid_tgid");
|
||||
if (!ASSERT_OK_PTR(nstoken, "open_netns"))
|
||||
goto cleanup;
|
||||
|
||||
test_ns_current_pid_tgid_new_ns(fn, arg);
|
||||
|
||||
cleanup:
|
||||
if (nstoken)
|
||||
close_netns(nstoken);
|
||||
SYS_NOFAIL("ip netns del ns_current_pid_tgid");
|
||||
}
|
||||
|
||||
/* TODO: use a different tracepoint */
|
||||
void serial_test_ns_current_pid_tgid(void)
|
||||
{
|
||||
if (test__start_subtest("ns_current_pid_tgid_root_ns"))
|
||||
test_current_pid_tgid(NULL);
|
||||
if (test__start_subtest("ns_current_pid_tgid_new_ns"))
|
||||
test_ns_current_pid_tgid_new_ns();
|
||||
if (test__start_subtest("root_ns_tp"))
|
||||
test_current_pid_tgid_tp(NULL);
|
||||
if (test__start_subtest("new_ns_tp"))
|
||||
test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL);
|
||||
if (test__start_subtest("new_ns_cgrp")) {
|
||||
int cgroup_fd = -1;
|
||||
|
||||
cgroup_fd = test__join_cgroup("/sock_addr");
|
||||
if (ASSERT_GE(cgroup_fd, 0, "join_cgroup")) {
|
||||
test_in_netns(test_current_pid_tgid_cgrp, &cgroup_fd);
|
||||
close(cgroup_fd);
|
||||
}
|
||||
}
|
||||
if (test__start_subtest("new_ns_sk_msg"))
|
||||
test_in_netns(test_current_pid_tgid_sk_msg, NULL);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -93,9 +93,56 @@ cleanup:
|
|||
struct_ops_module__destroy(skel);
|
||||
}
|
||||
|
||||
static void test_struct_ops_not_zeroed(void)
|
||||
{
|
||||
struct struct_ops_module *skel;
|
||||
int err;
|
||||
|
||||
/* zeroed is 0, and zeroed_op is null */
|
||||
skel = struct_ops_module__open();
|
||||
if (!ASSERT_OK_PTR(skel, "struct_ops_module_open"))
|
||||
return;
|
||||
|
||||
err = struct_ops_module__load(skel);
|
||||
ASSERT_OK(err, "struct_ops_module_load");
|
||||
|
||||
struct_ops_module__destroy(skel);
|
||||
|
||||
/* zeroed is not 0 */
|
||||
skel = struct_ops_module__open();
|
||||
if (!ASSERT_OK_PTR(skel, "struct_ops_module_open_not_zeroed"))
|
||||
return;
|
||||
|
||||
/* libbpf should reject the testmod_zeroed since struct
|
||||
* bpf_testmod_ops in the kernel has no "zeroed" field and the
|
||||
* value of "zeroed" is non-zero.
|
||||
*/
|
||||
skel->struct_ops.testmod_zeroed->zeroed = 0xdeadbeef;
|
||||
err = struct_ops_module__load(skel);
|
||||
ASSERT_ERR(err, "struct_ops_module_load_not_zeroed");
|
||||
|
||||
struct_ops_module__destroy(skel);
|
||||
|
||||
/* zeroed_op is not null */
|
||||
skel = struct_ops_module__open();
|
||||
if (!ASSERT_OK_PTR(skel, "struct_ops_module_open_not_zeroed_op"))
|
||||
return;
|
||||
|
||||
/* libbpf should reject the testmod_zeroed since the value of its
|
||||
* "zeroed_op" is not null.
|
||||
*/
|
||||
skel->struct_ops.testmod_zeroed->zeroed_op = skel->progs.test_3;
|
||||
err = struct_ops_module__load(skel);
|
||||
ASSERT_ERR(err, "struct_ops_module_load_not_zeroed_op");
|
||||
|
||||
struct_ops_module__destroy(skel);
|
||||
}
|
||||
|
||||
void serial_test_struct_ops_module(void)
|
||||
{
|
||||
if (test__start_subtest("test_struct_ops_load"))
|
||||
test_struct_ops_load();
|
||||
if (test__start_subtest("test_struct_ops_not_zeroed"))
|
||||
test_struct_ops_not_zeroed();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -673,7 +673,7 @@ static __noinline void fill(struct bpf_iter_num *it, int *arr, __u32 n, int mul)
|
|||
|
||||
static __noinline int sum(struct bpf_iter_num *it, int *arr, __u32 n)
|
||||
{
|
||||
int *t, i, sum = 0;;
|
||||
int *t, i, sum = 0;
|
||||
|
||||
while ((t = bpf_iter_num_next(it))) {
|
||||
i = *t;
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ void BPF_PROG(test_2, int a, int b)
|
|||
test_2_result = a + b;
|
||||
}
|
||||
|
||||
SEC("struct_ops/test_3")
|
||||
SEC("?struct_ops/test_3")
|
||||
int BPF_PROG(test_3, int a, int b)
|
||||
{
|
||||
test_2_result = a + b + 3;
|
||||
|
|
@ -54,3 +54,17 @@ struct bpf_testmod_ops___v2 testmod_2 = {
|
|||
.test_1 = (void *)test_1,
|
||||
.test_2 = (void *)test_2_v2,
|
||||
};
|
||||
|
||||
struct bpf_testmod_ops___zeroed {
|
||||
int (*test_1)(void);
|
||||
void (*test_2)(int a, int b);
|
||||
int (*test_maybe_null)(int dummy, struct task_struct *task);
|
||||
void (*zeroed_op)(int a, int b);
|
||||
int zeroed;
|
||||
};
|
||||
|
||||
SEC(".struct_ops.link")
|
||||
struct bpf_testmod_ops___zeroed testmod_zeroed = {
|
||||
.test_1 = (void *)test_1,
|
||||
.test_2 = (void *)test_2_v2,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ __u64 uprobe_res;
|
|||
__u64 uretprobe_res;
|
||||
__u64 tp_res;
|
||||
__u64 pe_res;
|
||||
__u64 raw_tp_res;
|
||||
__u64 tp_btf_res;
|
||||
__u64 fentry_res;
|
||||
__u64 fexit_res;
|
||||
__u64 fmod_ret_res;
|
||||
|
|
@ -87,6 +89,20 @@ int handle_pe(struct pt_regs *ctx)
|
|||
return 0;
|
||||
}
|
||||
|
||||
SEC("raw_tp/sys_enter")
|
||||
int handle_raw_tp(void *ctx)
|
||||
{
|
||||
update(ctx, &raw_tp_res);
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("tp_btf/sys_enter")
|
||||
int handle_tp_btf(void *ctx)
|
||||
{
|
||||
update(ctx, &tp_btf_res);
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fentry/bpf_fentry_test1")
|
||||
int BPF_PROG(fentry_test1, int a)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -5,23 +5,48 @@
|
|||
#include <stdint.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_SOCKMAP);
|
||||
__uint(max_entries, 2);
|
||||
__type(key, __u32);
|
||||
__type(value, __u32);
|
||||
} sock_map SEC(".maps");
|
||||
|
||||
__u64 user_pid = 0;
|
||||
__u64 user_tgid = 0;
|
||||
__u64 dev = 0;
|
||||
__u64 ino = 0;
|
||||
|
||||
SEC("tracepoint/syscalls/sys_enter_nanosleep")
|
||||
int handler(const void *ctx)
|
||||
static void get_pid_tgid(void)
|
||||
{
|
||||
struct bpf_pidns_info nsdata;
|
||||
|
||||
if (bpf_get_ns_current_pid_tgid(dev, ino, &nsdata, sizeof(struct bpf_pidns_info)))
|
||||
return 0;
|
||||
return;
|
||||
|
||||
user_pid = nsdata.pid;
|
||||
user_tgid = nsdata.tgid;
|
||||
}
|
||||
|
||||
SEC("?tracepoint/syscalls/sys_enter_nanosleep")
|
||||
int tp_handler(const void *ctx)
|
||||
{
|
||||
get_pid_tgid();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("?cgroup/bind4")
|
||||
int cgroup_bind4(struct bpf_sock_addr *ctx)
|
||||
{
|
||||
get_pid_tgid();
|
||||
return 1;
|
||||
}
|
||||
|
||||
SEC("?sk_msg")
|
||||
int sk_msg(struct sk_msg_md *msg)
|
||||
{
|
||||
get_pid_tgid();
|
||||
return SK_PASS;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
|
|||
|
|
@ -9,12 +9,27 @@
|
|||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
||||
long hits = 0;
|
||||
#define CPU_MASK 255
|
||||
#define MAX_CPUS (CPU_MASK + 1) /* should match MAX_BUCKETS in benchs/bench_trigger.c */
|
||||
|
||||
/* matches struct counter in bench.h */
|
||||
struct counter {
|
||||
long value;
|
||||
} __attribute__((aligned(128)));
|
||||
|
||||
struct counter hits[MAX_CPUS];
|
||||
|
||||
static __always_inline void inc_counter(void)
|
||||
{
|
||||
int cpu = bpf_get_smp_processor_id();
|
||||
|
||||
__sync_add_and_fetch(&hits[cpu & CPU_MASK].value, 1);
|
||||
}
|
||||
|
||||
SEC("tp/syscalls/sys_enter_getpgid")
|
||||
int bench_trigger_tp(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -22,69 +37,69 @@ SEC("raw_tp/sys_enter")
|
|||
int BPF_PROG(bench_trigger_raw_tp, struct pt_regs *regs, long id)
|
||||
{
|
||||
if (id == __NR_getpgid)
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kprobe/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_kprobe(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kretprobe/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_kretprobe(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kprobe.multi/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_kprobe_multi(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("kretprobe.multi/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_kretprobe_multi(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fentry/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_fentry(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_fexit(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_fentry_sleep(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fmod_ret/" SYS_PREFIX "sys_getpgid")
|
||||
int bench_trigger_fmodret(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return -22;
|
||||
}
|
||||
|
||||
SEC("uprobe")
|
||||
int bench_trigger_uprobe(void *ctx)
|
||||
{
|
||||
__sync_add_and_fetch(&hits, 1);
|
||||
inc_counter();
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include <bpf/bpf.h>
|
||||
#include <bpf/btf.h>
|
||||
#include "test_core_extern.skel.h"
|
||||
#include "struct_ops_module.skel.h"
|
||||
|
||||
template <typename T>
|
||||
class Skeleton {
|
||||
|
|
@ -98,6 +99,7 @@ int main(int argc, char *argv[])
|
|||
{
|
||||
struct btf_dump_opts opts = { };
|
||||
struct test_core_extern *skel;
|
||||
struct struct_ops_module *skel2;
|
||||
struct btf *btf;
|
||||
int fd;
|
||||
|
||||
|
|
@ -118,6 +120,9 @@ int main(int argc, char *argv[])
|
|||
skel = test_core_extern__open_and_load();
|
||||
test_core_extern__destroy(skel);
|
||||
|
||||
skel2 = struct_ops_module__open_and_load();
|
||||
struct_ops_module__destroy(skel2);
|
||||
|
||||
fd = bpf_enable_stats(BPF_STATS_RUN_TIME);
|
||||
if (fd < 0)
|
||||
std::cout << "FAILED to enable stats: " << fd << std::endl;
|
||||
|
|
|
|||
|
|
@ -72,7 +72,6 @@ cleanup() {
|
|||
server_listen() {
|
||||
ip netns exec "${ns2}" nc "${netcat_opt}" -l "${port}" > "${outfile}" &
|
||||
server_pid=$!
|
||||
sleep 0.2
|
||||
}
|
||||
|
||||
client_connect() {
|
||||
|
|
@ -93,6 +92,16 @@ verify_data() {
|
|||
fi
|
||||
}
|
||||
|
||||
wait_for_port() {
|
||||
for i in $(seq 20); do
|
||||
if ip netns exec "${ns2}" ss ${2:--4}OHntl | grep -q "$1"; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
# no arguments: automated test, run all
|
||||
|
|
@ -193,6 +202,7 @@ setup
|
|||
# basic communication works
|
||||
echo "test basic connectivity"
|
||||
server_listen
|
||||
wait_for_port ${port} ${netcat_opt}
|
||||
client_connect
|
||||
verify_data
|
||||
|
||||
|
|
@ -204,6 +214,7 @@ ip netns exec "${ns1}" tc filter add dev veth1 egress \
|
|||
section "encap_${tuntype}_${mac}"
|
||||
echo "test bpf encap without decap (expect failure)"
|
||||
server_listen
|
||||
wait_for_port ${port} ${netcat_opt}
|
||||
! client_connect
|
||||
|
||||
if [[ "$tuntype" =~ "udp" ]]; then
|
||||
|
|
|
|||
Loading…
Reference in New Issue