2020-03-05 05:02:04 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* Common functionality for RV32 and RV64 BPF JIT compilers
|
|
|
|
*
|
|
|
|
* Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/bpf.h>
|
|
|
|
#include <linux/filter.h>
|
|
|
|
#include "bpf_jit.h"
|
|
|
|
|
|
|
|
/* Number of iterations to try until offsets converge. */
|
2021-10-28 16:10:54 +00:00
|
|
|
#define NR_JIT_ITERATIONS 32
|
2020-03-05 05:02:04 +00:00
|
|
|
|
|
|
|
static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
|
|
|
|
{
|
|
|
|
const struct bpf_prog *prog = ctx->prog;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < prog->len; i++) {
|
|
|
|
const struct bpf_insn *insn = &prog->insnsi[i];
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
|
|
|
|
/* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
|
|
|
|
if (ret > 0)
|
|
|
|
i++;
|
|
|
|
if (offset)
|
|
|
|
offset[i] = ctx->ninsns;
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool bpf_jit_needs_zext(void)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
|
|
|
|
{
|
2021-10-27 11:18:22 +00:00
|
|
|
unsigned int prog_size = 0, extable_size = 0;
|
2020-03-05 05:02:04 +00:00
|
|
|
bool tmp_blinded = false, extra_pass = false;
|
|
|
|
struct bpf_prog *tmp, *orig_prog = prog;
|
|
|
|
int pass = 0, prev_ninsns = 0, i;
|
|
|
|
struct rv_jit_data *jit_data;
|
|
|
|
struct rv_jit_context *ctx;
|
|
|
|
|
|
|
|
if (!prog->jit_requested)
|
|
|
|
return orig_prog;
|
|
|
|
|
|
|
|
tmp = bpf_jit_blind_constants(prog);
|
|
|
|
if (IS_ERR(tmp))
|
|
|
|
return orig_prog;
|
|
|
|
if (tmp != prog) {
|
|
|
|
tmp_blinded = true;
|
|
|
|
prog = tmp;
|
|
|
|
}
|
|
|
|
|
|
|
|
jit_data = prog->aux->jit_data;
|
|
|
|
if (!jit_data) {
|
|
|
|
jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
|
|
|
|
if (!jit_data) {
|
|
|
|
prog = orig_prog;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
prog->aux->jit_data = jit_data;
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx = &jit_data->ctx;
|
|
|
|
|
|
|
|
if (ctx->offset) {
|
|
|
|
extra_pass = true;
|
2021-10-27 11:18:22 +00:00
|
|
|
prog_size = sizeof(*ctx->insns) * ctx->ninsns;
|
2020-03-05 05:02:04 +00:00
|
|
|
goto skip_init_ctx;
|
|
|
|
}
|
|
|
|
|
|
|
|
ctx->prog = prog;
|
|
|
|
ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
|
|
|
|
if (!ctx->offset) {
|
|
|
|
prog = orig_prog;
|
|
|
|
goto out_offset;
|
|
|
|
}
|
|
|
|
for (i = 0; i < prog->len; i++) {
|
|
|
|
prev_ninsns += 32;
|
|
|
|
ctx->offset[i] = prev_ninsns;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < NR_JIT_ITERATIONS; i++) {
|
|
|
|
pass++;
|
|
|
|
ctx->ninsns = 0;
|
|
|
|
if (build_body(ctx, extra_pass, ctx->offset)) {
|
|
|
|
prog = orig_prog;
|
|
|
|
goto out_offset;
|
|
|
|
}
|
|
|
|
bpf_jit_build_prologue(ctx);
|
|
|
|
ctx->epilogue_offset = ctx->ninsns;
|
|
|
|
bpf_jit_build_epilogue(ctx);
|
|
|
|
|
|
|
|
if (ctx->ninsns == prev_ninsns) {
|
|
|
|
if (jit_data->header)
|
|
|
|
break;
|
2021-10-27 11:18:22 +00:00
|
|
|
/* obtain the actual image size */
|
|
|
|
extable_size = prog->aux->num_exentries *
|
|
|
|
sizeof(struct exception_table_entry);
|
|
|
|
prog_size = sizeof(*ctx->insns) * ctx->ninsns;
|
2020-03-05 05:02:04 +00:00
|
|
|
|
|
|
|
jit_data->header =
|
2021-10-27 11:18:22 +00:00
|
|
|
bpf_jit_binary_alloc(prog_size + extable_size,
|
2020-03-05 05:02:04 +00:00
|
|
|
&jit_data->image,
|
|
|
|
sizeof(u32),
|
|
|
|
bpf_fill_ill_insns);
|
|
|
|
if (!jit_data->header) {
|
|
|
|
prog = orig_prog;
|
|
|
|
goto out_offset;
|
|
|
|
}
|
|
|
|
|
2020-07-21 02:52:38 +00:00
|
|
|
ctx->insns = (u16 *)jit_data->image;
|
2020-03-05 05:02:04 +00:00
|
|
|
/*
|
|
|
|
* Now, when the image is allocated, the image can
|
|
|
|
* potentially shrink more (auipc/jalr -> jal).
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
prev_ninsns = ctx->ninsns;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i == NR_JIT_ITERATIONS) {
|
|
|
|
pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
|
2021-10-28 12:51:15 +00:00
|
|
|
if (jit_data->header)
|
|
|
|
bpf_jit_binary_free(jit_data->header);
|
2020-03-05 05:02:04 +00:00
|
|
|
prog = orig_prog;
|
|
|
|
goto out_offset;
|
|
|
|
}
|
|
|
|
|
2021-10-27 11:18:22 +00:00
|
|
|
if (extable_size)
|
|
|
|
prog->aux->extable = (void *)ctx->insns + prog_size;
|
|
|
|
|
2020-03-05 05:02:04 +00:00
|
|
|
skip_init_ctx:
|
|
|
|
pass++;
|
|
|
|
ctx->ninsns = 0;
|
2021-10-27 11:18:22 +00:00
|
|
|
ctx->nexentries = 0;
|
2020-03-05 05:02:04 +00:00
|
|
|
|
|
|
|
bpf_jit_build_prologue(ctx);
|
|
|
|
if (build_body(ctx, extra_pass, NULL)) {
|
|
|
|
bpf_jit_binary_free(jit_data->header);
|
|
|
|
prog = orig_prog;
|
|
|
|
goto out_offset;
|
|
|
|
}
|
|
|
|
bpf_jit_build_epilogue(ctx);
|
|
|
|
|
|
|
|
if (bpf_jit_enable > 1)
|
2021-10-27 11:18:22 +00:00
|
|
|
bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
|
2020-03-05 05:02:04 +00:00
|
|
|
|
|
|
|
prog->bpf_func = (void *)ctx->insns;
|
|
|
|
prog->jited = 1;
|
2021-10-27 11:18:22 +00:00
|
|
|
prog->jited_len = prog_size;
|
2020-03-05 05:02:04 +00:00
|
|
|
|
|
|
|
bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
|
|
|
|
|
|
|
|
if (!prog->is_func || extra_pass) {
|
2021-03-29 18:25:21 +00:00
|
|
|
bpf_jit_binary_lock_ro(jit_data->header);
|
2020-03-05 05:02:04 +00:00
|
|
|
out_offset:
|
|
|
|
kfree(ctx->offset);
|
|
|
|
kfree(jit_data);
|
|
|
|
prog->aux->jit_data = NULL;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
|
|
|
|
if (tmp_blinded)
|
|
|
|
bpf_jit_prog_release_other(prog, prog == orig_prog ?
|
|
|
|
tmp : orig_prog);
|
|
|
|
return prog;
|
|
|
|
}
|
2021-03-29 18:24:54 +00:00
|
|
|
|
2021-10-14 14:25:51 +00:00
|
|
|
u64 bpf_jit_alloc_exec_limit(void)
|
|
|
|
{
|
|
|
|
return BPF_JIT_REGION_SIZE;
|
|
|
|
}
|
|
|
|
|
2021-03-29 18:24:54 +00:00
|
|
|
void *bpf_jit_alloc_exec(unsigned long size)
|
|
|
|
{
|
|
|
|
return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
|
|
|
|
BPF_JIT_REGION_END, GFP_KERNEL,
|
2021-03-29 18:25:21 +00:00
|
|
|
PAGE_KERNEL, 0, NUMA_NO_NODE,
|
2021-03-29 18:24:54 +00:00
|
|
|
__builtin_return_address(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
void bpf_jit_free_exec(void *addr)
|
|
|
|
{
|
|
|
|
return vfree(addr);
|
|
|
|
}
|