mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-05 18:39:59 +00:00
![Naveen N. Rao](/assets/img/avatar_default.png)
Add checks to ensure that we never emit branch instructions with truncated branch offsets. Suggested-by: Michael Ellerman <mpe@ellerman.id.au> Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> Tested-by: Johan Almbladh <johan.almbladh@anyfinetworks.com> Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu> Acked-by: Song Liu <songliubraving@fb.com> Acked-by: Johan Almbladh <johan.almbladh@anyfinetworks.com> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/71d33a6b7603ec1013c9734dd8bdd4ff5e929142.1633464148.git.naveen.n.rao@linux.vnet.ibm.com
264 lines
6.4 KiB
C
264 lines
6.4 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* eBPF JIT compiler
|
|
*
|
|
* Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
|
|
* IBM Corporation
|
|
*
|
|
* Based on the powerpc classic BPF JIT compiler by Matt Evans
|
|
*/
|
|
#include <linux/moduleloader.h>
|
|
#include <asm/cacheflush.h>
|
|
#include <asm/asm-compat.h>
|
|
#include <linux/netdevice.h>
|
|
#include <linux/filter.h>
|
|
#include <linux/if_vlan.h>
|
|
#include <asm/kprobes.h>
|
|
#include <linux/bpf.h>
|
|
|
|
#include "bpf_jit.h"
|
|
|
|
static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
|
|
{
|
|
memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
|
|
}
|
|
|
|
/* Fix the branch target addresses for subprog calls */
|
|
static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image,
|
|
struct codegen_context *ctx, u32 *addrs)
|
|
{
|
|
const struct bpf_insn *insn = fp->insnsi;
|
|
bool func_addr_fixed;
|
|
u64 func_addr;
|
|
u32 tmp_idx;
|
|
int i, ret;
|
|
|
|
for (i = 0; i < fp->len; i++) {
|
|
/*
|
|
* During the extra pass, only the branch target addresses for
|
|
* the subprog calls need to be fixed. All other instructions
|
|
* can left untouched.
|
|
*
|
|
* The JITed image length does not change because we already
|
|
* ensure that the JITed instruction sequence for these calls
|
|
* are of fixed length by padding them with NOPs.
|
|
*/
|
|
if (insn[i].code == (BPF_JMP | BPF_CALL) &&
|
|
insn[i].src_reg == BPF_PSEUDO_CALL) {
|
|
ret = bpf_jit_get_func_addr(fp, &insn[i], true,
|
|
&func_addr,
|
|
&func_addr_fixed);
|
|
if (ret < 0)
|
|
return ret;
|
|
|
|
/*
|
|
* Save ctx->idx as this would currently point to the
|
|
* end of the JITed image and set it to the offset of
|
|
* the instruction sequence corresponding to the
|
|
* subprog call temporarily.
|
|
*/
|
|
tmp_idx = ctx->idx;
|
|
ctx->idx = addrs[i] / 4;
|
|
bpf_jit_emit_func_call_rel(image, ctx, func_addr);
|
|
|
|
/*
|
|
* Restore ctx->idx here. This is safe as the length
|
|
* of the JITed sequence remains unchanged.
|
|
*/
|
|
ctx->idx = tmp_idx;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct powerpc64_jit_data {
|
|
struct bpf_binary_header *header;
|
|
u32 *addrs;
|
|
u8 *image;
|
|
u32 proglen;
|
|
struct codegen_context ctx;
|
|
};
|
|
|
|
bool bpf_jit_needs_zext(void)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
|
|
{
|
|
u32 proglen;
|
|
u32 alloclen;
|
|
u8 *image = NULL;
|
|
u32 *code_base;
|
|
u32 *addrs;
|
|
struct powerpc64_jit_data *jit_data;
|
|
struct codegen_context cgctx;
|
|
int pass;
|
|
int flen;
|
|
struct bpf_binary_header *bpf_hdr;
|
|
struct bpf_prog *org_fp = fp;
|
|
struct bpf_prog *tmp_fp;
|
|
bool bpf_blinded = false;
|
|
bool extra_pass = false;
|
|
|
|
if (!fp->jit_requested)
|
|
return org_fp;
|
|
|
|
tmp_fp = bpf_jit_blind_constants(org_fp);
|
|
if (IS_ERR(tmp_fp))
|
|
return org_fp;
|
|
|
|
if (tmp_fp != org_fp) {
|
|
bpf_blinded = true;
|
|
fp = tmp_fp;
|
|
}
|
|
|
|
jit_data = fp->aux->jit_data;
|
|
if (!jit_data) {
|
|
jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
|
|
if (!jit_data) {
|
|
fp = org_fp;
|
|
goto out;
|
|
}
|
|
fp->aux->jit_data = jit_data;
|
|
}
|
|
|
|
flen = fp->len;
|
|
addrs = jit_data->addrs;
|
|
if (addrs) {
|
|
cgctx = jit_data->ctx;
|
|
image = jit_data->image;
|
|
bpf_hdr = jit_data->header;
|
|
proglen = jit_data->proglen;
|
|
alloclen = proglen + FUNCTION_DESCR_SIZE;
|
|
extra_pass = true;
|
|
goto skip_init_ctx;
|
|
}
|
|
|
|
addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
|
|
if (addrs == NULL) {
|
|
fp = org_fp;
|
|
goto out_addrs;
|
|
}
|
|
|
|
memset(&cgctx, 0, sizeof(struct codegen_context));
|
|
memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p));
|
|
|
|
/* Make sure that the stack is quadword aligned. */
|
|
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);
|
|
|
|
/* Scouting faux-generate pass 0 */
|
|
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
|
|
/* We hit something illegal or unsupported. */
|
|
fp = org_fp;
|
|
goto out_addrs;
|
|
}
|
|
|
|
/*
|
|
* If we have seen a tail call, we need a second pass.
|
|
* This is because bpf_jit_emit_common_epilogue() is called
|
|
* from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
|
|
*/
|
|
if (cgctx.seen & SEEN_TAILCALL) {
|
|
cgctx.idx = 0;
|
|
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) {
|
|
fp = org_fp;
|
|
goto out_addrs;
|
|
}
|
|
}
|
|
|
|
bpf_jit_realloc_regs(&cgctx);
|
|
/*
|
|
* Pretend to build prologue, given the features we've seen. This will
|
|
* update ctgtx.idx as it pretends to output instructions, then we can
|
|
* calculate total size from idx.
|
|
*/
|
|
bpf_jit_build_prologue(0, &cgctx);
|
|
bpf_jit_build_epilogue(0, &cgctx);
|
|
|
|
proglen = cgctx.idx * 4;
|
|
alloclen = proglen + FUNCTION_DESCR_SIZE;
|
|
|
|
bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
|
|
if (!bpf_hdr) {
|
|
fp = org_fp;
|
|
goto out_addrs;
|
|
}
|
|
|
|
skip_init_ctx:
|
|
code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
|
|
|
|
if (extra_pass) {
|
|
/*
|
|
* Do not touch the prologue and epilogue as they will remain
|
|
* unchanged. Only fix the branch target address for subprog
|
|
* calls in the body.
|
|
*
|
|
* This does not change the offsets and lengths of the subprog
|
|
* call instruction sequences and hence, the size of the JITed
|
|
* image as well.
|
|
*/
|
|
bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs);
|
|
|
|
/* There is no need to perform the usual passes. */
|
|
goto skip_codegen_passes;
|
|
}
|
|
|
|
/* Code generation passes 1-2 */
|
|
for (pass = 1; pass < 3; pass++) {
|
|
/* Now build the prologue, body code & epilogue for real. */
|
|
cgctx.idx = 0;
|
|
bpf_jit_build_prologue(code_base, &cgctx);
|
|
if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass)) {
|
|
bpf_jit_binary_free(bpf_hdr);
|
|
fp = org_fp;
|
|
goto out_addrs;
|
|
}
|
|
bpf_jit_build_epilogue(code_base, &cgctx);
|
|
|
|
if (bpf_jit_enable > 1)
|
|
pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
|
|
proglen - (cgctx.idx * 4), cgctx.seen);
|
|
}
|
|
|
|
skip_codegen_passes:
|
|
if (bpf_jit_enable > 1)
|
|
/*
|
|
* Note that we output the base address of the code_base
|
|
* rather than image, since opcodes are in code_base.
|
|
*/
|
|
bpf_jit_dump(flen, proglen, pass, code_base);
|
|
|
|
#ifdef PPC64_ELF_ABI_v1
|
|
/* Function descriptor nastiness: Address + TOC */
|
|
((u64 *)image)[0] = (u64)code_base;
|
|
((u64 *)image)[1] = local_paca->kernel_toc;
|
|
#endif
|
|
|
|
fp->bpf_func = (void *)image;
|
|
fp->jited = 1;
|
|
fp->jited_len = alloclen;
|
|
|
|
bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
|
|
bpf_jit_binary_lock_ro(bpf_hdr);
|
|
if (!fp->is_func || extra_pass) {
|
|
bpf_prog_fill_jited_linfo(fp, addrs);
|
|
out_addrs:
|
|
kfree(addrs);
|
|
kfree(jit_data);
|
|
fp->aux->jit_data = NULL;
|
|
} else {
|
|
jit_data->addrs = addrs;
|
|
jit_data->ctx = cgctx;
|
|
jit_data->proglen = proglen;
|
|
jit_data->image = image;
|
|
jit_data->header = bpf_hdr;
|
|
}
|
|
|
|
out:
|
|
if (bpf_blinded)
|
|
bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
|
|
|
|
return fp;
|
|
}
|