mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-08 01:29:02 +00:00
59d2dc7710
commit35b6b28e69
upstream. When branch target identifiers are in use, code reachable via an indirect branch requires a BTI landing pad at the branch target site. When building FTRACE_WITH_REGS atop patchable-function-entry, we miss BTIs at the start start of the `ftrace_caller` and `ftrace_regs_caller` trampolines, and when these are called from a module via a PLT (which will use a `BR X16`), we will encounter a BTI failure, e.g. | # insmod lkdtm.ko | lkdtm: No crash points registered, enable through debugfs | # echo function_graph > /sys/kernel/debug/tracing/current_tracer | # cat /sys/kernel/debug/provoke-crash/DIRECT | Unhandled 64-bit el1h sync exception on CPU0, ESR 0x34000001 -- BTI | CPU: 0 PID: 174 Comm: cat Not tainted 5.16.0-rc2-dirty #3 | Hardware name: linux,dummy-virt (DT) | pstate: 60400405 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=jc) | pc : ftrace_caller+0x0/0x3c | lr : lkdtm_debugfs_open+0xc/0x20 [lkdtm] | sp : ffff800012e43b00 | x29: ffff800012e43b00 x28: 0000000000000000 x27: ffff800012e43c88 | x26: 0000000000000000 x25: 0000000000000000 x24: ffff0000c171f200 | x23: ffff0000c27b1e00 x22: ffff0000c2265240 x21: ffff0000c23c8c30 | x20: ffff8000090ba380 x19: 0000000000000000 x18: 0000000000000000 | x17: 0000000000000000 x16: ffff80001002bb4c x15: 0000000000000000 | x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000900ff0 | x11: ffff0000c4166310 x10: ffff800012e43b00 x9 : ffff8000104f2384 | x8 : 0000000000000001 x7 : 0000000000000000 x6 : 000000000000003f | x5 : 0000000000000040 x4 : ffff800012e43af0 x3 : 0000000000000001 | x2 : ffff8000090b0000 x1 : ffff0000c171f200 x0 : ffff0000c23c8c30 | Kernel panic - not syncing: Unhandled exception | CPU: 0 PID: 174 Comm: cat Not tainted 5.16.0-rc2-dirty #3 | Hardware name: linux,dummy-virt (DT) | Call trace: | dump_backtrace+0x0/0x1a4 | show_stack+0x24/0x30 | dump_stack_lvl+0x68/0x84 | dump_stack+0x1c/0x38 | panic+0x168/0x360 | arm64_exit_nmi.isra.0+0x0/0x80 | el1h_64_sync_handler+0x68/0xd4 | el1h_64_sync+0x78/0x7c | ftrace_caller+0x0/0x3c | do_dentry_open+0x134/0x3b0 | vfs_open+0x38/0x44 | path_openat+0x89c/0xe40 | do_filp_open+0x8c/0x13c | do_sys_openat2+0xbc/0x174 | __arm64_sys_openat+0x6c/0xbc | invoke_syscall+0x50/0x120 | el0_svc_common.constprop.0+0xdc/0x100 | do_el0_svc+0x84/0xa0 | el0_svc+0x28/0x80 | el0t_64_sync_handler+0xa8/0x130 | el0t_64_sync+0x1a0/0x1a4 | SMP: stopping secondary CPUs | Kernel Offset: disabled | CPU features: 0x0,00000f42,da660c5f | Memory Limit: none | ---[ end Kernel panic - not syncing: Unhandled exception ]--- Fix this by adding the required `BTI C`, as we only require these to be reachable via BL for direct calls or BR X16/X17 for PLTs. For now, these are open-coded in the function prologue, matching the style of the `__hwasan_tag_mismatch` trampoline. In future we may wish to consider adding a new SYM_CODE_START_*() variant which has an implicit BTI. When ftrace is built atop mcount, the trampolines are marked with SYM_FUNC_START(), and so get an implicit BTI. We may need to change these over to SYM_CODE_START() in future for RELIABLE_STACKTRACE, in case we need to apply special care aroud the return address being rewritten. Fixes:97fed779f2
("arm64: bti: Provide Kconfig for kernel mode BTI") Signed-off-by: Mark Rutland <mark.rutland@arm.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Mark Brown <broonie@kernel.org> Cc: Will Deacon <will@kernel.org> Reviewed-by: Mark Brown <broonie@kernel.org> Link: https://lore.kernel.org/r/20211129135709.2274019-1-mark.rutland@arm.com Signed-off-by: Will Deacon <will@kernel.org> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
351 lines
9.6 KiB
ArmAsm
351 lines
9.6 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* arch/arm64/kernel/entry-ftrace.S
|
|
*
|
|
* Copyright (C) 2013 Linaro Limited
|
|
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <asm/asm-offsets.h>
|
|
#include <asm/assembler.h>
|
|
#include <asm/ftrace.h>
|
|
#include <asm/insn.h>
|
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
|
/*
|
|
* Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
|
|
* the regular function prologue. For an enabled callsite, ftrace_init_nop() and
|
|
* ftrace_make_call() have patched those NOPs to:
|
|
*
|
|
* MOV X9, LR
|
|
* BL <entry>
|
|
*
|
|
* ... where <entry> is either ftrace_caller or ftrace_regs_caller.
|
|
*
|
|
* Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
|
|
* live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
|
|
* clobber.
|
|
*
|
|
* We save the callsite's context into a pt_regs before invoking any ftrace
|
|
* callbacks. So that we can get a sensible backtrace, we create a stack record
|
|
* for the callsite and the ftrace entry assembly. This is not sufficient for
|
|
* reliable stacktrace: until we create the callsite stack record, its caller
|
|
* is missing from the LR and existing chain of frame records.
|
|
*/
|
|
.macro ftrace_regs_entry, allregs=0
|
|
/* Make room for pt_regs, plus a callee frame */
|
|
sub sp, sp, #(S_FRAME_SIZE + 16)
|
|
|
|
/* Save function arguments (and x9 for simplicity) */
|
|
stp x0, x1, [sp, #S_X0]
|
|
stp x2, x3, [sp, #S_X2]
|
|
stp x4, x5, [sp, #S_X4]
|
|
stp x6, x7, [sp, #S_X6]
|
|
stp x8, x9, [sp, #S_X8]
|
|
|
|
/* Optionally save the callee-saved registers, always save the FP */
|
|
.if \allregs == 1
|
|
stp x10, x11, [sp, #S_X10]
|
|
stp x12, x13, [sp, #S_X12]
|
|
stp x14, x15, [sp, #S_X14]
|
|
stp x16, x17, [sp, #S_X16]
|
|
stp x18, x19, [sp, #S_X18]
|
|
stp x20, x21, [sp, #S_X20]
|
|
stp x22, x23, [sp, #S_X22]
|
|
stp x24, x25, [sp, #S_X24]
|
|
stp x26, x27, [sp, #S_X26]
|
|
stp x28, x29, [sp, #S_X28]
|
|
.else
|
|
str x29, [sp, #S_FP]
|
|
.endif
|
|
|
|
/* Save the callsite's SP and LR */
|
|
add x10, sp, #(S_FRAME_SIZE + 16)
|
|
stp x9, x10, [sp, #S_LR]
|
|
|
|
/* Save the PC after the ftrace callsite */
|
|
str x30, [sp, #S_PC]
|
|
|
|
/* Create a frame record for the callsite above pt_regs */
|
|
stp x29, x9, [sp, #S_FRAME_SIZE]
|
|
add x29, sp, #S_FRAME_SIZE
|
|
|
|
/* Create our frame record within pt_regs. */
|
|
stp x29, x30, [sp, #S_STACKFRAME]
|
|
add x29, sp, #S_STACKFRAME
|
|
.endm
|
|
|
|
SYM_CODE_START(ftrace_regs_caller)
|
|
#ifdef BTI_C
|
|
BTI_C
|
|
#endif
|
|
ftrace_regs_entry 1
|
|
b ftrace_common
|
|
SYM_CODE_END(ftrace_regs_caller)
|
|
|
|
SYM_CODE_START(ftrace_caller)
|
|
#ifdef BTI_C
|
|
BTI_C
|
|
#endif
|
|
ftrace_regs_entry 0
|
|
b ftrace_common
|
|
SYM_CODE_END(ftrace_caller)
|
|
|
|
SYM_CODE_START(ftrace_common)
|
|
sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
|
|
mov x1, x9 // parent_ip (callsite's LR)
|
|
ldr_l x2, function_trace_op // op
|
|
mov x3, sp // regs
|
|
|
|
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
|
|
bl ftrace_stub
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
|
|
nop // If enabled, this will be replaced
|
|
// "b ftrace_graph_caller"
|
|
#endif
|
|
|
|
/*
|
|
* At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
|
|
* x19-x29 per the AAPCS, and we created frame records upon entry, so we need
|
|
* to restore x0-x8, x29, and x30.
|
|
*/
|
|
ftrace_common_return:
|
|
/* Restore function arguments */
|
|
ldp x0, x1, [sp]
|
|
ldp x2, x3, [sp, #S_X2]
|
|
ldp x4, x5, [sp, #S_X4]
|
|
ldp x6, x7, [sp, #S_X6]
|
|
ldr x8, [sp, #S_X8]
|
|
|
|
/* Restore the callsite's FP, LR, PC */
|
|
ldr x29, [sp, #S_FP]
|
|
ldr x30, [sp, #S_LR]
|
|
ldr x9, [sp, #S_PC]
|
|
|
|
/* Restore the callsite's SP */
|
|
add sp, sp, #S_FRAME_SIZE + 16
|
|
|
|
ret x9
|
|
SYM_CODE_END(ftrace_common)
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
SYM_CODE_START(ftrace_graph_caller)
|
|
ldr x0, [sp, #S_PC]
|
|
sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
|
|
add x1, sp, #S_LR // parent_ip (callsite's LR)
|
|
ldr x2, [sp, #S_FRAME_SIZE] // parent fp (callsite's FP)
|
|
bl prepare_ftrace_return
|
|
b ftrace_common_return
|
|
SYM_CODE_END(ftrace_graph_caller)
|
|
#endif
|
|
|
|
#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
|
|
|
|
/*
|
|
* Gcc with -pg will put the following code in the beginning of each function:
|
|
* mov x0, x30
|
|
* bl _mcount
|
|
* [function's body ...]
|
|
* "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
|
|
* ftrace is enabled.
|
|
*
|
|
* Please note that x0 as an argument will not be used here because we can
|
|
* get lr(x30) of instrumented function at any time by winding up call stack
|
|
* as long as the kernel is compiled without -fomit-frame-pointer.
|
|
* (or CONFIG_FRAME_POINTER, this is forced on arm64)
|
|
*
|
|
* stack layout after mcount_enter in _mcount():
|
|
*
|
|
* current sp/fp => 0:+-----+
|
|
* in _mcount() | x29 | -> instrumented function's fp
|
|
* +-----+
|
|
* | x30 | -> _mcount()'s lr (= instrumented function's pc)
|
|
* old sp => +16:+-----+
|
|
* when instrumented | |
|
|
* function calls | ... |
|
|
* _mcount() | |
|
|
* | |
|
|
* instrumented => +xx:+-----+
|
|
* function's fp | x29 | -> parent's fp
|
|
* +-----+
|
|
* | x30 | -> instrumented function's lr (= parent's pc)
|
|
* +-----+
|
|
* | ... |
|
|
*/
|
|
|
|
.macro mcount_enter
|
|
stp x29, x30, [sp, #-16]!
|
|
mov x29, sp
|
|
.endm
|
|
|
|
.macro mcount_exit
|
|
ldp x29, x30, [sp], #16
|
|
ret
|
|
.endm
|
|
|
|
.macro mcount_adjust_addr rd, rn
|
|
sub \rd, \rn, #AARCH64_INSN_SIZE
|
|
.endm
|
|
|
|
/* for instrumented function's parent */
|
|
.macro mcount_get_parent_fp reg
|
|
ldr \reg, [x29]
|
|
ldr \reg, [\reg]
|
|
.endm
|
|
|
|
/* for instrumented function */
|
|
.macro mcount_get_pc0 reg
|
|
mcount_adjust_addr \reg, x30
|
|
.endm
|
|
|
|
.macro mcount_get_pc reg
|
|
ldr \reg, [x29, #8]
|
|
mcount_adjust_addr \reg, \reg
|
|
.endm
|
|
|
|
.macro mcount_get_lr reg
|
|
ldr \reg, [x29]
|
|
ldr \reg, [\reg, #8]
|
|
.endm
|
|
|
|
.macro mcount_get_lr_addr reg
|
|
ldr \reg, [x29]
|
|
add \reg, \reg, #8
|
|
.endm
|
|
|
|
#ifndef CONFIG_DYNAMIC_FTRACE
|
|
/*
|
|
* void _mcount(unsigned long return_address)
|
|
* @return_address: return address to instrumented function
|
|
*
|
|
* This function makes calls, if enabled, to:
|
|
* - tracer function to probe instrumented function's entry,
|
|
* - ftrace_graph_caller to set up an exit hook
|
|
*/
|
|
SYM_FUNC_START(_mcount)
|
|
mcount_enter
|
|
|
|
ldr_l x2, ftrace_trace_function
|
|
adr x0, ftrace_stub
|
|
cmp x0, x2 // if (ftrace_trace_function
|
|
b.eq skip_ftrace_call // != ftrace_stub) {
|
|
|
|
mcount_get_pc x0 // function's pc
|
|
mcount_get_lr x1 // function's lr (= parent's pc)
|
|
blr x2 // (*ftrace_trace_function)(pc, lr);
|
|
|
|
skip_ftrace_call: // }
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
ldr_l x2, ftrace_graph_return
|
|
cmp x0, x2 // if ((ftrace_graph_return
|
|
b.ne ftrace_graph_caller // != ftrace_stub)
|
|
|
|
ldr_l x2, ftrace_graph_entry // || (ftrace_graph_entry
|
|
adr_l x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub))
|
|
cmp x0, x2
|
|
b.ne ftrace_graph_caller // ftrace_graph_caller();
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|
|
mcount_exit
|
|
SYM_FUNC_END(_mcount)
|
|
EXPORT_SYMBOL(_mcount)
|
|
NOKPROBE(_mcount)
|
|
|
|
#else /* CONFIG_DYNAMIC_FTRACE */
|
|
/*
|
|
* _mcount() is used to build the kernel with -pg option, but all the branch
|
|
* instructions to _mcount() are replaced to NOP initially at kernel start up,
|
|
* and later on, NOP to branch to ftrace_caller() when enabled or branch to
|
|
* NOP when disabled per-function base.
|
|
*/
|
|
SYM_FUNC_START(_mcount)
|
|
ret
|
|
SYM_FUNC_END(_mcount)
|
|
EXPORT_SYMBOL(_mcount)
|
|
NOKPROBE(_mcount)
|
|
|
|
/*
|
|
* void ftrace_caller(unsigned long return_address)
|
|
* @return_address: return address to instrumented function
|
|
*
|
|
* This function is a counterpart of _mcount() in 'static' ftrace, and
|
|
* makes calls to:
|
|
* - tracer function to probe instrumented function's entry,
|
|
* - ftrace_graph_caller to set up an exit hook
|
|
*/
|
|
SYM_FUNC_START(ftrace_caller)
|
|
mcount_enter
|
|
|
|
mcount_get_pc0 x0 // function's pc
|
|
mcount_get_lr x1 // function's lr
|
|
|
|
SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr);
|
|
nop // This will be replaced with "bl xxx"
|
|
// where xxx can be any kind of tracer.
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
|
|
nop // If enabled, this will be replaced
|
|
// "b ftrace_graph_caller"
|
|
#endif
|
|
|
|
mcount_exit
|
|
SYM_FUNC_END(ftrace_caller)
|
|
#endif /* CONFIG_DYNAMIC_FTRACE */
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
/*
|
|
* void ftrace_graph_caller(void)
|
|
*
|
|
* Called from _mcount() or ftrace_caller() when function_graph tracer is
|
|
* selected.
|
|
* This function w/ prepare_ftrace_return() fakes link register's value on
|
|
* the call stack in order to intercept instrumented function's return path
|
|
* and run return_to_handler() later on its exit.
|
|
*/
|
|
SYM_FUNC_START(ftrace_graph_caller)
|
|
mcount_get_pc x0 // function's pc
|
|
mcount_get_lr_addr x1 // pointer to function's saved lr
|
|
mcount_get_parent_fp x2 // parent's fp
|
|
bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp)
|
|
|
|
mcount_exit
|
|
SYM_FUNC_END(ftrace_graph_caller)
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|
|
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
|
|
|
|
SYM_FUNC_START(ftrace_stub)
|
|
ret
|
|
SYM_FUNC_END(ftrace_stub)
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
/*
|
|
* void return_to_handler(void)
|
|
*
|
|
* Run ftrace_return_to_handler() before going back to parent.
|
|
* @fp is checked against the value passed by ftrace_graph_caller().
|
|
*/
|
|
SYM_CODE_START(return_to_handler)
|
|
/* save return value regs */
|
|
sub sp, sp, #64
|
|
stp x0, x1, [sp]
|
|
stp x2, x3, [sp, #16]
|
|
stp x4, x5, [sp, #32]
|
|
stp x6, x7, [sp, #48]
|
|
|
|
mov x0, x29 // parent's fp
|
|
bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
|
|
mov x30, x0 // restore the original return address
|
|
|
|
/* restore return value regs */
|
|
ldp x0, x1, [sp]
|
|
ldp x2, x3, [sp, #16]
|
|
ldp x4, x5, [sp, #32]
|
|
ldp x6, x7, [sp, #48]
|
|
add sp, sp, #64
|
|
|
|
ret
|
|
SYM_CODE_END(return_to_handler)
|
|
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
|