mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-30 22:26:55 +00:00
Merge branch 'for-next/ftrace' into for-next/core
* for-next/ftrace: arm64: ftrace: Simplify get_ftrace_plt arm64: ftrace: Add direct call support ftrace: selftest: remove broken trace_direct_tramp ftrace: Make DIRECT_CALLS work WITH_ARGS and !WITH_REGS ftrace: Store direct called addresses in their ops ftrace: Rename _ftrace_direct_multi APIs to _ftrace_direct APIs ftrace: Remove the legacy _ftrace_direct API ftrace: Replace uses of _ftrace_direct APIs with _ftrace_direct_multi ftrace: Let unregister_ftrace_direct_multi() call ftrace_free_filter()
This commit is contained in:
commit
ea88dc925c
19 changed files with 244 additions and 516 deletions
|
@ -185,6 +185,10 @@ config ARM64
|
|||
select HAVE_DEBUG_KMEMLEAK
|
||||
select HAVE_DMA_CONTIGUOUS
|
||||
select HAVE_DYNAMIC_FTRACE
|
||||
select HAVE_DYNAMIC_FTRACE_WITH_ARGS \
|
||||
if $(cc-option,-fpatchable-function-entry=2)
|
||||
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS \
|
||||
if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS
|
||||
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \
|
||||
if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG && \
|
||||
!CC_OPTIMIZE_FOR_SIZE)
|
||||
|
|
|
@ -70,10 +70,19 @@ struct ftrace_ops;
|
|||
|
||||
#define arch_ftrace_get_regs(regs) NULL
|
||||
|
||||
/*
|
||||
* Note: sizeof(struct ftrace_regs) must be a multiple of 16 to ensure correct
|
||||
* stack alignment
|
||||
*/
|
||||
struct ftrace_regs {
|
||||
/* x0 - x8 */
|
||||
unsigned long regs[9];
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
unsigned long direct_tramp;
|
||||
#else
|
||||
unsigned long __unused;
|
||||
#endif
|
||||
|
||||
unsigned long fp;
|
||||
unsigned long lr;
|
||||
|
@ -136,6 +145,19 @@ int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
|
|||
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
|
||||
struct ftrace_ops *op, struct ftrace_regs *fregs);
|
||||
#define ftrace_graph_func ftrace_graph_func
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
|
||||
unsigned long addr)
|
||||
{
|
||||
/*
|
||||
* The ftrace trampoline will return to this address instead of the
|
||||
* instrumented function.
|
||||
*/
|
||||
fregs->direct_tramp = addr;
|
||||
}
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
||||
|
||||
#endif
|
||||
|
||||
#define ftrace_return_address(n) return_address(n)
|
||||
|
|
|
@ -93,6 +93,9 @@ int main(void)
|
|||
DEFINE(FREGS_LR, offsetof(struct ftrace_regs, lr));
|
||||
DEFINE(FREGS_SP, offsetof(struct ftrace_regs, sp));
|
||||
DEFINE(FREGS_PC, offsetof(struct ftrace_regs, pc));
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
DEFINE(FREGS_DIRECT_TRAMP, offsetof(struct ftrace_regs, direct_tramp));
|
||||
#endif
|
||||
DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs));
|
||||
BLANK();
|
||||
#endif
|
||||
|
@ -197,6 +200,9 @@ int main(void)
|
|||
#endif
|
||||
#ifdef CONFIG_FUNCTION_TRACER
|
||||
DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func));
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call));
|
||||
#endif
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -36,6 +36,31 @@
|
|||
SYM_CODE_START(ftrace_caller)
|
||||
bti c
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
|
||||
/*
|
||||
* The literal pointer to the ops is at an 8-byte aligned boundary
|
||||
* which is either 12 or 16 bytes before the BL instruction in the call
|
||||
* site. See ftrace_call_adjust() for details.
|
||||
*
|
||||
* Therefore here the LR points at `literal + 16` or `literal + 20`,
|
||||
* and we can find the address of the literal in either case by
|
||||
* aligning to an 8-byte boundary and subtracting 16. We do the
|
||||
* alignment first as this allows us to fold the subtraction into the
|
||||
* LDR.
|
||||
*/
|
||||
bic x11, x30, 0x7
|
||||
ldr x11, [x11, #-(4 * AARCH64_INSN_SIZE)] // op
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
/*
|
||||
* If the op has a direct call, handle it immediately without
|
||||
* saving/restoring registers.
|
||||
*/
|
||||
ldr x17, [x11, #FTRACE_OPS_DIRECT_CALL] // op->direct_call
|
||||
cbnz x17, ftrace_caller_direct
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Save original SP */
|
||||
mov x10, sp
|
||||
|
||||
|
@ -49,6 +74,10 @@ SYM_CODE_START(ftrace_caller)
|
|||
stp x6, x7, [sp, #FREGS_X6]
|
||||
str x8, [sp, #FREGS_X8]
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
str xzr, [sp, #FREGS_DIRECT_TRAMP]
|
||||
#endif
|
||||
|
||||
/* Save the callsite's FP, LR, SP */
|
||||
str x29, [sp, #FREGS_FP]
|
||||
str x9, [sp, #FREGS_LR]
|
||||
|
@ -71,20 +100,7 @@ SYM_CODE_START(ftrace_caller)
|
|||
mov x3, sp // regs
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
|
||||
/*
|
||||
* The literal pointer to the ops is at an 8-byte aligned boundary
|
||||
* which is either 12 or 16 bytes before the BL instruction in the call
|
||||
* site. See ftrace_call_adjust() for details.
|
||||
*
|
||||
* Therefore here the LR points at `literal + 16` or `literal + 20`,
|
||||
* and we can find the address of the literal in either case by
|
||||
* aligning to an 8-byte boundary and subtracting 16. We do the
|
||||
* alignment first as this allows us to fold the subtraction into the
|
||||
* LDR.
|
||||
*/
|
||||
bic x2, x30, 0x7
|
||||
ldr x2, [x2, #-16] // op
|
||||
|
||||
mov x2, x11 // op
|
||||
ldr x4, [x2, #FTRACE_OPS_FUNC] // op->func
|
||||
blr x4 // op->func(ip, parent_ip, op, regs)
|
||||
|
||||
|
@ -107,8 +123,15 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
|
|||
ldp x6, x7, [sp, #FREGS_X6]
|
||||
ldr x8, [sp, #FREGS_X8]
|
||||
|
||||
/* Restore the callsite's FP, LR, PC */
|
||||
/* Restore the callsite's FP */
|
||||
ldr x29, [sp, #FREGS_FP]
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
ldr x17, [sp, #FREGS_DIRECT_TRAMP]
|
||||
cbnz x17, ftrace_caller_direct_late
|
||||
#endif
|
||||
|
||||
/* Restore the callsite's LR and PC */
|
||||
ldr x30, [sp, #FREGS_LR]
|
||||
ldr x9, [sp, #FREGS_PC]
|
||||
|
||||
|
@ -116,8 +139,45 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
|
|||
add sp, sp, #FREGS_SIZE + 32
|
||||
|
||||
ret x9
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
SYM_INNER_LABEL(ftrace_caller_direct_late, SYM_L_LOCAL)
|
||||
/*
|
||||
* Head to a direct trampoline in x17 after having run other tracers.
|
||||
* The ftrace_regs are live, and x0-x8 and FP have been restored. The
|
||||
* LR, PC, and SP have not been restored.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Restore the callsite's LR and PC matching the trampoline calling
|
||||
* convention.
|
||||
*/
|
||||
ldr x9, [sp, #FREGS_LR]
|
||||
ldr x30, [sp, #FREGS_PC]
|
||||
|
||||
/* Restore the callsite's SP */
|
||||
add sp, sp, #FREGS_SIZE + 32
|
||||
|
||||
SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL)
|
||||
/*
|
||||
* Head to a direct trampoline in x17.
|
||||
*
|
||||
* We use `BR X17` as this can safely land on a `BTI C` or `PACIASP` in
|
||||
* the trampoline, and will not unbalance any return stack.
|
||||
*/
|
||||
br x17
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
||||
SYM_CODE_END(ftrace_caller)
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
SYM_CODE_START(ftrace_stub_direct_tramp)
|
||||
bti c
|
||||
mov x10, x30
|
||||
mov x30, x9
|
||||
ret x10
|
||||
SYM_CODE_END(ftrace_stub_direct_tramp)
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
||||
|
||||
#else /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
|
||||
|
||||
/*
|
||||
|
|
|
@ -195,15 +195,22 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
|
|||
return ftrace_modify_code(pc, 0, new, false);
|
||||
}
|
||||
|
||||
static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
|
||||
static struct plt_entry *get_ftrace_plt(struct module *mod)
|
||||
{
|
||||
#ifdef CONFIG_ARM64_MODULE_PLTS
|
||||
struct plt_entry *plt = mod->arch.ftrace_trampolines;
|
||||
|
||||
if (addr == FTRACE_ADDR)
|
||||
return &plt[FTRACE_PLT_IDX];
|
||||
#endif
|
||||
return &plt[FTRACE_PLT_IDX];
|
||||
#else
|
||||
return NULL;
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool reachable_by_bl(unsigned long addr, unsigned long pc)
|
||||
{
|
||||
long offset = (long)addr - (long)pc;
|
||||
|
||||
return offset >= -SZ_128M && offset < SZ_128M;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -220,14 +227,21 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec,
|
|||
unsigned long *addr)
|
||||
{
|
||||
unsigned long pc = rec->ip;
|
||||
long offset = (long)*addr - (long)pc;
|
||||
struct plt_entry *plt;
|
||||
|
||||
/*
|
||||
* If a custom trampoline is unreachable, rely on the ftrace_caller
|
||||
* trampoline which knows how to indirectly reach that trampoline
|
||||
* through ops->direct_call.
|
||||
*/
|
||||
if (*addr != FTRACE_ADDR && !reachable_by_bl(*addr, pc))
|
||||
*addr = FTRACE_ADDR;
|
||||
|
||||
/*
|
||||
* When the target is within range of the 'BL' instruction, use 'addr'
|
||||
* as-is and branch to that directly.
|
||||
*/
|
||||
if (offset >= -SZ_128M && offset < SZ_128M)
|
||||
if (reachable_by_bl(*addr, pc))
|
||||
return true;
|
||||
|
||||
/*
|
||||
|
@ -256,7 +270,7 @@ static bool ftrace_find_callable_addr(struct dyn_ftrace *rec,
|
|||
if (WARN_ON(!mod))
|
||||
return false;
|
||||
|
||||
plt = get_ftrace_plt(mod, *addr);
|
||||
plt = get_ftrace_plt(mod);
|
||||
if (!plt) {
|
||||
pr_err("ftrace: no module PLT for %ps\n", (void *)*addr);
|
||||
return false;
|
||||
|
@ -330,12 +344,24 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
|
|||
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
|
||||
unsigned long addr)
|
||||
{
|
||||
if (WARN_ON_ONCE(old_addr != (unsigned long)ftrace_caller))
|
||||
unsigned long pc = rec->ip;
|
||||
u32 old, new;
|
||||
int ret;
|
||||
|
||||
ret = ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec));
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
|
||||
return -EINVAL;
|
||||
if (WARN_ON_ONCE(addr != (unsigned long)ftrace_caller))
|
||||
if (!ftrace_find_callable_addr(rec, NULL, &addr))
|
||||
return -EINVAL;
|
||||
|
||||
return ftrace_rec_update_ops(rec);
|
||||
old = aarch64_insn_gen_branch_imm(pc, old_addr,
|
||||
AARCH64_INSN_BRANCH_LINK);
|
||||
new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
|
||||
|
||||
return ftrace_modify_code(pc, old, new, true);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
@ -32,6 +32,11 @@ ENTRY(ftrace_stub)
|
|||
BR_EX %r14
|
||||
ENDPROC(ftrace_stub)
|
||||
|
||||
SYM_CODE_START(ftrace_stub_direct_tramp)
|
||||
lgr %r1, %r0
|
||||
BR_EX %r1
|
||||
SYM_CODE_END(ftrace_stub_direct_tramp)
|
||||
|
||||
.macro ftrace_regs_entry, allregs=0
|
||||
stg %r14,(__SF_GPRS+8*8)(%r15) # save traced function caller
|
||||
|
||||
|
|
|
@ -163,6 +163,11 @@ SYM_INNER_LABEL(ftrace_regs_call, SYM_L_GLOBAL)
|
|||
jmp .Lftrace_ret
|
||||
SYM_CODE_END(ftrace_regs_caller)
|
||||
|
||||
SYM_FUNC_START(ftrace_stub_direct_tramp)
|
||||
CALL_DEPTH_ACCOUNT
|
||||
RET
|
||||
SYM_FUNC_END(ftrace_stub_direct_tramp)
|
||||
|
||||
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
||||
SYM_CODE_START(ftrace_graph_caller)
|
||||
pushl %eax
|
||||
|
|
|
@ -309,6 +309,10 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL)
|
|||
SYM_FUNC_END(ftrace_regs_caller)
|
||||
STACK_FRAME_NON_STANDARD_FP(ftrace_regs_caller)
|
||||
|
||||
SYM_FUNC_START(ftrace_stub_direct_tramp)
|
||||
CALL_DEPTH_ACCOUNT
|
||||
RET
|
||||
SYM_FUNC_END(ftrace_stub_direct_tramp)
|
||||
|
||||
#else /* ! CONFIG_DYNAMIC_FTRACE */
|
||||
|
||||
|
|
|
@ -241,6 +241,12 @@ enum {
|
|||
FTRACE_OPS_FL_DIRECT = BIT(17),
|
||||
};
|
||||
|
||||
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
|
||||
#define FTRACE_OPS_FL_SAVE_ARGS FTRACE_OPS_FL_SAVE_REGS
|
||||
#else
|
||||
#define FTRACE_OPS_FL_SAVE_ARGS 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* FTRACE_OPS_CMD_* commands allow the ftrace core logic to request changes
|
||||
* to a ftrace_ops. Note, the requests may fail.
|
||||
|
@ -321,6 +327,9 @@ struct ftrace_ops {
|
|||
unsigned long trampoline_size;
|
||||
struct list_head list;
|
||||
ftrace_ops_func_t ops_func;
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
unsigned long direct_call;
|
||||
#endif
|
||||
#endif
|
||||
};
|
||||
|
||||
|
@ -397,64 +406,36 @@ struct ftrace_func_entry {
|
|||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
extern int ftrace_direct_func_count;
|
||||
int register_ftrace_direct(unsigned long ip, unsigned long addr);
|
||||
int unregister_ftrace_direct(unsigned long ip, unsigned long addr);
|
||||
int modify_ftrace_direct(unsigned long ip, unsigned long old_addr, unsigned long new_addr);
|
||||
struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr);
|
||||
int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
|
||||
struct dyn_ftrace *rec,
|
||||
unsigned long old_addr,
|
||||
unsigned long new_addr);
|
||||
unsigned long ftrace_find_rec_direct(unsigned long ip);
|
||||
int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
|
||||
int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
|
||||
int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr);
|
||||
int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr);
|
||||
int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
|
||||
int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
|
||||
bool free_filters);
|
||||
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr);
|
||||
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr);
|
||||
|
||||
void ftrace_stub_direct_tramp(void);
|
||||
|
||||
#else
|
||||
struct ftrace_ops;
|
||||
# define ftrace_direct_func_count 0
|
||||
static inline int register_ftrace_direct(unsigned long ip, unsigned long addr)
|
||||
{
|
||||
return -ENOTSUPP;
|
||||
}
|
||||
static inline int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
|
||||
{
|
||||
return -ENOTSUPP;
|
||||
}
|
||||
static inline int modify_ftrace_direct(unsigned long ip,
|
||||
unsigned long old_addr, unsigned long new_addr)
|
||||
{
|
||||
return -ENOTSUPP;
|
||||
}
|
||||
static inline struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
static inline int ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
|
||||
struct dyn_ftrace *rec,
|
||||
unsigned long old_addr,
|
||||
unsigned long new_addr)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
static inline unsigned long ftrace_find_rec_direct(unsigned long ip)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
static inline int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
static inline int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
static inline int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
static inline int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
|
||||
bool free_filters)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
static inline int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
static inline int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
static inline int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
|
||||
static inline int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
return -ENODEV;
|
||||
}
|
||||
|
|
|
@ -45,8 +45,8 @@ static int bpf_tramp_ftrace_ops_func(struct ftrace_ops *ops, enum ftrace_ops_cmd
|
|||
lockdep_assert_held_once(&tr->mutex);
|
||||
|
||||
/* Instead of updating the trampoline here, we propagate
|
||||
* -EAGAIN to register_ftrace_direct_multi(). Then we can
|
||||
* retry register_ftrace_direct_multi() after updating the
|
||||
* -EAGAIN to register_ftrace_direct(). Then we can
|
||||
* retry register_ftrace_direct() after updating the
|
||||
* trampoline.
|
||||
*/
|
||||
if ((tr->flags & BPF_TRAMP_F_CALL_ORIG) &&
|
||||
|
@ -198,7 +198,7 @@ static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr)
|
|||
int ret;
|
||||
|
||||
if (tr->func.ftrace_managed)
|
||||
ret = unregister_ftrace_direct_multi(tr->fops, (long)old_addr);
|
||||
ret = unregister_ftrace_direct(tr->fops, (long)old_addr, false);
|
||||
else
|
||||
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, NULL);
|
||||
|
||||
|
@ -215,9 +215,9 @@ static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_ad
|
|||
|
||||
if (tr->func.ftrace_managed) {
|
||||
if (lock_direct_mutex)
|
||||
ret = modify_ftrace_direct_multi(tr->fops, (long)new_addr);
|
||||
ret = modify_ftrace_direct(tr->fops, (long)new_addr);
|
||||
else
|
||||
ret = modify_ftrace_direct_multi_nolock(tr->fops, (long)new_addr);
|
||||
ret = modify_ftrace_direct_nolock(tr->fops, (long)new_addr);
|
||||
} else {
|
||||
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, old_addr, new_addr);
|
||||
}
|
||||
|
@ -243,7 +243,7 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr)
|
|||
|
||||
if (tr->func.ftrace_managed) {
|
||||
ftrace_set_filter_ip(tr->fops, (unsigned long)ip, 0, 1);
|
||||
ret = register_ftrace_direct_multi(tr->fops, (long)new_addr);
|
||||
ret = register_ftrace_direct(tr->fops, (long)new_addr);
|
||||
} else {
|
||||
ret = bpf_arch_text_poke(ip, BPF_MOD_CALL, NULL, new_addr);
|
||||
}
|
||||
|
|
|
@ -257,7 +257,7 @@ config DYNAMIC_FTRACE_WITH_REGS
|
|||
|
||||
config DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
def_bool y
|
||||
depends on DYNAMIC_FTRACE_WITH_REGS
|
||||
depends on DYNAMIC_FTRACE_WITH_REGS || DYNAMIC_FTRACE_WITH_ARGS
|
||||
depends on HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
|
||||
config DYNAMIC_FTRACE_WITH_CALL_OPS
|
||||
|
|
|
@ -2583,28 +2583,13 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
|
|||
static void call_direct_funcs(unsigned long ip, unsigned long pip,
|
||||
struct ftrace_ops *ops, struct ftrace_regs *fregs)
|
||||
{
|
||||
unsigned long addr;
|
||||
unsigned long addr = READ_ONCE(ops->direct_call);
|
||||
|
||||
addr = ftrace_find_rec_direct(ip);
|
||||
if (!addr)
|
||||
return;
|
||||
|
||||
arch_ftrace_set_direct_caller(fregs, addr);
|
||||
}
|
||||
|
||||
static struct ftrace_ops direct_ops = {
|
||||
.func = call_direct_funcs,
|
||||
.flags = FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS
|
||||
| FTRACE_OPS_FL_PERMANENT,
|
||||
/*
|
||||
* By declaring the main trampoline as this trampoline
|
||||
* it will never have one allocated for it. Allocated
|
||||
* trampolines should not call direct functions.
|
||||
* The direct_ops should only be called by the builtin
|
||||
* ftrace_regs_caller trampoline.
|
||||
*/
|
||||
.trampoline = FTRACE_REGS_ADDR,
|
||||
};
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
||||
|
||||
/**
|
||||
|
@ -5301,388 +5286,9 @@ struct ftrace_direct_func {
|
|||
|
||||
static LIST_HEAD(ftrace_direct_funcs);
|
||||
|
||||
/**
|
||||
* ftrace_find_direct_func - test an address if it is a registered direct caller
|
||||
* @addr: The address of a registered direct caller
|
||||
*
|
||||
* This searches to see if a ftrace direct caller has been registered
|
||||
* at a specific address, and if so, it returns a descriptor for it.
|
||||
*
|
||||
* This can be used by architecture code to see if an address is
|
||||
* a direct caller (trampoline) attached to a fentry/mcount location.
|
||||
* This is useful for the function_graph tracer, as it may need to
|
||||
* do adjustments if it traced a location that also has a direct
|
||||
* trampoline attached to it.
|
||||
*/
|
||||
struct ftrace_direct_func *ftrace_find_direct_func(unsigned long addr)
|
||||
{
|
||||
struct ftrace_direct_func *entry;
|
||||
bool found = false;
|
||||
|
||||
/* May be called by fgraph trampoline (protected by rcu tasks) */
|
||||
list_for_each_entry_rcu(entry, &ftrace_direct_funcs, next) {
|
||||
if (entry->addr == addr) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found)
|
||||
return entry;
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static struct ftrace_direct_func *ftrace_alloc_direct_func(unsigned long addr)
|
||||
{
|
||||
struct ftrace_direct_func *direct;
|
||||
|
||||
direct = kmalloc(sizeof(*direct), GFP_KERNEL);
|
||||
if (!direct)
|
||||
return NULL;
|
||||
direct->addr = addr;
|
||||
direct->count = 0;
|
||||
list_add_rcu(&direct->next, &ftrace_direct_funcs);
|
||||
ftrace_direct_func_count++;
|
||||
return direct;
|
||||
}
|
||||
|
||||
static int register_ftrace_function_nolock(struct ftrace_ops *ops);
|
||||
|
||||
/**
|
||||
* register_ftrace_direct - Call a custom trampoline directly
|
||||
* @ip: The address of the nop at the beginning of a function
|
||||
* @addr: The address of the trampoline to call at @ip
|
||||
*
|
||||
* This is used to connect a direct call from the nop location (@ip)
|
||||
* at the start of ftrace traced functions. The location that it calls
|
||||
* (@addr) must be able to handle a direct call, and save the parameters
|
||||
* of the function being traced, and restore them (or inject new ones
|
||||
* if needed), before returning.
|
||||
*
|
||||
* Returns:
|
||||
* 0 on success
|
||||
* -EBUSY - Another direct function is already attached (there can be only one)
|
||||
* -ENODEV - @ip does not point to a ftrace nop location (or not supported)
|
||||
* -ENOMEM - There was an allocation failure.
|
||||
*/
|
||||
int register_ftrace_direct(unsigned long ip, unsigned long addr)
|
||||
{
|
||||
struct ftrace_direct_func *direct;
|
||||
struct ftrace_func_entry *entry;
|
||||
struct ftrace_hash *free_hash = NULL;
|
||||
struct dyn_ftrace *rec;
|
||||
int ret = -ENODEV;
|
||||
|
||||
mutex_lock(&direct_mutex);
|
||||
|
||||
ip = ftrace_location(ip);
|
||||
if (!ip)
|
||||
goto out_unlock;
|
||||
|
||||
/* See if there's a direct function at @ip already */
|
||||
ret = -EBUSY;
|
||||
if (ftrace_find_rec_direct(ip))
|
||||
goto out_unlock;
|
||||
|
||||
ret = -ENODEV;
|
||||
rec = lookup_rec(ip, ip);
|
||||
if (!rec)
|
||||
goto out_unlock;
|
||||
|
||||
/*
|
||||
* Check if the rec says it has a direct call but we didn't
|
||||
* find one earlier?
|
||||
*/
|
||||
if (WARN_ON(rec->flags & FTRACE_FL_DIRECT))
|
||||
goto out_unlock;
|
||||
|
||||
/* Make sure the ip points to the exact record */
|
||||
if (ip != rec->ip) {
|
||||
ip = rec->ip;
|
||||
/* Need to check this ip for a direct. */
|
||||
if (ftrace_find_rec_direct(ip))
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
ret = -ENOMEM;
|
||||
direct = ftrace_find_direct_func(addr);
|
||||
if (!direct) {
|
||||
direct = ftrace_alloc_direct_func(addr);
|
||||
if (!direct)
|
||||
goto out_unlock;
|
||||
}
|
||||
|
||||
entry = ftrace_add_rec_direct(ip, addr, &free_hash);
|
||||
if (!entry)
|
||||
goto out_unlock;
|
||||
|
||||
ret = ftrace_set_filter_ip(&direct_ops, ip, 0, 0);
|
||||
|
||||
if (!ret && !(direct_ops.flags & FTRACE_OPS_FL_ENABLED)) {
|
||||
ret = register_ftrace_function_nolock(&direct_ops);
|
||||
if (ret)
|
||||
ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
remove_hash_entry(direct_functions, entry);
|
||||
kfree(entry);
|
||||
if (!direct->count) {
|
||||
list_del_rcu(&direct->next);
|
||||
synchronize_rcu_tasks();
|
||||
kfree(direct);
|
||||
if (free_hash)
|
||||
free_ftrace_hash(free_hash);
|
||||
free_hash = NULL;
|
||||
ftrace_direct_func_count--;
|
||||
}
|
||||
} else {
|
||||
direct->count++;
|
||||
}
|
||||
out_unlock:
|
||||
mutex_unlock(&direct_mutex);
|
||||
|
||||
if (free_hash) {
|
||||
synchronize_rcu_tasks();
|
||||
free_ftrace_hash(free_hash);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(register_ftrace_direct);
|
||||
|
||||
static struct ftrace_func_entry *find_direct_entry(unsigned long *ip,
|
||||
struct dyn_ftrace **recp)
|
||||
{
|
||||
struct ftrace_func_entry *entry;
|
||||
struct dyn_ftrace *rec;
|
||||
|
||||
rec = lookup_rec(*ip, *ip);
|
||||
if (!rec)
|
||||
return NULL;
|
||||
|
||||
entry = __ftrace_lookup_ip(direct_functions, rec->ip);
|
||||
if (!entry) {
|
||||
WARN_ON(rec->flags & FTRACE_FL_DIRECT);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
WARN_ON(!(rec->flags & FTRACE_FL_DIRECT));
|
||||
|
||||
/* Passed in ip just needs to be on the call site */
|
||||
*ip = rec->ip;
|
||||
|
||||
if (recp)
|
||||
*recp = rec;
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
int unregister_ftrace_direct(unsigned long ip, unsigned long addr)
|
||||
{
|
||||
struct ftrace_direct_func *direct;
|
||||
struct ftrace_func_entry *entry;
|
||||
struct ftrace_hash *hash;
|
||||
int ret = -ENODEV;
|
||||
|
||||
mutex_lock(&direct_mutex);
|
||||
|
||||
ip = ftrace_location(ip);
|
||||
if (!ip)
|
||||
goto out_unlock;
|
||||
|
||||
entry = find_direct_entry(&ip, NULL);
|
||||
if (!entry)
|
||||
goto out_unlock;
|
||||
|
||||
hash = direct_ops.func_hash->filter_hash;
|
||||
if (hash->count == 1)
|
||||
unregister_ftrace_function(&direct_ops);
|
||||
|
||||
ret = ftrace_set_filter_ip(&direct_ops, ip, 1, 0);
|
||||
|
||||
WARN_ON(ret);
|
||||
|
||||
remove_hash_entry(direct_functions, entry);
|
||||
|
||||
direct = ftrace_find_direct_func(addr);
|
||||
if (!WARN_ON(!direct)) {
|
||||
/* This is the good path (see the ! before WARN) */
|
||||
direct->count--;
|
||||
WARN_ON(direct->count < 0);
|
||||
if (!direct->count) {
|
||||
list_del_rcu(&direct->next);
|
||||
synchronize_rcu_tasks();
|
||||
kfree(direct);
|
||||
kfree(entry);
|
||||
ftrace_direct_func_count--;
|
||||
}
|
||||
}
|
||||
out_unlock:
|
||||
mutex_unlock(&direct_mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
|
||||
|
||||
static struct ftrace_ops stub_ops = {
|
||||
.func = ftrace_stub,
|
||||
};
|
||||
|
||||
/**
|
||||
* ftrace_modify_direct_caller - modify ftrace nop directly
|
||||
* @entry: The ftrace hash entry of the direct helper for @rec
|
||||
* @rec: The record representing the function site to patch
|
||||
* @old_addr: The location that the site at @rec->ip currently calls
|
||||
* @new_addr: The location that the site at @rec->ip should call
|
||||
*
|
||||
* An architecture may overwrite this function to optimize the
|
||||
* changing of the direct callback on an ftrace nop location.
|
||||
* This is called with the ftrace_lock mutex held, and no other
|
||||
* ftrace callbacks are on the associated record (@rec). Thus,
|
||||
* it is safe to modify the ftrace record, where it should be
|
||||
* currently calling @old_addr directly, to call @new_addr.
|
||||
*
|
||||
* This is called with direct_mutex locked.
|
||||
*
|
||||
* Safety checks should be made to make sure that the code at
|
||||
* @rec->ip is currently calling @old_addr. And this must
|
||||
* also update entry->direct to @new_addr.
|
||||
*/
|
||||
int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry,
|
||||
struct dyn_ftrace *rec,
|
||||
unsigned long old_addr,
|
||||
unsigned long new_addr)
|
||||
{
|
||||
unsigned long ip = rec->ip;
|
||||
int ret;
|
||||
|
||||
lockdep_assert_held(&direct_mutex);
|
||||
|
||||
/*
|
||||
* The ftrace_lock was used to determine if the record
|
||||
* had more than one registered user to it. If it did,
|
||||
* we needed to prevent that from changing to do the quick
|
||||
* switch. But if it did not (only a direct caller was attached)
|
||||
* then this function is called. But this function can deal
|
||||
* with attached callers to the rec that we care about, and
|
||||
* since this function uses standard ftrace calls that take
|
||||
* the ftrace_lock mutex, we need to release it.
|
||||
*/
|
||||
mutex_unlock(&ftrace_lock);
|
||||
|
||||
/*
|
||||
* By setting a stub function at the same address, we force
|
||||
* the code to call the iterator and the direct_ops helper.
|
||||
* This means that @ip does not call the direct call, and
|
||||
* we can simply modify it.
|
||||
*/
|
||||
ret = ftrace_set_filter_ip(&stub_ops, ip, 0, 0);
|
||||
if (ret)
|
||||
goto out_lock;
|
||||
|
||||
ret = register_ftrace_function_nolock(&stub_ops);
|
||||
if (ret) {
|
||||
ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
|
||||
goto out_lock;
|
||||
}
|
||||
|
||||
entry->direct = new_addr;
|
||||
|
||||
/*
|
||||
* By removing the stub, we put back the direct call, calling
|
||||
* the @new_addr.
|
||||
*/
|
||||
unregister_ftrace_function(&stub_ops);
|
||||
ftrace_set_filter_ip(&stub_ops, ip, 1, 0);
|
||||
|
||||
out_lock:
|
||||
mutex_lock(&ftrace_lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* modify_ftrace_direct - Modify an existing direct call to call something else
|
||||
* @ip: The instruction pointer to modify
|
||||
* @old_addr: The address that the current @ip calls directly
|
||||
* @new_addr: The address that the @ip should call
|
||||
*
|
||||
* This modifies a ftrace direct caller at an instruction pointer without
|
||||
* having to disable it first. The direct call will switch over to the
|
||||
* @new_addr without missing anything.
|
||||
*
|
||||
* Returns: zero on success. Non zero on error, which includes:
|
||||
* -ENODEV : the @ip given has no direct caller attached
|
||||
* -EINVAL : the @old_addr does not match the current direct caller
|
||||
*/
|
||||
int modify_ftrace_direct(unsigned long ip,
|
||||
unsigned long old_addr, unsigned long new_addr)
|
||||
{
|
||||
struct ftrace_direct_func *direct, *new_direct = NULL;
|
||||
struct ftrace_func_entry *entry;
|
||||
struct dyn_ftrace *rec;
|
||||
int ret = -ENODEV;
|
||||
|
||||
mutex_lock(&direct_mutex);
|
||||
|
||||
mutex_lock(&ftrace_lock);
|
||||
|
||||
ip = ftrace_location(ip);
|
||||
if (!ip)
|
||||
goto out_unlock;
|
||||
|
||||
entry = find_direct_entry(&ip, &rec);
|
||||
if (!entry)
|
||||
goto out_unlock;
|
||||
|
||||
ret = -EINVAL;
|
||||
if (entry->direct != old_addr)
|
||||
goto out_unlock;
|
||||
|
||||
direct = ftrace_find_direct_func(old_addr);
|
||||
if (WARN_ON(!direct))
|
||||
goto out_unlock;
|
||||
if (direct->count > 1) {
|
||||
ret = -ENOMEM;
|
||||
new_direct = ftrace_alloc_direct_func(new_addr);
|
||||
if (!new_direct)
|
||||
goto out_unlock;
|
||||
direct->count--;
|
||||
new_direct->count++;
|
||||
} else {
|
||||
direct->addr = new_addr;
|
||||
}
|
||||
|
||||
/*
|
||||
* If there's no other ftrace callback on the rec->ip location,
|
||||
* then it can be changed directly by the architecture.
|
||||
* If there is another caller, then we just need to change the
|
||||
* direct caller helper to point to @new_addr.
|
||||
*/
|
||||
if (ftrace_rec_count(rec) == 1) {
|
||||
ret = ftrace_modify_direct_caller(entry, rec, old_addr, new_addr);
|
||||
} else {
|
||||
entry->direct = new_addr;
|
||||
ret = 0;
|
||||
}
|
||||
|
||||
if (unlikely(ret && new_direct)) {
|
||||
direct->count++;
|
||||
list_del_rcu(&new_direct->next);
|
||||
synchronize_rcu_tasks();
|
||||
kfree(new_direct);
|
||||
ftrace_direct_func_count--;
|
||||
}
|
||||
|
||||
out_unlock:
|
||||
mutex_unlock(&ftrace_lock);
|
||||
mutex_unlock(&direct_mutex);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
|
||||
|
||||
#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS)
|
||||
#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS)
|
||||
|
||||
static int check_direct_multi(struct ftrace_ops *ops)
|
||||
{
|
||||
|
@ -5711,7 +5317,7 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long
|
|||
}
|
||||
|
||||
/**
|
||||
* register_ftrace_direct_multi - Call a custom trampoline directly
|
||||
* register_ftrace_direct - Call a custom trampoline directly
|
||||
* for multiple functions registered in @ops
|
||||
* @ops: The address of the struct ftrace_ops object
|
||||
* @addr: The address of the trampoline to call at @ops functions
|
||||
|
@ -5732,7 +5338,7 @@ static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long
|
|||
* -ENODEV - @ip does not point to a ftrace nop location (or not supported)
|
||||
* -ENOMEM - There was an allocation failure.
|
||||
*/
|
||||
int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
struct ftrace_hash *hash, *free_hash = NULL;
|
||||
struct ftrace_func_entry *entry, *new;
|
||||
|
@ -5774,6 +5380,7 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
ops->func = call_direct_funcs;
|
||||
ops->flags = MULTI_FLAGS;
|
||||
ops->trampoline = FTRACE_REGS_ADDR;
|
||||
ops->direct_call = addr;
|
||||
|
||||
err = register_ftrace_function_nolock(ops);
|
||||
|
||||
|
@ -5790,11 +5397,11 @@ int register_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
}
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
|
||||
EXPORT_SYMBOL_GPL(register_ftrace_direct);
|
||||
|
||||
/**
|
||||
* unregister_ftrace_direct_multi - Remove calls to custom trampoline
|
||||
* previously registered by register_ftrace_direct_multi for @ops object.
|
||||
* unregister_ftrace_direct - Remove calls to custom trampoline
|
||||
* previously registered by register_ftrace_direct for @ops object.
|
||||
* @ops: The address of the struct ftrace_ops object
|
||||
*
|
||||
* This is used to remove a direct calls to @addr from the nop locations
|
||||
|
@ -5805,7 +5412,8 @@ EXPORT_SYMBOL_GPL(register_ftrace_direct_multi);
|
|||
* 0 on success
|
||||
* -EINVAL - The @ops object was not properly registered.
|
||||
*/
|
||||
int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
|
||||
bool free_filters)
|
||||
{
|
||||
struct ftrace_hash *hash = ops->func_hash->filter_hash;
|
||||
int err;
|
||||
|
@ -5823,12 +5431,15 @@ int unregister_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
/* cleanup for possible another register call */
|
||||
ops->func = NULL;
|
||||
ops->trampoline = 0;
|
||||
|
||||
if (free_filters)
|
||||
ftrace_free_filter(ops);
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(unregister_ftrace_direct_multi);
|
||||
EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
|
||||
|
||||
static int
|
||||
__modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
struct ftrace_hash *hash;
|
||||
struct ftrace_func_entry *entry, *iter;
|
||||
|
@ -5844,6 +5455,7 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
/* Enable the tmp_ops to have the same functions as the direct ops */
|
||||
ftrace_ops_init(&tmp_ops);
|
||||
tmp_ops.func_hash = ops->func_hash;
|
||||
tmp_ops.direct_call = addr;
|
||||
|
||||
err = register_ftrace_function_nolock(&tmp_ops);
|
||||
if (err)
|
||||
|
@ -5865,6 +5477,8 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
entry->direct = addr;
|
||||
}
|
||||
}
|
||||
/* Prevent store tearing if a trampoline concurrently accesses the value */
|
||||
WRITE_ONCE(ops->direct_call, addr);
|
||||
|
||||
mutex_unlock(&ftrace_lock);
|
||||
|
||||
|
@ -5875,7 +5489,7 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
}
|
||||
|
||||
/**
|
||||
* modify_ftrace_direct_multi_nolock - Modify an existing direct 'multi' call
|
||||
* modify_ftrace_direct_nolock - Modify an existing direct 'multi' call
|
||||
* to call something else
|
||||
* @ops: The address of the struct ftrace_ops object
|
||||
* @addr: The address of the new trampoline to call at @ops functions
|
||||
|
@ -5892,19 +5506,19 @@ __modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
* Returns: zero on success. Non zero on error, which includes:
|
||||
* -EINVAL - The @ops object was not properly registered.
|
||||
*/
|
||||
int modify_ftrace_direct_multi_nolock(struct ftrace_ops *ops, unsigned long addr)
|
||||
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
if (check_direct_multi(ops))
|
||||
return -EINVAL;
|
||||
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
|
||||
return -EINVAL;
|
||||
|
||||
return __modify_ftrace_direct_multi(ops, addr);
|
||||
return __modify_ftrace_direct(ops, addr);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock);
|
||||
EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock);
|
||||
|
||||
/**
|
||||
* modify_ftrace_direct_multi - Modify an existing direct 'multi' call
|
||||
* modify_ftrace_direct - Modify an existing direct 'multi' call
|
||||
* to call something else
|
||||
* @ops: The address of the struct ftrace_ops object
|
||||
* @addr: The address of the new trampoline to call at @ops functions
|
||||
|
@ -5918,7 +5532,7 @@ EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi_nolock);
|
|||
* Returns: zero on success. Non zero on error, which includes:
|
||||
* -EINVAL - The @ops object was not properly registered.
|
||||
*/
|
||||
int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
||||
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
|
||||
{
|
||||
int err;
|
||||
|
||||
|
@ -5928,11 +5542,11 @@ int modify_ftrace_direct_multi(struct ftrace_ops *ops, unsigned long addr)
|
|||
return -EINVAL;
|
||||
|
||||
mutex_lock(&direct_mutex);
|
||||
err = __modify_ftrace_direct_multi(ops, addr);
|
||||
err = __modify_ftrace_direct(ops, addr);
|
||||
mutex_unlock(&direct_mutex);
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(modify_ftrace_direct_multi);
|
||||
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
|
||||
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
|
||||
|
||||
/**
|
||||
|
|
|
@ -785,14 +785,7 @@ static struct fgraph_ops fgraph_ops __initdata = {
|
|||
};
|
||||
|
||||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
|
||||
#ifndef CALL_DEPTH_ACCOUNT
|
||||
#define CALL_DEPTH_ACCOUNT ""
|
||||
#endif
|
||||
|
||||
noinline __noclone static void trace_direct_tramp(void)
|
||||
{
|
||||
asm(CALL_DEPTH_ACCOUNT);
|
||||
}
|
||||
static struct ftrace_ops direct;
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
@ -870,8 +863,9 @@ trace_selftest_startup_function_graph(struct tracer *trace,
|
|||
* Register direct function together with graph tracer
|
||||
* and make sure we get graph trace.
|
||||
*/
|
||||
ret = register_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
|
||||
(unsigned long) trace_direct_tramp);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long)DYN_FTRACE_TEST_NAME, 0, 0);
|
||||
ret = register_ftrace_direct(&direct,
|
||||
(unsigned long)ftrace_stub_direct_tramp);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
|
@ -891,8 +885,9 @@ trace_selftest_startup_function_graph(struct tracer *trace,
|
|||
|
||||
unregister_ftrace_graph(&fgraph_ops);
|
||||
|
||||
ret = unregister_ftrace_direct((unsigned long) DYN_FTRACE_TEST_NAME,
|
||||
(unsigned long) trace_direct_tramp);
|
||||
ret = unregister_ftrace_direct(&direct,
|
||||
(unsigned long)ftrace_stub_direct_tramp,
|
||||
true);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
|
|
|
@ -38,7 +38,7 @@ config SAMPLE_FTRACE_DIRECT
|
|||
that hooks to wake_up_process and prints the parameters.
|
||||
|
||||
config SAMPLE_FTRACE_DIRECT_MULTI
|
||||
tristate "Build register_ftrace_direct_multi() example"
|
||||
tristate "Build register_ftrace_direct() on multiple ips example"
|
||||
depends on DYNAMIC_FTRACE_WITH_DIRECT_CALLS && m
|
||||
depends on HAVE_SAMPLE_FTRACE_DIRECT_MULTI
|
||||
help
|
||||
|
|
|
@ -96,6 +96,8 @@ asm (
|
|||
|
||||
#endif /* CONFIG_S390 */
|
||||
|
||||
static struct ftrace_ops direct;
|
||||
|
||||
static unsigned long my_tramp = (unsigned long)my_tramp1;
|
||||
static unsigned long tramps[2] = {
|
||||
(unsigned long)my_tramp1,
|
||||
|
@ -114,7 +116,7 @@ static int simple_thread(void *arg)
|
|||
if (ret)
|
||||
continue;
|
||||
t ^= 1;
|
||||
ret = modify_ftrace_direct(my_ip, my_tramp, tramps[t]);
|
||||
ret = modify_ftrace_direct(&direct, tramps[t]);
|
||||
if (!ret)
|
||||
my_tramp = tramps[t];
|
||||
WARN_ON_ONCE(ret);
|
||||
|
@ -129,7 +131,9 @@ static int __init ftrace_direct_init(void)
|
|||
{
|
||||
int ret;
|
||||
|
||||
ret = register_ftrace_direct(my_ip, my_tramp);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long) my_ip, 0, 0);
|
||||
ret = register_ftrace_direct(&direct, my_tramp);
|
||||
|
||||
if (!ret)
|
||||
simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn");
|
||||
return ret;
|
||||
|
@ -138,7 +142,7 @@ static int __init ftrace_direct_init(void)
|
|||
static void __exit ftrace_direct_exit(void)
|
||||
{
|
||||
kthread_stop(simple_tsk);
|
||||
unregister_ftrace_direct(my_ip, my_tramp);
|
||||
unregister_ftrace_direct(&direct, my_tramp, true);
|
||||
}
|
||||
|
||||
module_init(ftrace_direct_init);
|
||||
|
|
|
@ -123,7 +123,7 @@ static int simple_thread(void *arg)
|
|||
if (ret)
|
||||
continue;
|
||||
t ^= 1;
|
||||
ret = modify_ftrace_direct_multi(&direct, tramps[t]);
|
||||
ret = modify_ftrace_direct(&direct, tramps[t]);
|
||||
if (!ret)
|
||||
my_tramp = tramps[t];
|
||||
WARN_ON_ONCE(ret);
|
||||
|
@ -141,7 +141,7 @@ static int __init ftrace_direct_multi_init(void)
|
|||
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long) schedule, 0, 0);
|
||||
|
||||
ret = register_ftrace_direct_multi(&direct, my_tramp);
|
||||
ret = register_ftrace_direct(&direct, my_tramp);
|
||||
|
||||
if (!ret)
|
||||
simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn");
|
||||
|
@ -151,13 +151,12 @@ static int __init ftrace_direct_multi_init(void)
|
|||
static void __exit ftrace_direct_multi_exit(void)
|
||||
{
|
||||
kthread_stop(simple_tsk);
|
||||
unregister_ftrace_direct_multi(&direct, my_tramp);
|
||||
ftrace_free_filter(&direct);
|
||||
unregister_ftrace_direct(&direct, my_tramp, true);
|
||||
}
|
||||
|
||||
module_init(ftrace_direct_multi_init);
|
||||
module_exit(ftrace_direct_multi_exit);
|
||||
|
||||
MODULE_AUTHOR("Jiri Olsa");
|
||||
MODULE_DESCRIPTION("Example use case of using modify_ftrace_direct_multi()");
|
||||
MODULE_DESCRIPTION("Example use case of using modify_ftrace_direct()");
|
||||
MODULE_LICENSE("GPL");
|
||||
|
|
|
@ -73,13 +73,12 @@ static int __init ftrace_direct_multi_init(void)
|
|||
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long) schedule, 0, 0);
|
||||
|
||||
return register_ftrace_direct_multi(&direct, (unsigned long) my_tramp);
|
||||
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
|
||||
}
|
||||
|
||||
static void __exit ftrace_direct_multi_exit(void)
|
||||
{
|
||||
unregister_ftrace_direct_multi(&direct, (unsigned long) my_tramp);
|
||||
ftrace_free_filter(&direct);
|
||||
unregister_ftrace_direct(&direct, (unsigned long) my_tramp, true);
|
||||
}
|
||||
|
||||
module_init(ftrace_direct_multi_init);
|
||||
|
|
|
@ -70,16 +70,18 @@ asm (
|
|||
|
||||
#endif /* CONFIG_S390 */
|
||||
|
||||
static struct ftrace_ops direct;
|
||||
|
||||
static int __init ftrace_direct_init(void)
|
||||
{
|
||||
return register_ftrace_direct((unsigned long)handle_mm_fault,
|
||||
(unsigned long)my_tramp);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long) handle_mm_fault, 0, 0);
|
||||
|
||||
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
|
||||
}
|
||||
|
||||
static void __exit ftrace_direct_exit(void)
|
||||
{
|
||||
unregister_ftrace_direct((unsigned long)handle_mm_fault,
|
||||
(unsigned long)my_tramp);
|
||||
unregister_ftrace_direct(&direct, (unsigned long)my_tramp, true);
|
||||
}
|
||||
|
||||
module_init(ftrace_direct_init);
|
||||
|
|
|
@ -63,16 +63,18 @@ asm (
|
|||
|
||||
#endif /* CONFIG_S390 */
|
||||
|
||||
static struct ftrace_ops direct;
|
||||
|
||||
static int __init ftrace_direct_init(void)
|
||||
{
|
||||
return register_ftrace_direct((unsigned long)wake_up_process,
|
||||
(unsigned long)my_tramp);
|
||||
ftrace_set_filter_ip(&direct, (unsigned long) wake_up_process, 0, 0);
|
||||
|
||||
return register_ftrace_direct(&direct, (unsigned long) my_tramp);
|
||||
}
|
||||
|
||||
static void __exit ftrace_direct_exit(void)
|
||||
{
|
||||
unregister_ftrace_direct((unsigned long)wake_up_process,
|
||||
(unsigned long)my_tramp);
|
||||
unregister_ftrace_direct(&direct, (unsigned long)my_tramp, true);
|
||||
}
|
||||
|
||||
module_init(ftrace_direct_init);
|
||||
|
|
Loading…
Reference in a new issue