2019-05-19 12:08:55 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2020-09-04 15:30:27 +00:00
|
|
|
#include <linux/objtool.h>
|
2017-07-24 23:36:57 +00:00
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/sort.h>
|
|
|
|
#include <asm/ptrace.h>
|
|
|
|
#include <asm/stacktrace.h>
|
|
|
|
#include <asm/unwind.h>
|
|
|
|
#include <asm/orc_types.h>
|
|
|
|
#include <asm/orc_lookup.h>
|
|
|
|
|
|
|
|
#define orc_warn(fmt, ...) \
|
2020-04-25 10:03:06 +00:00
|
|
|
printk_deferred_once(KERN_WARNING "WARNING: " fmt, ##__VA_ARGS__)
|
|
|
|
|
|
|
|
#define orc_warn_current(args...) \
|
|
|
|
({ \
|
2021-02-05 14:24:03 +00:00
|
|
|
if (state->task == current && !state->error) \
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn(args); \
|
|
|
|
})
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
extern int __start_orc_unwind_ip[];
|
|
|
|
extern int __stop_orc_unwind_ip[];
|
|
|
|
extern struct orc_entry __start_orc_unwind[];
|
|
|
|
extern struct orc_entry __stop_orc_unwind[];
|
|
|
|
|
2020-04-25 10:03:05 +00:00
|
|
|
static bool orc_init __ro_after_init;
|
|
|
|
static unsigned int lookup_num_blocks __ro_after_init;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
static inline unsigned long orc_ip(const int *ip)
|
|
|
|
{
|
|
|
|
return (unsigned long)ip + *ip;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct orc_entry *__orc_find(int *ip_table, struct orc_entry *u_table,
|
|
|
|
unsigned int num_entries, unsigned long ip)
|
|
|
|
{
|
|
|
|
int *first = ip_table;
|
|
|
|
int *last = ip_table + num_entries - 1;
|
|
|
|
int *mid = first, *found = first;
|
|
|
|
|
|
|
|
if (!num_entries)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do a binary range search to find the rightmost duplicate of a given
|
|
|
|
* starting address. Some entries are section terminators which are
|
|
|
|
* "weak" entries for ensuring there are no gaps. They should be
|
|
|
|
* ignored when they conflict with a real entry.
|
|
|
|
*/
|
|
|
|
while (first <= last) {
|
|
|
|
mid = first + ((last - first) / 2);
|
|
|
|
|
|
|
|
if (orc_ip(mid) <= ip) {
|
|
|
|
found = mid;
|
|
|
|
first = mid + 1;
|
|
|
|
} else
|
|
|
|
last = mid - 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return u_table + (found - ip_table);
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_MODULES
|
|
|
|
static struct orc_entry *orc_module_find(unsigned long ip)
|
|
|
|
{
|
|
|
|
struct module *mod;
|
|
|
|
|
|
|
|
mod = __module_address(ip);
|
|
|
|
if (!mod || !mod->arch.orc_unwind || !mod->arch.orc_unwind_ip)
|
|
|
|
return NULL;
|
|
|
|
return __orc_find(mod->arch.orc_unwind_ip, mod->arch.orc_unwind,
|
|
|
|
mod->arch.num_orcs, ip);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static struct orc_entry *orc_module_find(unsigned long ip)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2018-01-23 03:32:51 +00:00
|
|
|
#ifdef CONFIG_DYNAMIC_FTRACE
|
|
|
|
static struct orc_entry *orc_find(unsigned long ip);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Ftrace dynamic trampolines do not have orc entries of their own.
|
|
|
|
* But they are copies of the ftrace entries that are static and
|
|
|
|
* defined in ftrace_*.S, which do have orc entries.
|
|
|
|
*
|
2019-06-27 00:33:55 +00:00
|
|
|
* If the unwinder comes across a ftrace trampoline, then find the
|
2018-01-23 03:32:51 +00:00
|
|
|
* ftrace function that was used to create it, and use that ftrace
|
2019-06-27 00:33:55 +00:00
|
|
|
* function's orc entry, as the placement of the return code in
|
2018-01-23 03:32:51 +00:00
|
|
|
* the stack will be identical.
|
|
|
|
*/
|
|
|
|
static struct orc_entry *orc_ftrace_find(unsigned long ip)
|
|
|
|
{
|
|
|
|
struct ftrace_ops *ops;
|
|
|
|
unsigned long caller;
|
|
|
|
|
|
|
|
ops = ftrace_ops_trampoline(ip);
|
|
|
|
if (!ops)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
|
|
|
|
caller = (unsigned long)ftrace_regs_call;
|
|
|
|
else
|
|
|
|
caller = (unsigned long)ftrace_call;
|
|
|
|
|
|
|
|
/* Prevent unlikely recursion */
|
|
|
|
if (ip == caller)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return orc_find(caller);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static struct orc_entry *orc_ftrace_find(unsigned long ip)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2019-03-01 03:12:01 +00:00
|
|
|
/*
|
|
|
|
* If we crash with IP==0, the last successfully executed instruction
|
|
|
|
* was probably an indirect function call with a NULL function pointer,
|
|
|
|
* and we don't have unwind information for NULL.
|
|
|
|
* This hardcoded ORC entry for IP==0 allows us to unwind from a NULL function
|
|
|
|
* pointer into its parent and then continue normally from there.
|
|
|
|
*/
|
|
|
|
static struct orc_entry null_orc_entry = {
|
|
|
|
.sp_offset = sizeof(long),
|
|
|
|
.sp_reg = ORC_REG_SP,
|
|
|
|
.bp_reg = ORC_REG_UNDEFINED,
|
2020-09-04 15:30:27 +00:00
|
|
|
.type = UNWIND_HINT_TYPE_CALL
|
2019-03-01 03:12:01 +00:00
|
|
|
};
|
|
|
|
|
2019-06-27 00:33:55 +00:00
|
|
|
/* Fake frame pointer entry -- used as a fallback for generated code */
|
|
|
|
static struct orc_entry orc_fp_entry = {
|
2020-09-04 15:30:27 +00:00
|
|
|
.type = UNWIND_HINT_TYPE_CALL,
|
2019-06-27 00:33:55 +00:00
|
|
|
.sp_reg = ORC_REG_BP,
|
|
|
|
.sp_offset = 16,
|
|
|
|
.bp_reg = ORC_REG_PREV_SP,
|
|
|
|
.bp_offset = -16,
|
|
|
|
.end = 0,
|
|
|
|
};
|
|
|
|
|
2017-07-24 23:36:57 +00:00
|
|
|
static struct orc_entry *orc_find(unsigned long ip)
|
|
|
|
{
|
2018-01-23 03:32:51 +00:00
|
|
|
static struct orc_entry *orc;
|
|
|
|
|
2019-03-01 03:12:01 +00:00
|
|
|
if (ip == 0)
|
|
|
|
return &null_orc_entry;
|
|
|
|
|
2017-07-24 23:36:57 +00:00
|
|
|
/* For non-init vmlinux addresses, use the fast lookup table: */
|
|
|
|
if (ip >= LOOKUP_START_IP && ip < LOOKUP_STOP_IP) {
|
|
|
|
unsigned int idx, start, stop;
|
|
|
|
|
|
|
|
idx = (ip - LOOKUP_START_IP) / LOOKUP_BLOCK_SIZE;
|
|
|
|
|
|
|
|
if (unlikely((idx >= lookup_num_blocks-1))) {
|
2017-10-20 16:21:34 +00:00
|
|
|
orc_warn("WARNING: bad lookup idx: idx=%u num=%u ip=%pB\n",
|
|
|
|
idx, lookup_num_blocks, (void *)ip);
|
2017-07-24 23:36:57 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
start = orc_lookup[idx];
|
|
|
|
stop = orc_lookup[idx + 1] + 1;
|
|
|
|
|
|
|
|
if (unlikely((__start_orc_unwind + start >= __stop_orc_unwind) ||
|
|
|
|
(__start_orc_unwind + stop > __stop_orc_unwind))) {
|
2017-10-20 16:21:34 +00:00
|
|
|
orc_warn("WARNING: bad lookup value: idx=%u num=%u start=%u stop=%u ip=%pB\n",
|
|
|
|
idx, lookup_num_blocks, start, stop, (void *)ip);
|
2017-07-24 23:36:57 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return __orc_find(__start_orc_unwind_ip + start,
|
|
|
|
__start_orc_unwind + start, stop - start, ip);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* vmlinux .init slow lookup: */
|
2021-11-09 02:33:54 +00:00
|
|
|
if (is_kernel_inittext(ip))
|
2017-07-24 23:36:57 +00:00
|
|
|
return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
|
|
|
|
__stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
|
|
|
|
|
|
|
|
/* Module lookup: */
|
2018-01-23 03:32:51 +00:00
|
|
|
orc = orc_module_find(ip);
|
|
|
|
if (orc)
|
|
|
|
return orc;
|
|
|
|
|
|
|
|
return orc_ftrace_find(ip);
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
2019-12-16 03:07:29 +00:00
|
|
|
#ifdef CONFIG_MODULES
|
|
|
|
|
2020-04-28 07:16:40 +00:00
|
|
|
static DEFINE_MUTEX(sort_mutex);
|
|
|
|
static int *cur_orc_ip_table = __start_orc_unwind_ip;
|
|
|
|
static struct orc_entry *cur_orc_table = __start_orc_unwind;
|
|
|
|
|
2017-07-24 23:36:57 +00:00
|
|
|
static void orc_sort_swap(void *_a, void *_b, int size)
|
|
|
|
{
|
|
|
|
struct orc_entry *orc_a, *orc_b;
|
|
|
|
struct orc_entry orc_tmp;
|
|
|
|
int *a = _a, *b = _b, tmp;
|
|
|
|
int delta = _b - _a;
|
|
|
|
|
|
|
|
/* Swap the .orc_unwind_ip entries: */
|
|
|
|
tmp = *a;
|
|
|
|
*a = *b + delta;
|
|
|
|
*b = tmp - delta;
|
|
|
|
|
|
|
|
/* Swap the corresponding .orc_unwind entries: */
|
|
|
|
orc_a = cur_orc_table + (a - cur_orc_ip_table);
|
|
|
|
orc_b = cur_orc_table + (b - cur_orc_ip_table);
|
|
|
|
orc_tmp = *orc_a;
|
|
|
|
*orc_a = *orc_b;
|
|
|
|
*orc_b = orc_tmp;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int orc_sort_cmp(const void *_a, const void *_b)
|
|
|
|
{
|
|
|
|
struct orc_entry *orc_a;
|
|
|
|
const int *a = _a, *b = _b;
|
|
|
|
unsigned long a_val = orc_ip(a);
|
|
|
|
unsigned long b_val = orc_ip(b);
|
|
|
|
|
|
|
|
if (a_val > b_val)
|
|
|
|
return 1;
|
|
|
|
if (a_val < b_val)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The "weak" section terminator entries need to always be on the left
|
|
|
|
* to ensure the lookup code skips them in favor of real entries.
|
|
|
|
* These terminator entries exist to handle any gaps created by
|
|
|
|
* whitelisted .o files which didn't get objtool generation.
|
|
|
|
*/
|
|
|
|
orc_a = cur_orc_table + (a - cur_orc_ip_table);
|
2018-05-18 06:47:12 +00:00
|
|
|
return orc_a->sp_reg == ORC_REG_UNDEFINED && !orc_a->end ? -1 : 1;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void unwind_module_init(struct module *mod, void *_orc_ip, size_t orc_ip_size,
|
|
|
|
void *_orc, size_t orc_size)
|
|
|
|
{
|
|
|
|
int *orc_ip = _orc_ip;
|
|
|
|
struct orc_entry *orc = _orc;
|
|
|
|
unsigned int num_entries = orc_ip_size / sizeof(int);
|
|
|
|
|
|
|
|
WARN_ON_ONCE(orc_ip_size % sizeof(int) != 0 ||
|
|
|
|
orc_size % sizeof(*orc) != 0 ||
|
|
|
|
num_entries != orc_size / sizeof(*orc));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The 'cur_orc_*' globals allow the orc_sort_swap() callback to
|
|
|
|
* associate an .orc_unwind_ip table entry with its corresponding
|
|
|
|
* .orc_unwind entry so they can both be swapped.
|
|
|
|
*/
|
|
|
|
mutex_lock(&sort_mutex);
|
|
|
|
cur_orc_ip_table = orc_ip;
|
|
|
|
cur_orc_table = orc;
|
|
|
|
sort(orc_ip, num_entries, sizeof(int), orc_sort_cmp, orc_sort_swap);
|
|
|
|
mutex_unlock(&sort_mutex);
|
|
|
|
|
|
|
|
mod->arch.orc_unwind_ip = orc_ip;
|
|
|
|
mod->arch.orc_unwind = orc;
|
|
|
|
mod->arch.num_orcs = num_entries;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
void __init unwind_init(void)
|
|
|
|
{
|
|
|
|
size_t orc_ip_size = (void *)__stop_orc_unwind_ip - (void *)__start_orc_unwind_ip;
|
|
|
|
size_t orc_size = (void *)__stop_orc_unwind - (void *)__start_orc_unwind;
|
|
|
|
size_t num_entries = orc_ip_size / sizeof(int);
|
|
|
|
struct orc_entry *orc;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!num_entries || orc_ip_size % sizeof(int) != 0 ||
|
|
|
|
orc_size % sizeof(struct orc_entry) != 0 ||
|
|
|
|
num_entries != orc_size / sizeof(struct orc_entry)) {
|
|
|
|
orc_warn("WARNING: Bad or missing .orc_unwind table. Disabling unwinder.\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-12-04 00:46:33 +00:00
|
|
|
/*
|
|
|
|
* Note, the orc_unwind and orc_unwind_ip tables were already
|
|
|
|
* sorted at build time via the 'sorttable' tool.
|
|
|
|
* It's ready for binary search straight away, no need to sort it.
|
|
|
|
*/
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
/* Initialize the fast lookup table: */
|
|
|
|
lookup_num_blocks = orc_lookup_end - orc_lookup;
|
|
|
|
for (i = 0; i < lookup_num_blocks-1; i++) {
|
|
|
|
orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
|
|
|
|
num_entries,
|
|
|
|
LOOKUP_START_IP + (LOOKUP_BLOCK_SIZE * i));
|
|
|
|
if (!orc) {
|
|
|
|
orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
orc_lookup[i] = orc - __start_orc_unwind;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialize the ending block: */
|
|
|
|
orc = __orc_find(__start_orc_unwind_ip, __start_orc_unwind, num_entries,
|
|
|
|
LOOKUP_STOP_IP);
|
|
|
|
if (!orc) {
|
|
|
|
orc_warn("WARNING: Corrupt .orc_unwind table. Disabling unwinder.\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
orc_lookup[lookup_num_blocks-1] = orc - __start_orc_unwind;
|
|
|
|
|
|
|
|
orc_init = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned long unwind_get_return_address(struct unwind_state *state)
|
|
|
|
{
|
|
|
|
if (unwind_done(state))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return __kernel_text_address(state->ip) ? state->ip : 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(unwind_get_return_address);
|
|
|
|
|
|
|
|
unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)
|
|
|
|
{
|
|
|
|
if (unwind_done(state))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (state->regs)
|
|
|
|
return &state->regs->ip;
|
|
|
|
|
|
|
|
if (state->sp)
|
|
|
|
return (unsigned long *)state->sp - 1;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
static bool stack_access_ok(struct unwind_state *state, unsigned long _addr,
|
2017-07-24 23:36:57 +00:00
|
|
|
size_t len)
|
|
|
|
{
|
|
|
|
struct stack_info *info = &state->stack_info;
|
2017-12-04 14:07:09 +00:00
|
|
|
void *addr = (void *)_addr;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
if (!on_stack(info, addr, len) &&
|
|
|
|
(get_stack_info(addr, state->task, info, &state->stack_mask)))
|
|
|
|
return false;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool deref_stack_reg(struct unwind_state *state, unsigned long addr,
|
|
|
|
unsigned long *val)
|
|
|
|
{
|
|
|
|
if (!stack_access_ok(state, addr, sizeof(long)))
|
|
|
|
return false;
|
|
|
|
|
2017-11-08 02:19:34 +00:00
|
|
|
*val = READ_ONCE_NOCHECK(*(unsigned long *)addr);
|
2017-07-24 23:36:57 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool deref_stack_regs(struct unwind_state *state, unsigned long addr,
|
2017-12-04 14:07:09 +00:00
|
|
|
unsigned long *ip, unsigned long *sp)
|
2017-07-24 23:36:57 +00:00
|
|
|
{
|
2017-12-04 14:07:09 +00:00
|
|
|
struct pt_regs *regs = (struct pt_regs *)addr;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
/* x86-32 support will be more complicated due to the ®s->sp hack */
|
|
|
|
BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_32));
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
if (!stack_access_ok(state, addr, sizeof(struct pt_regs)))
|
2017-07-24 23:36:57 +00:00
|
|
|
return false;
|
|
|
|
|
x86/unwind/orc: Disable KASAN checking in the ORC unwinder, part 2
KASAN reserves "redzone" areas between stack frames in order to detect
stack overruns. A read or write to such an area triggers a KASAN
"stack-out-of-bounds" BUG.
Normally, the ORC unwinder stays in-bounds and doesn't access the
redzone. But sometimes it can't find ORC metadata for a given
instruction. This can happen for code which is missing ORC metadata, or
for generated code. In such cases, the unwinder attempts to fall back
to frame pointers, as a best-effort type thing.
This fallback often works, but when it doesn't, the unwinder can get
confused and go off into the weeds into the KASAN redzone, triggering
the aforementioned KASAN BUG.
But in this case, the unwinder's confusion is actually harmless and
working as designed. It already has checks in place to prevent
off-stack accesses, but those checks get short-circuited by the KASAN
BUG. And a BUG is a lot more disruptive than a harmless unwinder
warning.
Disable the KASAN checks by using READ_ONCE_NOCHECK() for all stack
accesses. This finishes the job started by commit 881125bfe65b
("x86/unwind: Disable KASAN checking in the ORC unwinder"), which only
partially fixed the issue.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reported-by: Ivan Babrou <ivan@cloudflare.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Tested-by: Ivan Babrou <ivan@cloudflare.com>
Cc: stable@kernel.org
Link: https://lkml.kernel.org/r/9583327904ebbbeda399eca9c56d6c7085ac20fe.1612534649.git.jpoimboe@redhat.com
2021-02-05 14:24:02 +00:00
|
|
|
*ip = READ_ONCE_NOCHECK(regs->ip);
|
|
|
|
*sp = READ_ONCE_NOCHECK(regs->sp);
|
2017-12-04 14:07:09 +00:00
|
|
|
return true;
|
|
|
|
}
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
static bool deref_stack_iret_regs(struct unwind_state *state, unsigned long addr,
|
|
|
|
unsigned long *ip, unsigned long *sp)
|
|
|
|
{
|
|
|
|
struct pt_regs *regs = (void *)addr - IRET_FRAME_OFFSET;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2017-12-04 14:07:09 +00:00
|
|
|
if (!stack_access_ok(state, addr, IRET_FRAME_SIZE))
|
|
|
|
return false;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
x86/unwind/orc: Disable KASAN checking in the ORC unwinder, part 2
KASAN reserves "redzone" areas between stack frames in order to detect
stack overruns. A read or write to such an area triggers a KASAN
"stack-out-of-bounds" BUG.
Normally, the ORC unwinder stays in-bounds and doesn't access the
redzone. But sometimes it can't find ORC metadata for a given
instruction. This can happen for code which is missing ORC metadata, or
for generated code. In such cases, the unwinder attempts to fall back
to frame pointers, as a best-effort type thing.
This fallback often works, but when it doesn't, the unwinder can get
confused and go off into the weeds into the KASAN redzone, triggering
the aforementioned KASAN BUG.
But in this case, the unwinder's confusion is actually harmless and
working as designed. It already has checks in place to prevent
off-stack accesses, but those checks get short-circuited by the KASAN
BUG. And a BUG is a lot more disruptive than a harmless unwinder
warning.
Disable the KASAN checks by using READ_ONCE_NOCHECK() for all stack
accesses. This finishes the job started by commit 881125bfe65b
("x86/unwind: Disable KASAN checking in the ORC unwinder"), which only
partially fixed the issue.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reported-by: Ivan Babrou <ivan@cloudflare.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Tested-by: Ivan Babrou <ivan@cloudflare.com>
Cc: stable@kernel.org
Link: https://lkml.kernel.org/r/9583327904ebbbeda399eca9c56d6c7085ac20fe.1612534649.git.jpoimboe@redhat.com
2021-02-05 14:24:02 +00:00
|
|
|
*ip = READ_ONCE_NOCHECK(regs->ip);
|
|
|
|
*sp = READ_ONCE_NOCHECK(regs->sp);
|
2017-07-24 23:36:57 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
/*
|
|
|
|
* If state->regs is non-NULL, and points to a full pt_regs, just get the reg
|
|
|
|
* value from state->regs.
|
|
|
|
*
|
|
|
|
* Otherwise, if state->regs just points to IRET regs, and the previous frame
|
|
|
|
* had full regs, it's safe to get the value from the previous regs. This can
|
|
|
|
* happen when early/late IRQ entry code gets interrupted by an NMI.
|
|
|
|
*/
|
|
|
|
static bool get_reg(struct unwind_state *state, unsigned int reg_off,
|
|
|
|
unsigned long *val)
|
|
|
|
{
|
|
|
|
unsigned int reg = reg_off/8;
|
|
|
|
|
|
|
|
if (!state->regs)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
if (state->full_regs) {
|
x86/unwind/orc: Disable KASAN checking in the ORC unwinder, part 2
KASAN reserves "redzone" areas between stack frames in order to detect
stack overruns. A read or write to such an area triggers a KASAN
"stack-out-of-bounds" BUG.
Normally, the ORC unwinder stays in-bounds and doesn't access the
redzone. But sometimes it can't find ORC metadata for a given
instruction. This can happen for code which is missing ORC metadata, or
for generated code. In such cases, the unwinder attempts to fall back
to frame pointers, as a best-effort type thing.
This fallback often works, but when it doesn't, the unwinder can get
confused and go off into the weeds into the KASAN redzone, triggering
the aforementioned KASAN BUG.
But in this case, the unwinder's confusion is actually harmless and
working as designed. It already has checks in place to prevent
off-stack accesses, but those checks get short-circuited by the KASAN
BUG. And a BUG is a lot more disruptive than a harmless unwinder
warning.
Disable the KASAN checks by using READ_ONCE_NOCHECK() for all stack
accesses. This finishes the job started by commit 881125bfe65b
("x86/unwind: Disable KASAN checking in the ORC unwinder"), which only
partially fixed the issue.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reported-by: Ivan Babrou <ivan@cloudflare.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Tested-by: Ivan Babrou <ivan@cloudflare.com>
Cc: stable@kernel.org
Link: https://lkml.kernel.org/r/9583327904ebbbeda399eca9c56d6c7085ac20fe.1612534649.git.jpoimboe@redhat.com
2021-02-05 14:24:02 +00:00
|
|
|
*val = READ_ONCE_NOCHECK(((unsigned long *)state->regs)[reg]);
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (state->prev_regs) {
|
x86/unwind/orc: Disable KASAN checking in the ORC unwinder, part 2
KASAN reserves "redzone" areas between stack frames in order to detect
stack overruns. A read or write to such an area triggers a KASAN
"stack-out-of-bounds" BUG.
Normally, the ORC unwinder stays in-bounds and doesn't access the
redzone. But sometimes it can't find ORC metadata for a given
instruction. This can happen for code which is missing ORC metadata, or
for generated code. In such cases, the unwinder attempts to fall back
to frame pointers, as a best-effort type thing.
This fallback often works, but when it doesn't, the unwinder can get
confused and go off into the weeds into the KASAN redzone, triggering
the aforementioned KASAN BUG.
But in this case, the unwinder's confusion is actually harmless and
working as designed. It already has checks in place to prevent
off-stack accesses, but those checks get short-circuited by the KASAN
BUG. And a BUG is a lot more disruptive than a harmless unwinder
warning.
Disable the KASAN checks by using READ_ONCE_NOCHECK() for all stack
accesses. This finishes the job started by commit 881125bfe65b
("x86/unwind: Disable KASAN checking in the ORC unwinder"), which only
partially fixed the issue.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reported-by: Ivan Babrou <ivan@cloudflare.com>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Tested-by: Ivan Babrou <ivan@cloudflare.com>
Cc: stable@kernel.org
Link: https://lkml.kernel.org/r/9583327904ebbbeda399eca9c56d6c7085ac20fe.1612534649.git.jpoimboe@redhat.com
2021-02-05 14:24:02 +00:00
|
|
|
*val = READ_ONCE_NOCHECK(((unsigned long *)state->prev_regs)[reg]);
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-07-24 23:36:57 +00:00
|
|
|
bool unwind_next_frame(struct unwind_state *state)
|
|
|
|
{
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
unsigned long ip_p, sp, tmp, orig_ip = state->ip, prev_sp = state->sp;
|
2017-07-24 23:36:57 +00:00
|
|
|
enum stack_type prev_type = state->stack_info.type;
|
|
|
|
struct orc_entry *orc;
|
|
|
|
bool indirect = false;
|
|
|
|
|
|
|
|
if (unwind_done(state))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Don't let modules unload while we're reading their ORC data. */
|
|
|
|
preempt_disable();
|
|
|
|
|
2018-05-18 06:47:12 +00:00
|
|
|
/* End-of-stack check for user tasks: */
|
2017-07-24 23:36:57 +00:00
|
|
|
if (state->regs && user_mode(state->regs))
|
2018-05-18 06:47:12 +00:00
|
|
|
goto the_end;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Find the orc_entry associated with the text address.
|
|
|
|
*
|
2020-07-17 14:04:25 +00:00
|
|
|
* For a call frame (as opposed to a signal frame), state->ip points to
|
|
|
|
* the instruction after the call. That instruction's stack layout
|
|
|
|
* could be different from the call instruction's layout, for example
|
|
|
|
* if the call was to a noreturn function. So get the ORC data for the
|
|
|
|
* call instruction itself.
|
2017-07-24 23:36:57 +00:00
|
|
|
*/
|
|
|
|
orc = orc_find(state->signal ? state->ip : state->ip - 1);
|
2019-06-27 00:33:55 +00:00
|
|
|
if (!orc) {
|
|
|
|
/*
|
|
|
|
* As a fallback, try to assume this code uses a frame pointer.
|
|
|
|
* This is useful for generated code, like BPF, which ORC
|
|
|
|
* doesn't know about. This is just a guess, so the rest of
|
|
|
|
* the unwind is no longer considered reliable.
|
|
|
|
*/
|
|
|
|
orc = &orc_fp_entry;
|
|
|
|
state->error = true;
|
|
|
|
}
|
2018-05-18 06:47:12 +00:00
|
|
|
|
|
|
|
/* End-of-stack check for kernel threads: */
|
|
|
|
if (orc->sp_reg == ORC_REG_UNDEFINED) {
|
|
|
|
if (!orc->end)
|
|
|
|
goto err;
|
|
|
|
|
|
|
|
goto the_end;
|
|
|
|
}
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
/* Find the previous frame's stack: */
|
|
|
|
switch (orc->sp_reg) {
|
|
|
|
case ORC_REG_SP:
|
|
|
|
sp = state->sp + orc->sp_offset;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_BP:
|
|
|
|
sp = state->bp + orc->sp_offset;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_SP_INDIRECT:
|
2021-02-03 11:02:21 +00:00
|
|
|
sp = state->sp;
|
2017-07-24 23:36:57 +00:00
|
|
|
indirect = true;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_BP_INDIRECT:
|
|
|
|
sp = state->bp + orc->sp_offset;
|
|
|
|
indirect = true;
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_R10:
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (!get_reg(state, offsetof(struct pt_regs, r10), &sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("missing R10 value at %pB\n",
|
|
|
|
(void *)state->ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_R13:
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (!get_reg(state, offsetof(struct pt_regs, r13), &sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("missing R13 value at %pB\n",
|
|
|
|
(void *)state->ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_DI:
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (!get_reg(state, offsetof(struct pt_regs, di), &sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("missing RDI value at %pB\n",
|
|
|
|
(void *)state->ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_DX:
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (!get_reg(state, offsetof(struct pt_regs, dx), &sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("missing DX value at %pB\n",
|
|
|
|
(void *)state->ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn("unknown SP base reg %d at %pB\n",
|
2017-07-24 23:36:57 +00:00
|
|
|
orc->sp_reg, (void *)state->ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (indirect) {
|
|
|
|
if (!deref_stack_reg(state, sp, &sp))
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2021-02-03 11:02:21 +00:00
|
|
|
|
|
|
|
if (orc->sp_reg == ORC_REG_SP_INDIRECT)
|
|
|
|
sp += orc->sp_offset;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Find IP, SP and possibly regs: */
|
|
|
|
switch (orc->type) {
|
2020-09-04 15:30:27 +00:00
|
|
|
case UNWIND_HINT_TYPE_CALL:
|
2017-07-24 23:36:57 +00:00
|
|
|
ip_p = sp - sizeof(long);
|
|
|
|
|
|
|
|
if (!deref_stack_reg(state, ip_p, &state->ip))
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
2021-09-14 14:42:31 +00:00
|
|
|
state->ip = unwind_recover_ret_addr(state, state->ip,
|
|
|
|
(unsigned long *)ip_p);
|
2017-07-24 23:36:57 +00:00
|
|
|
state->sp = sp;
|
|
|
|
state->regs = NULL;
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
state->prev_regs = NULL;
|
2017-07-24 23:36:57 +00:00
|
|
|
state->signal = false;
|
|
|
|
break;
|
|
|
|
|
2020-09-04 15:30:27 +00:00
|
|
|
case UNWIND_HINT_TYPE_REGS:
|
2017-12-04 14:07:09 +00:00
|
|
|
if (!deref_stack_regs(state, sp, &state->ip, &state->sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("can't access registers at %pB\n",
|
|
|
|
(void *)orig_ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
2021-09-14 14:42:31 +00:00
|
|
|
/*
|
|
|
|
* There is a small chance to interrupt at the entry of
|
|
|
|
* __kretprobe_trampoline() where the ORC info doesn't exist.
|
|
|
|
* That point is right after the RET to __kretprobe_trampoline()
|
|
|
|
* which was modified return address.
|
|
|
|
* At that point, the @addr_p of the unwind_recover_kretprobe()
|
|
|
|
* (this has to point the address of the stack entry storing
|
|
|
|
* the modified return address) must be "SP - (a stack entry)"
|
|
|
|
* because SP is incremented by the RET.
|
|
|
|
*/
|
|
|
|
state->ip = unwind_recover_kretprobe(state, state->ip,
|
|
|
|
(unsigned long *)(state->sp - sizeof(long)));
|
2017-07-24 23:36:57 +00:00
|
|
|
state->regs = (struct pt_regs *)sp;
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
state->prev_regs = NULL;
|
2017-07-24 23:36:57 +00:00
|
|
|
state->full_regs = true;
|
|
|
|
state->signal = true;
|
|
|
|
break;
|
|
|
|
|
2020-09-04 15:30:27 +00:00
|
|
|
case UNWIND_HINT_TYPE_REGS_PARTIAL:
|
2017-12-04 14:07:09 +00:00
|
|
|
if (!deref_stack_iret_regs(state, sp, &state->ip, &state->sp)) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("can't access iret registers at %pB\n",
|
|
|
|
(void *)orig_ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
2021-09-14 14:42:31 +00:00
|
|
|
/* See UNWIND_HINT_TYPE_REGS case comment. */
|
|
|
|
state->ip = unwind_recover_kretprobe(state, state->ip,
|
|
|
|
(unsigned long *)(state->sp - sizeof(long)));
|
2017-07-24 23:36:57 +00:00
|
|
|
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (state->full_regs)
|
|
|
|
state->prev_regs = state->regs;
|
2017-12-04 14:07:09 +00:00
|
|
|
state->regs = (void *)sp - IRET_FRAME_OFFSET;
|
|
|
|
state->full_regs = false;
|
2017-07-24 23:36:57 +00:00
|
|
|
state->signal = true;
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn("unknown .orc_unwind entry type %d at %pB\n",
|
2017-10-20 16:21:34 +00:00
|
|
|
orc->type, (void *)orig_ip);
|
2020-04-25 10:06:13 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Find BP: */
|
|
|
|
switch (orc->bp_reg) {
|
|
|
|
case ORC_REG_UNDEFINED:
|
x86/unwind/orc: Fix premature unwind stoppage due to IRET frames
The following execution path is possible:
fsnotify()
[ realign the stack and store previous SP in R10 ]
<IRQ>
[ only IRET regs saved ]
common_interrupt()
interrupt_entry()
<NMI>
[ full pt_regs saved ]
...
[ unwind stack ]
When the unwinder goes through the NMI and the IRQ on the stack, and
then sees fsnotify(), it doesn't have access to the value of R10,
because it only has the five IRET registers. So the unwind stops
prematurely.
However, because the interrupt_entry() code is careful not to clobber
R10 before saving the full regs, the unwinder should be able to read R10
from the previously saved full pt_regs associated with the NMI.
Handle this case properly. When encountering an IRET regs frame
immediately after a full pt_regs frame, use the pt_regs as a backup
which can be used to get the C register values.
Also, note that a call frame resets the 'prev_regs' value, because a
function is free to clobber the registers. For this fix to work, the
IRET and full regs frames must be adjacent, with no FUNC frames in
between. So replace the FUNC hint in interrupt_entry() with an
IRET_REGS hint.
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Signed-off-by: Josh Poimboeuf <jpoimboe@redhat.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Dave Jones <dsj@fb.com>
Cc: Jann Horn <jannh@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Link: https://lore.kernel.org/r/97a408167cc09f1cfa0de31a7b70dd88868d743f.1587808742.git.jpoimboe@redhat.com
2020-04-25 10:06:14 +00:00
|
|
|
if (get_reg(state, offsetof(struct pt_regs, bp), &tmp))
|
|
|
|
state->bp = tmp;
|
2017-07-24 23:36:57 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_PREV_SP:
|
|
|
|
if (!deref_stack_reg(state, sp + orc->bp_offset, &state->bp))
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case ORC_REG_BP:
|
|
|
|
if (!deref_stack_reg(state, state->bp + orc->bp_offset, &state->bp))
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
2017-10-20 16:21:34 +00:00
|
|
|
orc_warn("unknown BP base reg %d for ip %pB\n",
|
2017-07-24 23:36:57 +00:00
|
|
|
orc->bp_reg, (void *)orig_ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Prevent a recursive loop due to bad ORC data: */
|
|
|
|
if (state->stack_info.type == prev_type &&
|
|
|
|
on_stack(&state->stack_info, (void *)state->sp, sizeof(long)) &&
|
|
|
|
state->sp <= prev_sp) {
|
2020-04-25 10:03:06 +00:00
|
|
|
orc_warn_current("stack going in the wrong direction? at %pB\n",
|
|
|
|
(void *)orig_ip);
|
2018-05-18 06:47:12 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
preempt_enable();
|
|
|
|
return true;
|
|
|
|
|
2018-05-18 06:47:12 +00:00
|
|
|
err:
|
|
|
|
state->error = true;
|
|
|
|
|
|
|
|
the_end:
|
2017-07-24 23:36:57 +00:00
|
|
|
preempt_enable();
|
|
|
|
state->stack_info.type = STACK_TYPE_UNKNOWN;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(unwind_next_frame);
|
|
|
|
|
|
|
|
void __unwind_start(struct unwind_state *state, struct task_struct *task,
|
|
|
|
struct pt_regs *regs, unsigned long *first_frame)
|
|
|
|
{
|
|
|
|
memset(state, 0, sizeof(*state));
|
|
|
|
state->task = task;
|
|
|
|
|
2020-05-14 20:31:10 +00:00
|
|
|
if (!orc_init)
|
|
|
|
goto err;
|
|
|
|
|
2017-07-24 23:36:57 +00:00
|
|
|
/*
|
|
|
|
* Refuse to unwind the stack of a task while it's executing on another
|
|
|
|
* CPU. This check is racy, but that's ok: the unwinder has other
|
|
|
|
* checks to prevent it from going off the rails.
|
|
|
|
*/
|
|
|
|
if (task_on_another_cpu(task))
|
2020-05-14 20:31:10 +00:00
|
|
|
goto err;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
if (regs) {
|
|
|
|
if (user_mode(regs))
|
2020-05-14 20:31:10 +00:00
|
|
|
goto the_end;
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
state->ip = regs->ip;
|
2019-05-07 21:25:54 +00:00
|
|
|
state->sp = regs->sp;
|
2017-07-24 23:36:57 +00:00
|
|
|
state->bp = regs->bp;
|
|
|
|
state->regs = regs;
|
|
|
|
state->full_regs = true;
|
|
|
|
state->signal = true;
|
|
|
|
|
|
|
|
} else if (task == current) {
|
|
|
|
asm volatile("lea (%%rip), %0\n\t"
|
|
|
|
"mov %%rsp, %1\n\t"
|
|
|
|
"mov %%rbp, %2\n\t"
|
|
|
|
: "=r" (state->ip), "=r" (state->sp),
|
|
|
|
"=r" (state->bp));
|
|
|
|
|
|
|
|
} else {
|
|
|
|
struct inactive_task_frame *frame = (void *)task->thread.sp;
|
|
|
|
|
x86/unwind/orc: Fix inactive tasks with stack pointer in %sp on GCC 10 compiled kernels
GCC 10 optimizes the scheduler code differently than its predecessors.
When CONFIG_DEBUG_SECTION_MISMATCH=y, the Makefile forces GCC not
to inline some functions (-fno-inline-functions-called-once). Before GCC
10, "no-inlined" __schedule() starts with the usual prologue:
push %bp
mov %sp, %bp
So the ORC unwinder simply picks stack pointer from %bp and
unwinds from __schedule() just perfectly:
$ cat /proc/1/stack
[<0>] ep_poll+0x3e9/0x450
[<0>] do_epoll_wait+0xaa/0xc0
[<0>] __x64_sys_epoll_wait+0x1a/0x20
[<0>] do_syscall_64+0x33/0x40
[<0>] entry_SYSCALL_64_after_hwframe+0x44/0xa9
But now, with GCC 10, there is no %bp prologue in __schedule():
$ cat /proc/1/stack
<nothing>
The ORC entry of the point in __schedule() is:
sp:sp+88 bp:last_sp-48 type:call end:0
In this case, nobody subtracts sizeof "struct inactive_task_frame" in
__unwind_start(). The struct is put on the stack by __switch_to_asm() and
only then __switch_to_asm() stores %sp to task->thread.sp. But we start
unwinding from a point in __schedule() (stored in frame->ret_addr by
'call') and not in __switch_to_asm().
So for these example values in __unwind_start():
sp=ffff94b50001fdc8 bp=ffff8e1f41d29340 ip=__schedule+0x1f0
The stack is:
ffff94b50001fdc8: ffff8e1f41578000 # struct inactive_task_frame
ffff94b50001fdd0: 0000000000000000
ffff94b50001fdd8: ffff8e1f41d29340
ffff94b50001fde0: ffff8e1f41611d40 # ...
ffff94b50001fde8: ffffffff93c41920 # bx
ffff94b50001fdf0: ffff8e1f41d29340 # bp
ffff94b50001fdf8: ffffffff9376cad0 # ret_addr (and end of the struct)
0xffffffff9376cad0 is __schedule+0x1f0 (after the call to
__switch_to_asm). Now follow those 88 bytes from the ORC entry (sp+88).
The entry is correct, __schedule() really pushes 48 bytes (8*7) + 32 bytes
via subq to store some local values (like 4U below). So to unwind, look
at the offset 88-sizeof(long) = 0x50 from here:
ffff94b50001fe00: ffff8e1f41578618
ffff94b50001fe08: 00000cc000000255
ffff94b50001fe10: 0000000500000004
ffff94b50001fe18: 7793fab6956b2d00 # NOTE (see below)
ffff94b50001fe20: ffff8e1f41578000
ffff94b50001fe28: ffff8e1f41578000
ffff94b50001fe30: ffff8e1f41578000
ffff94b50001fe38: ffff8e1f41578000
ffff94b50001fe40: ffff94b50001fed8
ffff94b50001fe48: ffff8e1f41577ff0
ffff94b50001fe50: ffffffff9376cf12
Here ^^^^^^^^^^^^^^^^ is the correct ret addr from
__schedule(). It translates to schedule+0x42 (insn after a call to
__schedule()).
BUT, unwind_next_frame() tries to take the address starting from
0xffff94b50001fdc8. That is exactly from thread.sp+88-sizeof(long) =
0xffff94b50001fdc8+88-8 = 0xffff94b50001fe18, which is garbage marked as
NOTE above. So this quits the unwinding as 7793fab6956b2d00 is obviously
not a kernel address.
There was a fix to skip 'struct inactive_task_frame' in
unwind_get_return_address_ptr in the following commit:
187b96db5ca7 ("x86/unwind/orc: Fix unwind_get_return_address_ptr() for inactive tasks")
But we need to skip the struct already in the unwinder proper. So
subtract the size (increase the stack pointer) of the structure in
__unwind_start() directly. This allows for removal of the code added by
commit 187b96db5ca7 completely, as the address is now at
'(unsigned long *)state->sp - 1', the same as in the generic case.
[ mingo: Cleaned up the changelog a bit, for better readability. ]
Fixes: ee9f8fce9964 ("x86/unwind: Add the ORC unwinder")
Bug: https://bugzilla.suse.com/show_bug.cgi?id=1176907
Signed-off-by: Jiri Slaby <jslaby@suse.cz>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20201014053051.24199-1-jslaby@suse.cz
2020-10-14 05:30:51 +00:00
|
|
|
state->sp = task->thread.sp + sizeof(*frame);
|
2017-07-24 23:36:57 +00:00
|
|
|
state->bp = READ_ONCE_NOCHECK(frame->bp);
|
|
|
|
state->ip = READ_ONCE_NOCHECK(frame->ret_addr);
|
2020-07-17 14:04:25 +00:00
|
|
|
state->signal = (void *)state->ip == ret_from_fork;
|
2017-07-24 23:36:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (get_stack_info((unsigned long *)state->sp, state->task,
|
2017-12-04 14:07:08 +00:00
|
|
|
&state->stack_info, &state->stack_mask)) {
|
|
|
|
/*
|
|
|
|
* We weren't on a valid stack. It's possible that
|
|
|
|
* we overflowed a valid stack into a guard page.
|
|
|
|
* See if the next page up is valid so that we can
|
|
|
|
* generate some kind of backtrace if this happens.
|
|
|
|
*/
|
|
|
|
void *next_page = (void *)PAGE_ALIGN((unsigned long)state->sp);
|
2020-05-14 20:31:10 +00:00
|
|
|
state->error = true;
|
2017-12-04 14:07:08 +00:00
|
|
|
if (get_stack_info(next_page, state->task, &state->stack_info,
|
|
|
|
&state->stack_mask))
|
|
|
|
return;
|
|
|
|
}
|
2017-07-24 23:36:57 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The caller can provide the address of the first frame directly
|
|
|
|
* (first_frame) or indirectly (regs->sp) to indicate which stack frame
|
|
|
|
* to start unwinding at. Skip ahead until we reach it.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* When starting from regs, skip the regs frame: */
|
|
|
|
if (regs) {
|
|
|
|
unwind_next_frame(state);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Otherwise, skip ahead to the user-specified starting frame: */
|
|
|
|
while (!unwind_done(state) &&
|
|
|
|
(!on_stack(&state->stack_info, first_frame, sizeof(long)) ||
|
2020-04-25 10:03:07 +00:00
|
|
|
state->sp < (unsigned long)first_frame))
|
2017-07-24 23:36:57 +00:00
|
|
|
unwind_next_frame(state);
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
2020-05-14 20:31:10 +00:00
|
|
|
err:
|
|
|
|
state->error = true;
|
|
|
|
the_end:
|
2017-07-24 23:36:57 +00:00
|
|
|
state->stack_info.type = STACK_TYPE_UNKNOWN;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__unwind_start);
|