mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

x86/entry: Rename idtentry_enter/exit_cond_rcu() to idtentry_enter/exit() 

They were originally called _cond_rcu because they were special versions
with conditional RCU handling.  Now they're the standard entry and exit
path, so the _cond_rcu part is just confusing.  Drop it.

Also change the signature to make them more extensible and more foolproof.

No functional change -- it's pure refactoring.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/247fc67685263e0b673e1d7f808182d28ff80359.1593795633.git.luto@kernel.org
This commit is contained in:
Andy Lutomirski 2020-07-03 10:02:58 -07:00 committed by Thomas Gleixner
parent dcb7fd82c7
commit b037b09b90
5 changed files with 53 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
arch/x86/entry/common.c
View File

@ -559,8 +559,7 @@ SYSCALL_DEFINE0(ni_syscall)
}
/**
* idtentry_enter_cond_rcu - Handle state tracking on idtentry with conditional
* RCU handling
* idtentry_enter - Handle state tracking on ordinary idtentries
* @regs: Pointer to pt_regs of interrupted context
*
* Invokes:
@ -572,6 +571,9 @@ SYSCALL_DEFINE0(ni_syscall)
* - The hardirq tracer to keep the state consistent as low level ASM
* entry disabled interrupts.
*
* As a precondition, this requires that the entry came from user mode,
* idle, or a kernel context in which RCU is watching.
*
* For kernel mode entries RCU handling is done conditional. If RCU is
* watching then the only RCU requirement is to check whether the tick has
* to be restarted. If RCU is not watching then rcu_irq_enter() has to be
@ -585,18 +587,21 @@ SYSCALL_DEFINE0(ni_syscall)
* establish the proper context for NOHZ_FULL. Otherwise scheduling on exit
* would not be possible.
*
* Returns: True if RCU has been adjusted on a kernel entry
* False otherwise
* Returns: An opaque object that must be passed to idtentry_exit()
*
* The return value must be fed into the rcu_exit argument of
* idtentry_exit_cond_rcu().
* The return value must be fed into the state argument of
* idtentry_exit().
*/
bool noinstr idtentry_enter_cond_rcu(struct pt_regs *regs)
idtentry_state_t noinstr idtentry_enter(struct pt_regs *regs)
{
idtentry_state_t ret = {
.exit_rcu = false,
};
if (user_mode(regs)) {
check_user_regs(regs);
enter_from_user_mode();
return false;
return ret;
}
/*
@ -634,7 +639,8 @@ bool noinstr idtentry_enter_cond_rcu(struct pt_regs *regs)
trace_hardirqs_off_finish();
instrumentation_end();
return true;
ret.exit_rcu = true;
return ret;
}
/*
@ -649,7 +655,7 @@ bool noinstr idtentry_enter_cond_rcu(struct pt_regs *regs)
trace_hardirqs_off();
instrumentation_end();
return false;
return ret;
}
static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched)
@ -667,10 +673,9 @@ static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched)
}
/**
* idtentry_exit_cond_rcu - Handle return from exception with conditional RCU
* handling
* idtentry_exit - Handle return from exception that used idtentry_enter()
* @regs: Pointer to pt_regs (exception entry regs)
* @rcu_exit: Invoke rcu_irq_exit() if true
* @state: Return value from matching call to idtentry_enter()
*
* Depending on the return target (kernel/user) this runs the necessary
* preemption and work checks if possible and reguired and returns to
@ -679,10 +684,10 @@ static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched)
* This is the last action before returning to the low level ASM code which
* just needs to return to the appropriate context.
*
* Counterpart to idtentry_enter_cond_rcu(). The return value of the entry
* function must be fed into the @rcu_exit argument.
* Counterpart to idtentry_enter(). The return value of the entry
* function must be fed into the @state argument.
*/
void noinstr idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit)
void noinstr idtentry_exit(struct pt_regs *regs, idtentry_state_t state)
{
lockdep_assert_irqs_disabled();
@ -695,7 +700,7 @@ void noinstr idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit)
* carefully and needs the same ordering of lockdep/tracing
* and RCU as the return to user mode path.
*/
if (rcu_exit) {
if (state.exit_rcu) {
instrumentation_begin();
/* Tell the tracer that IRET will enable interrupts */
trace_hardirqs_on_prepare();
@ -714,7 +719,7 @@ void noinstr idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit)
* IRQ flags state is correct already. Just tell RCU if it
* was not watching on entry.
*/
if (rcu_exit)
if (state.exit_rcu)
rcu_irq_exit();
}
}
@ -800,9 +805,10 @@ static void __xen_pv_evtchn_do_upcall(void)
__visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
{
struct pt_regs *old_regs;
bool inhcall, rcu_exit;
bool inhcall;
idtentry_state_t state;
rcu_exit = idtentry_enter_cond_rcu(regs);
state = idtentry_enter(regs);
old_regs = set_irq_regs(regs);
instrumentation_begin();
@ -812,13 +818,13 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
set_irq_regs(old_regs);
inhcall = get_and_clear_inhcall();
if (inhcall && !WARN_ON_ONCE(rcu_exit)) {
if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
instrumentation_begin();
idtentry_exit_cond_resched(regs, true);
instrumentation_end();
restore_inhcall(inhcall);
} else {
idtentry_exit_cond_rcu(regs, rcu_exit);
idtentry_exit(regs, state);
}
}
#endif /* CONFIG_XEN_PV */

28
arch/x86/include/asm/idtentry.h
View File

@ -13,8 +13,12 @@
void idtentry_enter_user(struct pt_regs *regs);
void idtentry_exit_user(struct pt_regs *regs);
bool idtentry_enter_cond_rcu(struct pt_regs *regs);
void idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit);
typedef struct idtentry_state {
bool exit_rcu;
} idtentry_state_t;
idtentry_state_t idtentry_enter(struct pt_regs *regs);
void idtentry_exit(struct pt_regs *regs, idtentry_state_t state);
/**
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
@ -54,12 +58,12 @@ static __always_inline void __##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
bool rcu_exit = idtentry_enter_cond_rcu(regs); \
idtentry_state_t state = idtentry_enter(regs); \
\
instrumentation_begin(); \
__##func (regs); \
instrumentation_end(); \
idtentry_exit_cond_rcu(regs, rcu_exit); \
idtentry_exit(regs, state); \
} \
\
static __always_inline void __##func(struct pt_regs *regs)
@ -101,12 +105,12 @@ static __always_inline void __##func(struct pt_regs *regs, \
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
{ \
bool rcu_exit = idtentry_enter_cond_rcu(regs); \
idtentry_state_t state = idtentry_enter(regs); \
\
instrumentation_begin(); \
__##func (regs, error_code); \
instrumentation_end(); \
idtentry_exit_cond_rcu(regs, rcu_exit); \
idtentry_exit(regs, state); \
} \
\
static __always_inline void __##func(struct pt_regs *regs, \
@ -199,7 +203,7 @@ static __always_inline void __##func(struct pt_regs *regs, u8 vector); \
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
{ \
bool rcu_exit = idtentry_enter_cond_rcu(regs); \
idtentry_state_t state = idtentry_enter(regs); \
\
instrumentation_begin(); \
irq_enter_rcu(); \
@ -207,7 +211,7 @@ __visible noinstr void func(struct pt_regs *regs, \
__##func (regs, (u8)error_code); \
irq_exit_rcu(); \
instrumentation_end(); \
idtentry_exit_cond_rcu(regs, rcu_exit); \
idtentry_exit(regs, state); \
} \
\
static __always_inline void __##func(struct pt_regs *regs, u8 vector)
@ -241,7 +245,7 @@ static void __##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
bool rcu_exit = idtentry_enter_cond_rcu(regs); \
idtentry_state_t state = idtentry_enter(regs); \
\
instrumentation_begin(); \
irq_enter_rcu(); \
@ -249,7 +253,7 @@ __visible noinstr void func(struct pt_regs *regs) \
run_on_irqstack_cond(__##func, regs, regs); \
irq_exit_rcu(); \
instrumentation_end(); \
idtentry_exit_cond_rcu(regs, rcu_exit); \
idtentry_exit(regs, state); \
} \
\
static noinline void __##func(struct pt_regs *regs)
@ -270,7 +274,7 @@ static __always_inline void __##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
{ \
bool rcu_exit = idtentry_enter_cond_rcu(regs); \
idtentry_state_t state = idtentry_enter(regs); \
\
instrumentation_begin(); \
__irq_enter_raw(); \
@ -278,7 +282,7 @@ __visible noinstr void func(struct pt_regs *regs) \
__##func (regs); \
__irq_exit_raw(); \
instrumentation_end(); \
idtentry_exit_cond_rcu(regs, rcu_exit); \
idtentry_exit(regs, state); \
} \
\
static __always_inline void __##func(struct pt_regs *regs)

6
arch/x86/kernel/kvm.c
View File

@ -233,7 +233,7 @@ EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags);
noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
{
u32 reason = kvm_read_and_reset_apf_flags();
bool rcu_exit;
idtentry_state_t state;
switch (reason) {
case KVM_PV_REASON_PAGE_NOT_PRESENT:
@ -243,7 +243,7 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
return false;
}
rcu_exit = idtentry_enter_cond_rcu(regs);
state = idtentry_enter(regs);
instrumentation_begin();
/*
@ -264,7 +264,7 @@ noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token)
}
instrumentation_end();
idtentry_exit_cond_rcu(regs, rcu_exit);
idtentry_exit(regs, state);
return true;
}

6
arch/x86/kernel/traps.c
View File

@ -245,7 +245,7 @@ static noinstr bool handle_bug(struct pt_regs *regs)
DEFINE_IDTENTRY_RAW(exc_invalid_op)
{
bool rcu_exit;
idtentry_state_t state;
/*
* We use UD2 as a short encoding for 'CALL __WARN', as such
@ -255,11 +255,11 @@ DEFINE_IDTENTRY_RAW(exc_invalid_op)
if (!user_mode(regs) && handle_bug(regs))
return;
rcu_exit = idtentry_enter_cond_rcu(regs);
state = idtentry_enter(regs);
instrumentation_begin();
handle_invalid_op(regs);
instrumentation_end();
idtentry_exit_cond_rcu(regs, rcu_exit);
idtentry_exit(regs, state);
}
DEFINE_IDTENTRY(exc_coproc_segment_overrun)

6
arch/x86/mm/fault.c
View File

@ -1377,7 +1377,7 @@ handle_page_fault(struct pt_regs *regs, unsigned long error_code,
DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
{
unsigned long address = read_cr2();
bool rcu_exit;
idtentry_state_t state;
prefetchw(&current->mm->mmap_lock);
@ -1412,11 +1412,11 @@ DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)
* code reenabled RCU to avoid subsequent wreckage which helps
* debugability.
*/
rcu_exit = idtentry_enter_cond_rcu(regs);
state = idtentry_enter(regs);
instrumentation_begin();
handle_page_fault(regs, error_code, address);
instrumentation_end();
idtentry_exit_cond_rcu(regs, rcu_exit);
idtentry_exit(regs, state);
}