mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-29 05:44:11 +00:00
x86/entry: Optimize common_interrupt_return()
The code in common_interrupt_return() does a bunch of unconditional work that is really only needed on PTI kernels. Specifically it unconditionally copies the IRET frame back onto the entry stack, swizzles onto the entry stack and does IRET from there. However, without PTI we can simply IRET from whatever stack we're on. ivb-ep, mitigations=off, gettid-1m: PRE: 140,118,538 cycles:k ( +- 0.01% ) 236,692,878 instructions:k # 1.69 insn per cycle ( +- 0.00% ) POST: 140,026,608 cycles:k ( +- 0.01% ) 236,696,176 instructions:k # 1.69 insn per cycle ( +- 0.00% ) (this is with --repeat 100 and the run-to-run variance is bigger than the difference shown) Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lore.kernel.org/r/20231120143626.638107480@infradead.org
This commit is contained in:
parent
98b1cc82c4
commit
c516213726
2 changed files with 24 additions and 5 deletions
|
@ -175,8 +175,7 @@ For 32-bit we have the following conventions - kernel is built with
|
|||
#define THIS_CPU_user_pcid_flush_mask \
|
||||
PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask
|
||||
|
||||
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
|
||||
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
|
||||
.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req
|
||||
mov %cr3, \scratch_reg
|
||||
|
||||
ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
|
||||
|
@ -206,13 +205,20 @@ For 32-bit we have the following conventions - kernel is built with
|
|||
/* Flip the PGD to the user version */
|
||||
orq $(PTI_USER_PGTABLE_MASK), \scratch_reg
|
||||
mov \scratch_reg, %cr3
|
||||
.endm
|
||||
|
||||
.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
|
||||
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
|
||||
SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2
|
||||
.Lend_\@:
|
||||
.endm
|
||||
|
||||
.macro SWITCH_TO_USER_CR3_STACK scratch_reg:req
|
||||
ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
|
||||
pushq %rax
|
||||
SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax
|
||||
SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax
|
||||
popq %rax
|
||||
.Lend_\@:
|
||||
.endm
|
||||
|
||||
.macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
|
||||
|
|
|
@ -569,7 +569,18 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
|
|||
#ifdef CONFIG_XEN_PV
|
||||
ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
|
||||
#endif
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
|
||||
#endif
|
||||
|
||||
STACKLEAK_ERASE
|
||||
POP_REGS
|
||||
add $8, %rsp /* orig_ax */
|
||||
swapgs
|
||||
jmp .Lnative_iret
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
.Lpti_restore_regs_and_return_to_usermode:
|
||||
POP_REGS pop_rdi=0
|
||||
|
||||
/*
|
||||
|
@ -596,13 +607,15 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
|
|||
*/
|
||||
STACKLEAK_ERASE_NOCLOBBER
|
||||
|
||||
SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
|
||||
push %rax
|
||||
SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
|
||||
pop %rax
|
||||
|
||||
/* Restore RDI. */
|
||||
popq %rdi
|
||||
swapgs
|
||||
jmp .Lnative_iret
|
||||
|
||||
#endif
|
||||
|
||||
SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
|
||||
#ifdef CONFIG_DEBUG_ENTRY
|
||||
|
|
Loading…
Reference in a new issue