arm64: entry: Simplify tramp_alias macro and tramp_exit routine
The tramp_alias macro constructs the virtual alias of a symbol in the trampoline text mapping, based on its kernel text address, and does so in a way that is more convoluted than necessary. So let's simplify that. Also, now that the address of the vector table is kept in a per-CPU variable, there is no need to defer the load and the assignment of VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference to the per-CPU variable instead of storing its absolute address in a global variable in the trampoline rodata. And given that tramp_alias no longer needs a temp register, this means we can restore X30 earlier as well, and only leave X29 for tramp_exit() to restore. While at it, give some related symbols static linkage, considering that they are only referenced from the object file that defines them. Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Reviewed-by: Will Deacon <will@kernel.org> Link: https://lore.kernel.org/r/20230418143604.1176437-3-ardb@kernel.org Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
This commit is contained in:
parent
0936243cab
commit
211ceca377
|
@ -101,12 +101,11 @@
|
||||||
.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
|
.org .Lventry_start\@ + 128 // Did we overflow the ventry slot?
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro tramp_alias, dst, sym, tmp
|
.macro tramp_alias, dst, sym
|
||||||
mov_q \dst, TRAMP_VALIAS
|
.set .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text
|
||||||
adr_l \tmp, \sym
|
movz \dst, :abs_g2_s:.Lalias\@
|
||||||
add \dst, \dst, \tmp
|
movk \dst, :abs_g1_nc:.Lalias\@
|
||||||
adr_l \tmp, .entry.tramp.text
|
movk \dst, :abs_g0_nc:.Lalias\@
|
||||||
sub \dst, \dst, \tmp
|
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
|
||||||
alternative_else_nop_endif
|
alternative_else_nop_endif
|
||||||
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||||
msr far_el1, x29
|
msr far_el1, x29
|
||||||
tramp_alias x30, tramp_exit_native, x29
|
|
||||||
br x30
|
ldr_this_cpu x30, this_cpu_vector, x29
|
||||||
|
tramp_alias x29, tramp_exit
|
||||||
|
msr vbar_el1, x30 // install vector table
|
||||||
|
ldr lr, [sp, #S_LR] // restore x30
|
||||||
|
add sp, sp, #PT_REGS_SIZE // restore sp
|
||||||
|
br x29
|
||||||
#endif
|
#endif
|
||||||
.else
|
.else
|
||||||
ldr lr, [sp, #S_LR]
|
ldr lr, [sp, #S_LR]
|
||||||
|
@ -728,20 +732,6 @@ alternative_else_nop_endif
|
||||||
.org 1b + 128 // Did we overflow the ventry slot?
|
.org 1b + 128 // Did we overflow the ventry slot?
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro tramp_exit, regsize = 64
|
|
||||||
tramp_data_read_var x30, this_cpu_vector
|
|
||||||
get_this_cpu_offset x29
|
|
||||||
ldr x30, [x30, x29]
|
|
||||||
|
|
||||||
msr vbar_el1, x30
|
|
||||||
ldr lr, [sp, #S_LR]
|
|
||||||
tramp_unmap_kernel x29
|
|
||||||
mrs x29, far_el1
|
|
||||||
add sp, sp, #PT_REGS_SIZE // restore sp
|
|
||||||
eret
|
|
||||||
sb
|
|
||||||
.endm
|
|
||||||
|
|
||||||
.macro generate_tramp_vector, kpti, bhb
|
.macro generate_tramp_vector, kpti, bhb
|
||||||
.Lvector_start\@:
|
.Lvector_start\@:
|
||||||
.space 0x400
|
.space 0x400
|
||||||
|
@ -762,7 +752,7 @@ alternative_else_nop_endif
|
||||||
*/
|
*/
|
||||||
.pushsection ".entry.tramp.text", "ax"
|
.pushsection ".entry.tramp.text", "ax"
|
||||||
.align 11
|
.align 11
|
||||||
SYM_CODE_START_NOALIGN(tramp_vectors)
|
SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors)
|
||||||
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
|
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
|
||||||
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
|
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_LOOP
|
||||||
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
|
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_FW
|
||||||
|
@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors)
|
||||||
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
|
generate_tramp_vector kpti=1, bhb=BHB_MITIGATION_NONE
|
||||||
SYM_CODE_END(tramp_vectors)
|
SYM_CODE_END(tramp_vectors)
|
||||||
|
|
||||||
SYM_CODE_START(tramp_exit_native)
|
SYM_CODE_START_LOCAL(tramp_exit)
|
||||||
tramp_exit
|
tramp_unmap_kernel x29
|
||||||
SYM_CODE_END(tramp_exit_native)
|
mrs x29, far_el1 // restore x29
|
||||||
|
eret
|
||||||
|
sb
|
||||||
|
SYM_CODE_END(tramp_exit)
|
||||||
.popsection // .entry.tramp.text
|
.popsection // .entry.tramp.text
|
||||||
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
|
#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
|
||||||
|
|
||||||
|
@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
|
||||||
alternative_else_nop_endif
|
alternative_else_nop_endif
|
||||||
|
|
||||||
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||||
tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
|
tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline
|
||||||
br x5
|
br x5
|
||||||
#endif
|
#endif
|
||||||
SYM_CODE_END(__sdei_asm_handler)
|
SYM_CODE_END(__sdei_asm_handler)
|
||||||
|
|
Loading…
Reference in New Issue