arm64: entry: handle all vectors with C
We have 16 architectural exception vectors, and depending on kernel configuration we handle 8 or 12 of these with C code, with the remaining 8 or 4 of these handled as special cases in the entry assembly. It would be nicer if the entry assembly were uniform for all exceptions, and we deferred any specific handling of the exceptions to C code. This way the entry assembly can be more easily templated without ifdeffery or special cases, and it's easier to modify the handling of these cases in future (e.g. to dump additional registers other context). This patch reworks the entry code so that we always have a C handler for every architectural exception vector, with the entry assembly being completely uniform. We now have to handle exceptions from EL1t and EL1h, and also have to handle exceptions from AArch32 even when the kernel is built without CONFIG_COMPAT. To make this clear and to simplify templating, we rename the top-level exception handlers with a consistent naming scheme: asm: <el+sp>_<regsize>_<type> c: <el+sp>_<regsize>_<type>_handler .. where: <el+sp> is `el1t`, `el1h`, or `el0t` <regsize> is `64` or `32` <type> is `sync`, `irq`, `fiq`, or `error` ... e.g. asm: el1h_64_sync c: el1h_64_sync_handler ... with lower-level handlers simply using "el1" and "compat" as today. For unexpected exceptions, this information is passed to __panic_unhandled(), so it can report the specific vector an unexpected exception was taken from, e.g. | Unhandled 64-bit el1t sync exception For vectors we never expect to enter legitimately, the C code is generated using a macro to avoid code duplication. The exceptions are handled via __panic_unhandled(), replacing bad_mode() (which is removed). The `kernel_ventry` and `entry_handler` assembly macros are updated to handle the new naming scheme. In theory it should be possible to generate the entry functions at the same time as the vectors using a single table, but this will require reworking the linker script to split the two into separate sections, so for now we have separate tables. Signed-off-by: Mark Rutland <mark.rutland@arm.com> Acked-by: Catalin Marinas <catalin.marinas@arm.com> Acked-by: Marc Zyngier <maz@kernel.org> Reviewed-by: Joey Gouly <joey.gouly@arm.com> Cc: James Morse <james.morse@arm.com> Cc: Will Deacon <will@kernel.org> Link: https://lore.kernel.org/r/20210607094624.34689-15-mark.rutland@arm.com Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Mark Rutland
Will Deacon
parent
a5b43a87a7
commit
ec841aab8d
|
|
@ -31,18 +31,25 @@ static inline u32 disr_to_esr(u64 disr)
|
|||
return esr;
|
||||
}
|
||||
|
||||
asmlinkage void el1_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1_error_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_error_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_sync_compat_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_irq_compat_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_fiq_compat_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0_error_compat_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1t_64_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1t_64_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1t_64_error_handler(struct pt_regs *regs);
|
||||
|
||||
asmlinkage void el1h_64_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1h_64_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el1h_64_error_handler(struct pt_regs *regs);
|
||||
|
||||
asmlinkage void el0t_64_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_64_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_64_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_64_error_handler(struct pt_regs *regs);
|
||||
|
||||
asmlinkage void el0t_32_sync_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_32_irq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_32_fiq_handler(struct pt_regs *regs);
|
||||
asmlinkage void el0t_32_error_handler(struct pt_regs *regs);
|
||||
|
||||
asmlinkage void call_on_irq_stack(struct pt_regs *regs,
|
||||
void (*func)(struct pt_regs *));
|
||||
|
|
@ -53,7 +60,6 @@ void arm64_exit_nmi(struct pt_regs *regs);
|
|||
void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs);
|
||||
void do_undefinstr(struct pt_regs *regs);
|
||||
void do_bti(struct pt_regs *regs);
|
||||
asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);
|
||||
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
|
||||
struct pt_regs *regs);
|
||||
void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs);
|
||||
|
|
|
|||
|
|
@ -172,16 +172,11 @@ static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
|
|||
panic("Unhandled exception");
|
||||
}
|
||||
|
||||
asmlinkage void noinstr bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
|
||||
{
|
||||
const char *handler[] = {
|
||||
"Synchronous Abort",
|
||||
"IRQ",
|
||||
"FIQ",
|
||||
"Error"
|
||||
};
|
||||
|
||||
__panic_unhandled(regs, handler[reason], esr);
|
||||
#define UNHANDLED(el, regsize, vector) \
|
||||
asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
|
||||
{ \
|
||||
const char *desc = #regsize "-bit " #el " " #vector; \
|
||||
__panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_ERRATUM_1463225
|
||||
|
|
@ -233,6 +228,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
|
|||
}
|
||||
#endif /* CONFIG_ARM64_ERRATUM_1463225 */
|
||||
|
||||
UNHANDLED(el1t, 64, sync)
|
||||
UNHANDLED(el1t, 64, irq)
|
||||
UNHANDLED(el1t, 64, fiq)
|
||||
UNHANDLED(el1t, 64, error)
|
||||
|
||||
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long far = read_sysreg(far_el1);
|
||||
|
|
@ -268,7 +268,7 @@ static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
|
|||
{
|
||||
enter_from_kernel_mode(regs);
|
||||
local_daif_inherit(regs);
|
||||
bad_mode(regs, 0, esr);
|
||||
__panic_unhandled(regs, "64-bit el1h sync", esr);
|
||||
local_daif_mask();
|
||||
exit_to_kernel_mode(regs);
|
||||
}
|
||||
|
|
@ -316,7 +316,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
|
|||
exit_to_kernel_mode(regs);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
|
|
@ -370,17 +370,17 @@ static void noinstr el1_interrupt(struct pt_regs *regs,
|
|||
exit_el1_irq_or_nmi(regs);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el1_irq_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
|
||||
{
|
||||
el1_interrupt(regs, handle_arch_irq);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el1_fiq_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
|
||||
{
|
||||
el1_interrupt(regs, handle_arch_fiq);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el1_error_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
|
|
@ -526,7 +526,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
|
|||
do_ptrauth_fault(regs, esr);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
|
|
@ -597,7 +597,7 @@ static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
|
|||
el0_interrupt(regs, handle_arch_irq);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_irq_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_irq_handler_common(regs);
|
||||
}
|
||||
|
|
@ -607,7 +607,7 @@ static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
|
|||
el0_interrupt(regs, handle_arch_fiq);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_fiq_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_fiq_handler_common(regs);
|
||||
}
|
||||
|
|
@ -624,7 +624,7 @@ static void __el0_error_handler_common(struct pt_regs *regs)
|
|||
local_daif_restore(DAIF_PROCCTX);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_error_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_error_handler_common(regs);
|
||||
}
|
||||
|
|
@ -644,7 +644,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs)
|
|||
do_el0_svc_compat(regs);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long esr = read_sysreg(esr_el1);
|
||||
|
||||
|
|
@ -688,18 +688,23 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
|
|||
}
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_irq_compat_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_irq_handler_common(regs);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_fiq_compat_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_fiq_handler_common(regs);
|
||||
}
|
||||
|
||||
asmlinkage void noinstr el0_error_compat_handler(struct pt_regs *regs)
|
||||
asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
|
||||
{
|
||||
__el0_error_handler_common(regs);
|
||||
}
|
||||
#else /* CONFIG_COMPAT */
|
||||
UNHANDLED(el0t, 32, sync)
|
||||
UNHANDLED(el0t, 32, irq)
|
||||
UNHANDLED(el0t, 32, fiq)
|
||||
UNHANDLED(el0t, 32, error)
|
||||
#endif /* CONFIG_COMPAT */
|
||||
|
|
|
|||
|
|
@ -45,16 +45,7 @@
|
|||
.endr
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Bad Abort numbers
|
||||
*-----------------
|
||||
*/
|
||||
#define BAD_SYNC 0
|
||||
#define BAD_IRQ 1
|
||||
#define BAD_FIQ 2
|
||||
#define BAD_ERROR 3
|
||||
|
||||
.macro kernel_ventry, el:req, regsize:req, label:req
|
||||
.macro kernel_ventry, el:req, ht:req, regsize:req, label:req
|
||||
.align 7
|
||||
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||
.if \el == 0
|
||||
|
|
@ -81,7 +72,7 @@ alternative_else_nop_endif
|
|||
tbnz x0, #THREAD_SHIFT, 0f
|
||||
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
|
||||
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
|
||||
b el\()\el\()_\label
|
||||
b el\el\ht\()_\regsize\()_\label
|
||||
|
||||
0:
|
||||
/*
|
||||
|
|
@ -113,7 +104,7 @@ alternative_else_nop_endif
|
|||
sub sp, sp, x0
|
||||
mrs x0, tpidrro_el0
|
||||
#endif
|
||||
b el\()\el\()_\label
|
||||
b el\el\ht\()_\regsize\()_\label
|
||||
.endm
|
||||
|
||||
.macro tramp_alias, dst, sym
|
||||
|
|
@ -504,32 +495,25 @@ tsk .req x28 // current thread_info
|
|||
|
||||
.align 11
|
||||
SYM_CODE_START(vectors)
|
||||
kernel_ventry 1, 64, sync_invalid // Synchronous EL1t
|
||||
kernel_ventry 1, 64, irq_invalid // IRQ EL1t
|
||||
kernel_ventry 1, 64, fiq_invalid // FIQ EL1t
|
||||
kernel_ventry 1, 64, error_invalid // Error EL1t
|
||||
kernel_ventry 1, t, 64, sync // Synchronous EL1t
|
||||
kernel_ventry 1, t, 64, irq // IRQ EL1t
|
||||
kernel_ventry 1, t, 64, fiq // FIQ EL1h
|
||||
kernel_ventry 1, t, 64, error // Error EL1t
|
||||
|
||||
kernel_ventry 1, 64, sync // Synchronous EL1h
|
||||
kernel_ventry 1, 64, irq // IRQ EL1h
|
||||
kernel_ventry 1, 64, fiq // FIQ EL1h
|
||||
kernel_ventry 1, 64, error // Error EL1h
|
||||
kernel_ventry 1, h, 64, sync // Synchronous EL1h
|
||||
kernel_ventry 1, h, 64, irq // IRQ EL1h
|
||||
kernel_ventry 1, h, 64, fiq // FIQ EL1h
|
||||
kernel_ventry 1, h, 64, error // Error EL1h
|
||||
|
||||
kernel_ventry 0, 64, sync // Synchronous 64-bit EL0
|
||||
kernel_ventry 0, 64, irq // IRQ 64-bit EL0
|
||||
kernel_ventry 0, 64, fiq // FIQ 64-bit EL0
|
||||
kernel_ventry 0, 64, error // Error 64-bit EL0
|
||||
kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0
|
||||
kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0
|
||||
kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0
|
||||
kernel_ventry 0, t, 64, error // Error 64-bit EL0
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
kernel_ventry 0, 32, sync_compat // Synchronous 32-bit EL0
|
||||
kernel_ventry 0, 32, irq_compat // IRQ 32-bit EL0
|
||||
kernel_ventry 0, 32, fiq_compat // FIQ 32-bit EL0
|
||||
kernel_ventry 0, 32, error_compat // Error 32-bit EL0
|
||||
#else
|
||||
kernel_ventry 0, 32, sync_invalid // Synchronous 32-bit EL0
|
||||
kernel_ventry 0, 32, irq_invalid // IRQ 32-bit EL0
|
||||
kernel_ventry 0, 32, fiq_invalid // FIQ 32-bit EL0
|
||||
kernel_ventry 0, 32, error_invalid // Error 32-bit EL0
|
||||
#endif
|
||||
kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0
|
||||
kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0
|
||||
kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0
|
||||
kernel_ventry 0, t, 32, error // Error 32-bit EL0
|
||||
SYM_CODE_END(vectors)
|
||||
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
|
|
@ -560,82 +544,42 @@ __bad_stack:
|
|||
ASM_BUG()
|
||||
#endif /* CONFIG_VMAP_STACK */
|
||||
|
||||
/*
|
||||
* Invalid mode handlers
|
||||
*/
|
||||
.macro inv_entry, el, reason, regsize = 64
|
||||
|
||||
.macro entry_handler el:req, ht:req, regsize:req, label:req
|
||||
SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label)
|
||||
kernel_entry \el, \regsize
|
||||
mov x0, sp
|
||||
mov x1, #\reason
|
||||
mrs x2, esr_el1
|
||||
bl bad_mode
|
||||
ASM_BUG()
|
||||
.endm
|
||||
|
||||
SYM_CODE_START_LOCAL(el0_sync_invalid)
|
||||
inv_entry 0, BAD_SYNC
|
||||
SYM_CODE_END(el0_sync_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el0_irq_invalid)
|
||||
inv_entry 0, BAD_IRQ
|
||||
SYM_CODE_END(el0_irq_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el0_fiq_invalid)
|
||||
inv_entry 0, BAD_FIQ
|
||||
SYM_CODE_END(el0_fiq_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el0_error_invalid)
|
||||
inv_entry 0, BAD_ERROR
|
||||
SYM_CODE_END(el0_error_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el1_sync_invalid)
|
||||
inv_entry 1, BAD_SYNC
|
||||
SYM_CODE_END(el1_sync_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el1_irq_invalid)
|
||||
inv_entry 1, BAD_IRQ
|
||||
SYM_CODE_END(el1_irq_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el1_fiq_invalid)
|
||||
inv_entry 1, BAD_FIQ
|
||||
SYM_CODE_END(el1_fiq_invalid)
|
||||
|
||||
SYM_CODE_START_LOCAL(el1_error_invalid)
|
||||
inv_entry 1, BAD_ERROR
|
||||
SYM_CODE_END(el1_error_invalid)
|
||||
|
||||
.macro entry_handler el:req, regsize:req, label:req
|
||||
SYM_CODE_START_LOCAL(el\el\()_\label)
|
||||
kernel_entry \el, \regsize
|
||||
mov x0, sp
|
||||
bl el\el\()_\label\()_handler
|
||||
bl el\el\ht\()_\regsize\()_\label\()_handler
|
||||
.if \el == 0
|
||||
b ret_to_user
|
||||
.else
|
||||
b ret_to_kernel
|
||||
.endif
|
||||
SYM_CODE_END(el\el\()_\label)
|
||||
SYM_CODE_END(el\el\ht\()_\regsize\()_\label)
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Early exception handlers
|
||||
*/
|
||||
entry_handler 1, 64, sync
|
||||
entry_handler 1, 64, irq
|
||||
entry_handler 1, 64, fiq
|
||||
entry_handler 1, 64, error
|
||||
entry_handler 1, t, 64, sync
|
||||
entry_handler 1, t, 64, irq
|
||||
entry_handler 1, t, 64, fiq
|
||||
entry_handler 1, t, 64, error
|
||||
|
||||
entry_handler 0, 64, sync
|
||||
entry_handler 0, 64, irq
|
||||
entry_handler 0, 64, fiq
|
||||
entry_handler 0, 64, error
|
||||
entry_handler 1, h, 64, sync
|
||||
entry_handler 1, h, 64, irq
|
||||
entry_handler 1, h, 64, fiq
|
||||
entry_handler 1, h, 64, error
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
entry_handler 0, 32, sync_compat
|
||||
entry_handler 0, 32, irq_compat
|
||||
entry_handler 0, 32, fiq_compat
|
||||
entry_handler 0, 32, error_compat
|
||||
#endif
|
||||
entry_handler 0, t, 64, sync
|
||||
entry_handler 0, t, 64, irq
|
||||
entry_handler 0, t, 64, fiq
|
||||
entry_handler 0, t, 64, error
|
||||
|
||||
entry_handler 0, t, 32, sync
|
||||
entry_handler 0, t, 32, irq
|
||||
entry_handler 0, t, 32, fiq
|
||||
entry_handler 0, t, 32, error
|
||||
|
||||
SYM_CODE_START_LOCAL(ret_to_kernel)
|
||||
kernel_exit 1
|
||||
|
|
|
|||
|
|
@ -745,7 +745,7 @@ const char *esr_get_class_string(u32 esr)
|
|||
|
||||
/*
|
||||
* bad_el0_sync handles unexpected, but potentially recoverable synchronous
|
||||
* exceptions taken from EL0. Unlike bad_mode, this returns.
|
||||
* exceptions taken from EL0.
|
||||
*/
|
||||
void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue