mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-04 01:49:36 +00:00
arm64: Add percpu vectors for EL1
commit bd09128d16
upstream.
The Spectre-BHB workaround adds a firmware call to the vectors. This
is needed on some CPUs, but not others. To avoid the unaffected CPU in
a big/little pair from making the firmware call, create per cpu vectors.
The per-cpu vectors only apply when returning from EL0.
Systems using KPTI can use the canonical 'full-fat' vectors directly at
EL1, the trampoline exit code will switch to this_cpu_vector on exit to
EL0. Systems not using KPTI should always use this_cpu_vector.
this_cpu_vector will point at a vector in tramp_vecs or
__bp_harden_el1_vectors, depending on whether KPTI is in use.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
parent
cfd0c38125
commit
d7066114dc
4 changed files with 51 additions and 8 deletions
|
@ -5,6 +5,15 @@
|
|||
#ifndef __ASM_VECTORS_H
|
||||
#define __ASM_VECTORS_H
|
||||
|
||||
#include <linux/bug.h>
|
||||
#include <linux/percpu.h>
|
||||
|
||||
#include <asm/fixmap.h>
|
||||
|
||||
extern char vectors[];
|
||||
extern char tramp_vectors[];
|
||||
extern char __bp_harden_el1_vectors[];
|
||||
|
||||
/*
|
||||
* Note: the order of this enum corresponds to two arrays in entry.S:
|
||||
* tramp_vecs and __bp_harden_el1_vectors. By default the canonical
|
||||
|
@ -31,4 +40,22 @@ enum arm64_bp_harden_el1_vectors {
|
|||
EL1_VECTOR_KPTI,
|
||||
};
|
||||
|
||||
/* The vectors to use on return from EL0. e.g. to remap the kernel */
|
||||
DECLARE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector);
|
||||
|
||||
#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||
#define TRAMP_VALIAS 0
|
||||
#endif
|
||||
|
||||
static inline const char *
|
||||
arm64_get_bp_hardening_vector(enum arm64_bp_harden_el1_vectors slot)
|
||||
{
|
||||
if (arm64_kernel_unmapped_at_el0())
|
||||
return (char *)TRAMP_VALIAS + SZ_2K * slot;
|
||||
|
||||
WARN_ON_ONCE(slot == EL1_VECTOR_KPTI);
|
||||
|
||||
return __bp_harden_el1_vectors + SZ_2K * slot;
|
||||
}
|
||||
|
||||
#endif /* __ASM_VECTORS_H */
|
||||
|
|
|
@ -73,6 +73,8 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/kasan.h>
|
||||
#include <linux/percpu.h>
|
||||
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/cpu_ops.h>
|
||||
|
@ -85,6 +87,7 @@
|
|||
#include <asm/smp.h>
|
||||
#include <asm/sysreg.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/vectors.h>
|
||||
#include <asm/virt.h>
|
||||
|
||||
/* Kernel representation of AT_HWCAP and AT_HWCAP2 */
|
||||
|
@ -110,6 +113,8 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
|
|||
bool arm64_use_ng_mappings = false;
|
||||
EXPORT_SYMBOL(arm64_use_ng_mappings);
|
||||
|
||||
DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
|
||||
|
||||
/*
|
||||
* Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
|
||||
* support it?
|
||||
|
@ -1590,6 +1595,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
|
|||
|
||||
int cpu = smp_processor_id();
|
||||
|
||||
if (__this_cpu_read(this_cpu_vector) == vectors) {
|
||||
const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
|
||||
|
||||
__this_cpu_write(this_cpu_vector, v);
|
||||
}
|
||||
|
||||
/*
|
||||
* We don't need to rewrite the page-tables if either we've done
|
||||
* it already or we have KASLR enabled and therefore have not
|
||||
|
|
|
@ -38,7 +38,6 @@
|
|||
.macro kernel_ventry, el:req, ht:req, regsize:req, label:req
|
||||
.align 7
|
||||
.Lventry_start\@:
|
||||
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
|
||||
.if \el == 0
|
||||
/*
|
||||
* This must be the first instruction of the EL0 vector entries. It is
|
||||
|
@ -53,7 +52,6 @@
|
|||
.endif
|
||||
.Lskip_tramp_vectors_cleanup\@:
|
||||
.endif
|
||||
#endif
|
||||
|
||||
sub sp, sp, #PT_REGS_SIZE
|
||||
#ifdef CONFIG_VMAP_STACK
|
||||
|
@ -712,10 +710,10 @@ alternative_else_nop_endif
|
|||
.endm
|
||||
|
||||
.macro tramp_exit, regsize = 64
|
||||
adr x30, tramp_vectors
|
||||
#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
|
||||
add x30, x30, SZ_4K
|
||||
#endif
|
||||
tramp_data_read_var x30, this_cpu_vector
|
||||
get_this_cpu_offset x29
|
||||
ldr x30, [x30, x29]
|
||||
|
||||
msr vbar_el1, x30
|
||||
ldr lr, [sp, #S_LR]
|
||||
tramp_unmap_kernel x29
|
||||
|
@ -775,6 +773,8 @@ __entry_tramp_data_vectors:
|
|||
__entry_tramp_data___sdei_asm_handler:
|
||||
.quad __sdei_asm_handler
|
||||
#endif /* CONFIG_ARM_SDE_INTERFACE */
|
||||
__entry_tramp_data_this_cpu_vector:
|
||||
.quad this_cpu_vector
|
||||
SYM_DATA_END(__entry_tramp_data_start)
|
||||
.popsection // .rodata
|
||||
#endif /* CONFIG_RANDOMIZE_BASE */
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
#include <linux/kvm_host.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <uapi/linux/psci.h>
|
||||
|
||||
#include <kvm/arm_psci.h>
|
||||
|
@ -25,6 +26,7 @@
|
|||
#include <asm/debug-monitors.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/thread_info.h>
|
||||
#include <asm/vectors.h>
|
||||
|
||||
/* VHE specific context */
|
||||
DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
|
||||
|
@ -68,7 +70,7 @@ NOKPROBE_SYMBOL(__activate_traps);
|
|||
|
||||
static void __deactivate_traps(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
extern char vectors[]; /* kernel exception vectors */
|
||||
const char *host_vectors = vectors;
|
||||
|
||||
___deactivate_traps(vcpu);
|
||||
|
||||
|
@ -82,7 +84,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
|
|||
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
|
||||
|
||||
write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
|
||||
write_sysreg(vectors, vbar_el1);
|
||||
|
||||
if (!arm64_kernel_unmapped_at_el0())
|
||||
host_vectors = __this_cpu_read(this_cpu_vector);
|
||||
write_sysreg(host_vectors, vbar_el1);
|
||||
}
|
||||
NOKPROBE_SYMBOL(__deactivate_traps);
|
||||
|
||||
|
|
Loading…
Reference in a new issue