2019-08-13 23:04:50 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
|
|
|
/*
|
|
|
|
|
* Linker script variables to be set after section resolution, as
|
|
|
|
|
* ld.lld does not like variables assigned before SECTIONS is processed.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ARM64_KERNEL_IMAGE_VARS_H
|
|
|
|
|
#define __ARM64_KERNEL_IMAGE_VARS_H
|
|
|
|
|
|
|
|
|
|
#ifndef LINKER_SCRIPT
|
|
|
|
|
#error This file should only be included in vmlinux.lds.S
|
|
|
|
|
#endif
|
|
|
|
|
|
2023-01-11 10:22:36 +00:00
|
|
|
PROVIDE(__efistub_primary_entry = primary_entry);
|
2019-08-13 23:04:50 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The EFI stub has its own symbol namespace prefixed by __efistub_, to
|
|
|
|
|
* isolate it from the kernel proper. The following symbols are legally
|
|
|
|
|
* accessed by the stub, so provide some aliases to make them accessible.
|
|
|
|
|
* Only include data symbols here, or text symbols of functions that are
|
|
|
|
|
* guaranteed to be safe when executed at another offset than they were
|
|
|
|
|
* linked at. The routines below are all implemented in assembler in a
|
|
|
|
|
* position independent manner
|
|
|
|
|
*/
|
2023-01-11 10:22:36 +00:00
|
|
|
PROVIDE(__efistub_caches_clean_inval_pou = __pi_caches_clean_inval_pou);
|
2019-08-13 23:04:50 +00:00
|
|
|
|
2022-06-29 08:32:46 +00:00
|
|
|
PROVIDE(__efistub__text = _text);
|
|
|
|
|
PROVIDE(__efistub__end = _end);
|
2023-01-11 10:22:36 +00:00
|
|
|
PROVIDE(__efistub___inittext_end = __inittext_end);
|
2022-06-29 08:32:46 +00:00
|
|
|
PROVIDE(__efistub__edata = _edata);
|
efi: move screen_info into efi init code
After the vga console no longer relies on global screen_info, there are
only two remaining use cases:
- on the x86 architecture, it is used for multiple boot methods
(bzImage, EFI, Xen, kexec) to commucate the initial VGA or framebuffer
settings to a number of device drivers.
- on other architectures, it is only used as part of the EFI stub,
and only for the three sysfb framebuffers (simpledrm, simplefb, efifb).
Remove the duplicate data structure definitions by moving it into the
efi-init.c file that sets it up initially for the EFI case, leaving x86
as an exception that retains its own definition for non-EFI boots.
The added #ifdefs here are optional, I added them to further limit the
reach of screen_info to configurations that have at least one of the
users enabled.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Javier Martinez Canillas <javierm@redhat.com>
Acked-by: Helge Deller <deller@gmx.de>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Link: https://lore.kernel.org/r/20231017093947.3627976-1-arnd@kernel.org
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2023-10-17 09:39:46 +00:00
|
|
|
#if defined(CONFIG_EFI_EARLYCON) || defined(CONFIG_SYSFB)
|
2022-06-29 08:32:46 +00:00
|
|
|
PROVIDE(__efistub_screen_info = screen_info);
|
efi: move screen_info into efi init code
After the vga console no longer relies on global screen_info, there are
only two remaining use cases:
- on the x86 architecture, it is used for multiple boot methods
(bzImage, EFI, Xen, kexec) to commucate the initial VGA or framebuffer
settings to a number of device drivers.
- on other architectures, it is only used as part of the EFI stub,
and only for the three sysfb framebuffers (simpledrm, simplefb, efifb).
Remove the duplicate data structure definitions by moving it into the
efi-init.c file that sets it up initially for the EFI case, leaving x86
as an exception that retains its own definition for non-EFI boots.
The added #ifdefs here are optional, I added them to further limit the
reach of screen_info to configurations that have at least one of the
users enabled.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Javier Martinez Canillas <javierm@redhat.com>
Acked-by: Helge Deller <deller@gmx.de>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Link: https://lore.kernel.org/r/20231017093947.3627976-1-arnd@kernel.org
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2023-10-17 09:39:46 +00:00
|
|
|
#endif
|
2022-06-29 08:32:46 +00:00
|
|
|
PROVIDE(__efistub__ctype = _ctype);
|
2019-08-13 23:04:50 +00:00
|
|
|
|
2022-06-29 08:32:46 +00:00
|
|
|
PROVIDE(__pi___memcpy = __pi_memcpy);
|
|
|
|
|
PROVIDE(__pi___memmove = __pi_memmove);
|
|
|
|
|
PROVIDE(__pi___memset = __pi_memset);
|
2019-08-13 23:04:50 +00:00
|
|
|
|
2024-02-14 12:28:50 +00:00
|
|
|
PROVIDE(__pi_id_aa64isar1_override = id_aa64isar1_override);
|
|
|
|
|
PROVIDE(__pi_id_aa64isar2_override = id_aa64isar2_override);
|
2024-02-14 12:29:12 +00:00
|
|
|
PROVIDE(__pi_id_aa64mmfr0_override = id_aa64mmfr0_override);
|
2024-02-14 12:28:50 +00:00
|
|
|
PROVIDE(__pi_id_aa64mmfr1_override = id_aa64mmfr1_override);
|
2024-02-14 12:29:12 +00:00
|
|
|
PROVIDE(__pi_id_aa64mmfr2_override = id_aa64mmfr2_override);
|
2024-02-14 12:28:50 +00:00
|
|
|
PROVIDE(__pi_id_aa64pfr0_override = id_aa64pfr0_override);
|
|
|
|
|
PROVIDE(__pi_id_aa64pfr1_override = id_aa64pfr1_override);
|
|
|
|
|
PROVIDE(__pi_id_aa64smfr0_override = id_aa64smfr0_override);
|
|
|
|
|
PROVIDE(__pi_id_aa64zfr0_override = id_aa64zfr0_override);
|
|
|
|
|
PROVIDE(__pi_arm64_sw_feature_override = arm64_sw_feature_override);
|
2024-02-14 12:29:04 +00:00
|
|
|
PROVIDE(__pi_arm64_use_ng_mappings = arm64_use_ng_mappings);
|
|
|
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_27456
|
|
|
|
|
PROVIDE(__pi_cavium_erratum_27456_cpus = cavium_erratum_27456_cpus);
|
|
|
|
|
#endif
|
2024-02-14 12:28:50 +00:00
|
|
|
PROVIDE(__pi__ctype = _ctype);
|
2024-02-14 12:29:01 +00:00
|
|
|
PROVIDE(__pi_memstart_offset_seed = memstart_offset_seed);
|
2023-12-13 08:40:32 +00:00
|
|
|
|
arm64: kernel: Create initial ID map from C code
The asm code that creates the initial ID map is rather intricate and
hard to follow. This is problematic because it makes adding support for
things like LPA2 or WXN more difficult than necessary. Also, it is
parameterized like the rest of the MM code to run with a configurable
number of levels, which is rather pointless, given that all AArch64 CPUs
implement support for 48-bit virtual addressing, and that many systems
exist with DRAM located outside of the 39-bit addressable range, which
is the only smaller VA size that is widely used, and we need additional
tricks to make things work in that combination.
So let's bite the bullet, and rip out all the asm macros, and fiddly
code, and replace it with a C implementation based on the newly added
routines for creating the early kernel VA mappings. And while at it,
create the initial ID map based on 48-bit virtual addressing as well,
regardless of the number of configured levels for the kernel proper.
Note that this code may execute with the MMU and caches disabled, and is
therefore not permitted to make unaligned accesses. This shouldn't
generally happen in any case for the algorithm as implemented, but to be
sure, let's pass -mstrict-align to the compiler just in case.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-66-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2024-02-14 12:29:07 +00:00
|
|
|
PROVIDE(__pi_init_idmap_pg_dir = init_idmap_pg_dir);
|
arm64: Enable LPA2 at boot if supported by the system
Update the early kernel mapping code to take 52-bit virtual addressing
into account based on the LPA2 feature. This is a bit more involved than
LVA (which is supported with 64k pages only), given that some page table
descriptor bits change meaning in this case.
To keep the handling in asm to a minimum, the initial ID map is still
created with 48-bit virtual addressing, which implies that the kernel
image must be loaded into 48-bit addressable physical memory. This is
currently required by the boot protocol, even though we happen to
support placement outside of that for LVA/64k based configurations.
Enabling LPA2 involves more than setting TCR.T1SZ to a lower value,
there is also a DS bit in TCR that needs to be set, and which changes
the meaning of bits [9:8] in all page table descriptors. Since we cannot
enable DS and every live page table descriptor at the same time, let's
pivot through another temporary mapping. This avoids the need to
reintroduce manipulations of the page tables with the MMU and caches
disabled.
To permit the LPA2 feature to be overridden on the kernel command line,
which may be necessary to work around silicon errata, or to deal with
mismatched features on heterogeneous SoC designs, test for CPU feature
overrides first, and only then enable LPA2.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-78-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2024-02-14 12:29:19 +00:00
|
|
|
PROVIDE(__pi_init_idmap_pg_end = init_idmap_pg_end);
|
2024-02-14 12:29:04 +00:00
|
|
|
PROVIDE(__pi_init_pg_dir = init_pg_dir);
|
|
|
|
|
PROVIDE(__pi_init_pg_end = init_pg_end);
|
2024-02-14 12:29:09 +00:00
|
|
|
PROVIDE(__pi_swapper_pg_dir = swapper_pg_dir);
|
2024-02-14 12:29:04 +00:00
|
|
|
|
|
|
|
|
PROVIDE(__pi__text = _text);
|
|
|
|
|
PROVIDE(__pi__stext = _stext);
|
|
|
|
|
PROVIDE(__pi__etext = _etext);
|
|
|
|
|
PROVIDE(__pi___start_rodata = __start_rodata);
|
|
|
|
|
PROVIDE(__pi___inittext_begin = __inittext_begin);
|
|
|
|
|
PROVIDE(__pi___inittext_end = __inittext_end);
|
|
|
|
|
PROVIDE(__pi___initdata_begin = __initdata_begin);
|
|
|
|
|
PROVIDE(__pi___initdata_end = __initdata_end);
|
|
|
|
|
PROVIDE(__pi__data = _data);
|
|
|
|
|
PROVIDE(__pi___bss_start = __bss_start);
|
|
|
|
|
PROVIDE(__pi__end = _end);
|
|
|
|
|
|
2020-06-25 13:14:08 +00:00
|
|
|
#ifdef CONFIG_KVM
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_, to
|
|
|
|
|
* separate it from the kernel proper. The following symbols are legally
|
|
|
|
|
* accessed by it, therefore provide aliases to make them linkable.
|
|
|
|
|
* Do not include symbols which may not be safely accessed under hypervisor
|
|
|
|
|
* memory mappings.
|
|
|
|
|
*/
|
|
|
|
|
|
2020-06-25 13:14:11 +00:00
|
|
|
/* Alternative callbacks for init-time patching of nVHE hyp code. */
|
|
|
|
|
KVM_NVHE_ALIAS(kvm_patch_vector_branch);
|
|
|
|
|
KVM_NVHE_ALIAS(kvm_update_va_mask);
|
2020-10-24 15:33:38 +00:00
|
|
|
KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
|
2021-03-22 12:09:51 +00:00
|
|
|
KVM_NVHE_ALIAS(kvm_compute_final_ctr_el0);
|
arm64: Mitigate spectre style branch history side channels
Speculation attacks against some high-performance processors can
make use of branch history to influence future speculation.
When taking an exception from user-space, a sequence of branches
or a firmware call overwrites or invalidates the branch history.
The sequence of branches is added to the vectors, and should appear
before the first indirect branch. For systems using KPTI the sequence
is added to the kpti trampoline where it has a free register as the exit
from the trampoline is via a 'ret'. For systems not using KPTI, the same
register tricks are used to free up a register in the vectors.
For the firmware call, arch-workaround-3 clobbers 4 registers, so
there is no choice but to save them to the EL1 stack. This only happens
for entry from EL0, so if we take an exception due to the stack access,
it will not become re-entrant.
For KVM, the existing branch-predictor-hardening vectors are used.
When a spectre version of these vectors is in use, the firmware call
is sufficient to mitigate against Spectre-BHB. For the non-spectre
versions, the sequence of branches is added to the indirect vector.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: James Morse <james.morse@arm.com>
2021-11-10 14:48:00 +00:00
|
|
|
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
|
|
|
|
|
KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
|
|
|
|
|
KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
|
2021-12-10 14:32:56 +00:00
|
|
|
KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
|
arm64: alternatives: add shared NOP callback
For each instance of an alternative, the compiler outputs a distinct
copy of the alternative instructions into a subsection. As the compiler
doesn't have special knowledge of alternatives, it cannot coalesce these
to save space.
In a defconfig kernel built with GCC 12.1.0, there are approximately
10,000 instances of alternative_has_feature_likely(), where the
replacement instruction is always a NOP. As NOPs are
position-independent, we don't need a unique copy per alternative
sequence.
This patch adds a callback to patch an alternative sequence with NOPs,
and make use of this in alternative_has_feature_likely(). So that this
can be used for other sites in future, this is written to patch multiple
instructions up to the original sequence length.
For NVHE, an alias is added to image-vars.h.
For modules, the callback is exported. Note that as modules are loaded
within 2GiB of the kernel, an alt_instr entry in a module can always
refer directly to the callback, and no special handling is necessary.
When building with GCC 12.1.0, the vmlinux is ~158KiB smaller, though
the resulting Image size is unchanged due to alignment constraints and
padding:
| % ls -al vmlinux-*
| -rwxr-xr-x 1 mark mark 134644592 Sep 1 14:52 vmlinux-after
| -rwxr-xr-x 1 mark mark 134486232 Sep 1 14:50 vmlinux-before
| % ls -al Image-*
| -rw-r--r-- 1 mark mark 37108224 Sep 1 14:52 Image-after
| -rw-r--r-- 1 mark mark 37108224 Sep 1 14:50 Image-before
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220912162210.3626215-9-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2022-09-12 16:22:10 +00:00
|
|
|
KVM_NVHE_ALIAS(alt_cb_patch_nops);
|
2020-06-25 13:14:11 +00:00
|
|
|
|
|
|
|
|
/* Global kernel state accessed by nVHE hyp code. */
|
2020-06-25 13:14:14 +00:00
|
|
|
KVM_NVHE_ALIAS(kvm_vgic_global_state);
|
2020-06-25 13:14:11 +00:00
|
|
|
|
|
|
|
|
/* Kernel symbols used to call panic() from nVHE hyp code (via ERET). */
|
2021-03-18 14:33:11 +00:00
|
|
|
KVM_NVHE_ALIAS(nvhe_hyp_panic_handler);
|
2020-06-25 13:14:11 +00:00
|
|
|
|
2020-06-25 13:14:12 +00:00
|
|
|
/* Vectors installed by hyp-init on reset HVC. */
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_stub_vectors);
|
|
|
|
|
|
2020-06-25 13:14:14 +00:00
|
|
|
/* Static keys which are set if a vGIC trap should be handled in hyp. */
|
|
|
|
|
KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
|
|
|
|
|
KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_ARM64_PSEUDO_NMI
|
2020-09-12 15:37:07 +00:00
|
|
|
/* Static key checked in GIC_PRIO_IRQOFF. */
|
|
|
|
|
KVM_NVHE_ALIAS(gic_nonsecure_priorities);
|
2020-06-25 13:14:14 +00:00
|
|
|
#endif
|
|
|
|
|
|
2020-08-21 14:07:05 +00:00
|
|
|
/* EL2 exception handling */
|
|
|
|
|
KVM_NVHE_ALIAS(__start___kvm_ex_table);
|
|
|
|
|
KVM_NVHE_ALIAS(__stop___kvm_ex_table);
|
|
|
|
|
|
2021-03-05 18:52:51 +00:00
|
|
|
/* PMU available static key */
|
2021-11-11 02:07:36 +00:00
|
|
|
#ifdef CONFIG_HW_PERF_EVENTS
|
2021-03-05 18:52:51 +00:00
|
|
|
KVM_NVHE_ALIAS(kvm_arm_pmu_available);
|
2021-11-11 02:07:36 +00:00
|
|
|
#endif
|
2021-03-05 18:52:51 +00:00
|
|
|
|
2021-03-19 10:01:10 +00:00
|
|
|
/* Position-independent library routines */
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(clear_page, __pi_clear_page);
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(copy_page, __pi_copy_page);
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(memcpy, __pi_memcpy);
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(memset, __pi_memset);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_KASAN
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy);
|
|
|
|
|
KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);
|
|
|
|
|
#endif
|
|
|
|
|
|
KVM: arm64: Prepare the creation of s1 mappings at EL2
When memory protection is enabled, the EL2 code needs the ability to
create and manage its own page-table. To do so, introduce a new set of
hypercalls to bootstrap a memory management system at EL2.
This leads to the following boot flow in nVHE Protected mode:
1. the host allocates memory for the hypervisor very early on, using
the memblock API;
2. the host creates a set of stage 1 page-table for EL2, installs the
EL2 vectors, and issues the __pkvm_init hypercall;
3. during __pkvm_init, the hypervisor re-creates its stage 1 page-table
and stores it in the memory pool provided by the host;
4. the hypervisor then extends its stage 1 mappings to include a
vmemmap in the EL2 VA space, hence allowing to use the buddy
allocator introduced in a previous patch;
5. the hypervisor jumps back in the idmap page, switches from the
host-provided page-table to the new one, and wraps up its
initialization by enabling the new allocator, before returning to
the host.
6. the host can free the now unused page-table created for EL2, and
will now need to issue hypercalls to make changes to the EL2 stage 1
mappings instead of modifying them directly.
Note that for the sake of simplifying the review, this patch focuses on
the hypervisor side of things. In other words, this only implements the
new hypercalls, but does not make use of them from the host yet. The
host-side changes will follow in a subsequent patch.
Credits to Will for __pkvm_init_switch_pgd.
Acked-by: Will Deacon <will@kernel.org>
Co-authored-by: Will Deacon <will@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Quentin Perret <qperret@google.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20210319100146.1149909-18-qperret@google.com
2021-03-19 10:01:25 +00:00
|
|
|
/* Hyp memory sections */
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_idmap_text_start);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_idmap_text_end);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_text_start);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_text_end);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_bss_start);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_bss_end);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_rodata_start);
|
|
|
|
|
KVM_NVHE_ALIAS(__hyp_rodata_end);
|
|
|
|
|
|
2021-03-19 10:01:43 +00:00
|
|
|
/* pKVM static key */
|
|
|
|
|
KVM_NVHE_ALIAS(kvm_protected_mode_initialized);
|
|
|
|
|
|
2020-06-25 13:14:08 +00:00
|
|
|
#endif /* CONFIG_KVM */
|
|
|
|
|
|
2023-04-18 13:49:49 +00:00
|
|
|
#ifdef CONFIG_EFI_ZBOOT
|
|
|
|
|
_kernel_codesize = ABSOLUTE(__inittext_end - _text);
|
|
|
|
|
#endif
|
|
|
|
|
|
2019-08-13 23:04:50 +00:00
|
|
|
#endif /* __ARM64_KERNEL_IMAGE_VARS_H */
|