2019-06-03 05:44:50 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2012-03-05 11:49:32 +00:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_FP_H
|
|
|
|
|
#define __ASM_FP_H
|
|
|
|
|
|
2017-10-31 15:51:14 +00:00
|
|
|
#include <asm/errno.h>
|
2018-07-11 13:56:40 +00:00
|
|
|
#include <asm/ptrace.h>
|
2018-04-12 16:32:35 +00:00
|
|
|
#include <asm/processor.h>
|
|
|
|
|
#include <asm/sigcontext.h>
|
2018-07-11 13:56:40 +00:00
|
|
|
#include <asm/sysreg.h>
|
2012-03-05 11:49:32 +00:00
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
2018-09-28 13:39:21 +00:00
|
|
|
#include <linux/bitmap.h>
|
2018-07-11 13:56:40 +00:00
|
|
|
#include <linux/build_bug.h>
|
2018-09-28 13:39:21 +00:00
|
|
|
#include <linux/bug.h>
|
2017-10-31 15:51:08 +00:00
|
|
|
#include <linux/cache.h>
|
2018-02-13 17:43:23 +00:00
|
|
|
#include <linux/init.h>
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
#include <linux/stddef.h>
|
2018-09-28 13:39:21 +00:00
|
|
|
#include <linux/types.h>
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
|
2012-03-05 11:49:32 +00:00
|
|
|
/* Masks for extracting the FPSR and FPCR from the FPSCR */
|
|
|
|
|
#define VFP_FPSCR_STAT_MASK 0xf800009f
|
|
|
|
|
#define VFP_FPSCR_CTRL_MASK 0x07f79f00
|
|
|
|
|
/*
|
|
|
|
|
* The VFP state has 32x64-bit registers and a single 32-bit
|
|
|
|
|
* control/status register.
|
|
|
|
|
*/
|
|
|
|
|
#define VFP_STATE_SIZE ((32 * 8) + 4)
|
|
|
|
|
|
2023-10-16 10:24:33 +00:00
|
|
|
static inline unsigned long cpacr_save_enable_kernel_sve(void)
|
|
|
|
|
{
|
|
|
|
|
unsigned long old = read_sysreg(cpacr_el1);
|
|
|
|
|
unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;
|
|
|
|
|
|
|
|
|
|
write_sysreg(old | set, cpacr_el1);
|
|
|
|
|
isb();
|
|
|
|
|
return old;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned long cpacr_save_enable_kernel_sme(void)
|
|
|
|
|
{
|
|
|
|
|
unsigned long old = read_sysreg(cpacr_el1);
|
|
|
|
|
unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;
|
|
|
|
|
|
|
|
|
|
write_sysreg(old | set, cpacr_el1);
|
|
|
|
|
isb();
|
|
|
|
|
return old;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void cpacr_restore(unsigned long cpacr)
|
|
|
|
|
{
|
|
|
|
|
write_sysreg(cpacr, cpacr_el1);
|
|
|
|
|
isb();
|
|
|
|
|
}
|
|
|
|
|
|
2022-05-05 22:15:17 +00:00
|
|
|
/*
|
|
|
|
|
* When we defined the maximum SVE vector length we defined the ABI so
|
|
|
|
|
* that the maximum vector length included all the reserved for future
|
|
|
|
|
* expansion bits in ZCR rather than those just currently defined by
|
2024-02-13 18:24:38 +00:00
|
|
|
* the architecture. Using this length to allocate worst size buffers
|
|
|
|
|
* results in excessively large allocations, and this effect is even
|
|
|
|
|
* more pronounced for SME due to ZA. Define more suitable VLs for
|
|
|
|
|
* these situations.
|
2022-05-05 22:15:17 +00:00
|
|
|
*/
|
2024-02-13 18:24:38 +00:00
|
|
|
#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
|
|
|
|
|
#define SME_VQ_MAX ((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
|
2022-05-05 22:15:17 +00:00
|
|
|
|
2012-03-05 11:49:32 +00:00
|
|
|
struct task_struct;
|
|
|
|
|
|
2018-03-28 09:50:48 +00:00
|
|
|
extern void fpsimd_save_state(struct user_fpsimd_state *state);
|
|
|
|
|
extern void fpsimd_load_state(struct user_fpsimd_state *state);
|
2012-03-05 11:49:32 +00:00
|
|
|
|
|
|
|
|
extern void fpsimd_thread_switch(struct task_struct *next);
|
|
|
|
|
extern void fpsimd_flush_thread(void);
|
|
|
|
|
|
arm64/sve: Signal handling support
This patch implements support for saving and restoring the SVE
registers around signals.
A fixed-size header struct sve_context is always included in the
signal frame encoding the thread's vector length at the time of
signal delivery, optionally followed by a variable-layout structure
encoding the SVE registers.
Because of the need to preserve backwards compatibility, the FPSIMD
view of the SVE registers is always dumped as a struct
fpsimd_context in the usual way, in addition to any sve_context.
The SVE vector registers are dumped in full, including bits 127:0
of each register which alias the corresponding FPSIMD vector
registers in the hardware. To avoid any ambiguity about which
alias to restore during sigreturn, the kernel always restores bits
127:0 of each SVE vector register from the fpsimd_context in the
signal frame (which must be present): userspace needs to take this
into account if it wants to modify the SVE vector register contents
on return from a signal.
FPSR and FPCR, which are used by both FPSIMD and SVE, are not
included in sve_context because they are always present in
fpsimd_context anyway.
For signal delivery, a new helper
fpsimd_signal_preserve_current_state() is added to update _both_
the FPSIMD and SVE views in the task struct, to make it easier to
populate this information into the signal frame. Because of the
redundancy between the two views of the state, only one is updated
otherwise.
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Alex Bennée <alex.bennee@linaro.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:07 +00:00
|
|
|
extern void fpsimd_signal_preserve_current_state(void);
|
2014-02-24 14:26:27 +00:00
|
|
|
extern void fpsimd_preserve_current_state(void);
|
2014-05-08 09:20:23 +00:00
|
|
|
extern void fpsimd_restore_current_state(void);
|
arm64: fpsimd: Fix state leakage when migrating after sigreturn
When refactoring the sigreturn code to handle SVE, I changed the
sigreturn implementation to store the new FPSIMD state from the
user sigframe into task_struct before reloading the state into the
CPU regs. This makes it easier to convert the data for SVE when
needed.
However, it turns out that the fpsimd_state structure passed into
fpsimd_update_current_state is not fully initialised, so assigning
the structure as a whole corrupts current->thread.fpsimd_state.cpu
with uninitialised data.
This means that if the garbage data written to .cpu happens to be a
valid cpu number, and the task is subsequently migrated to the cpu
identified by the that number, and then tries to enter userspace,
the CPU FPSIMD regs will be assumed to be correct for the task and
not reloaded as they should be. This can result in returning to
userspace with the FPSIMD registers containing data that is stale or
that belongs to another task or to the kernel.
Knowingly handing around a kernel structure that is incompletely
initialised with user data is a potential source of mistakes,
especially across source file boundaries. To help avoid a repeat
of this issue, this patch adapts the relevant internal API to hand
around the user-accessible subset only: struct user_fpsimd_state.
To avoid future surprises, this patch also converts all uses of
struct fpsimd_state that really only access the user subset, to use
struct user_fpsimd_state. A few missing consts are added to
function prototypes for good measure.
Thanks to Will for spotting the cause of the bug here.
Reported-by: Geert Uytterhoeven <geert@linux-m68k.org>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2017-12-15 18:34:38 +00:00
|
|
|
extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
|
2022-11-15 09:46:33 +00:00
|
|
|
extern void fpsimd_kvm_prepare(void);
|
2014-02-24 14:26:27 +00:00
|
|
|
|
2022-11-15 09:46:40 +00:00
|
|
|
struct cpu_fp_state {
|
|
|
|
|
struct user_fpsimd_state *st;
|
|
|
|
|
void *sve_state;
|
2023-01-16 16:04:36 +00:00
|
|
|
void *sme_state;
|
2022-11-15 09:46:40 +00:00
|
|
|
u64 *svcr;
|
2024-03-06 23:14:48 +00:00
|
|
|
u64 *fpmr;
|
2022-11-15 09:46:40 +00:00
|
|
|
unsigned int sve_vl;
|
|
|
|
|
unsigned int sme_vl;
|
|
|
|
|
enum fp_type *fp_type;
|
|
|
|
|
enum fp_type to_save;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
|
2018-04-06 13:55:59 +00:00
|
|
|
|
2014-05-08 09:20:23 +00:00
|
|
|
extern void fpsimd_flush_task_state(struct task_struct *target);
|
2019-05-21 17:21:38 +00:00
|
|
|
extern void fpsimd_save_and_flush_cpu_state(void);
|
2014-05-08 09:20:23 +00:00
|
|
|
|
2022-04-19 11:22:22 +00:00
|
|
|
static inline bool thread_sm_enabled(struct thread_struct *thread)
|
|
|
|
|
{
|
2022-05-10 16:12:01 +00:00
|
|
|
return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
|
2022-04-19 11:22:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool thread_za_enabled(struct thread_struct *thread)
|
|
|
|
|
{
|
2022-05-10 16:12:01 +00:00
|
|
|
return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
|
2022-04-19 11:22:22 +00:00
|
|
|
}
|
|
|
|
|
|
2021-12-10 18:40:58 +00:00
|
|
|
/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
|
|
|
|
|
#define VL_ARCH_MAX 0x100
|
2017-10-31 15:51:08 +00:00
|
|
|
|
2018-04-12 16:32:35 +00:00
|
|
|
/* Offset of FFR in the SVE register dump */
|
|
|
|
|
static inline size_t sve_ffr_offset(int vl)
|
|
|
|
|
{
|
|
|
|
|
return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void *sve_pffr(struct thread_struct *thread)
|
|
|
|
|
{
|
2022-04-19 11:22:22 +00:00
|
|
|
unsigned int vl;
|
|
|
|
|
|
|
|
|
|
if (system_supports_sme() && thread_sm_enabled(thread))
|
|
|
|
|
vl = thread_get_sme_vl(thread);
|
|
|
|
|
else
|
|
|
|
|
vl = thread_get_sve_vl(thread);
|
|
|
|
|
|
|
|
|
|
return (char *)thread->sve_state + sve_ffr_offset(vl);
|
2018-04-12 16:32:35 +00:00
|
|
|
}
|
|
|
|
|
|
2023-01-16 16:04:44 +00:00
|
|
|
static inline void *thread_zt_state(struct thread_struct *thread)
|
|
|
|
|
{
|
|
|
|
|
/* The ZT register state is stored immediately after the ZA state */
|
|
|
|
|
unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
|
|
|
|
|
return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-19 17:22:09 +00:00
|
|
|
extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
|
2017-10-31 15:51:01 +00:00
|
|
|
extern void sve_load_state(void const *state, u32 const *pfpsr,
|
2021-10-19 17:22:13 +00:00
|
|
|
int restore_ffr);
|
2021-10-19 17:22:09 +00:00
|
|
|
extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
|
2017-10-31 15:51:01 +00:00
|
|
|
extern unsigned int sve_get_vl(void);
|
2021-03-12 19:03:13 +00:00
|
|
|
extern void sve_set_vq(unsigned long vq_minus_1);
|
2022-04-19 11:22:22 +00:00
|
|
|
extern void sme_set_vq(unsigned long vq_minus_1);
|
2023-01-16 16:04:45 +00:00
|
|
|
extern void sme_save_state(void *state, int zt);
|
|
|
|
|
extern void sme_load_state(void const *state, int zt);
|
2018-03-26 14:12:28 +00:00
|
|
|
|
|
|
|
|
struct arm64_cpu_capabilities;
|
arm64: Use a positive cpucap for FP/SIMD
Currently we have a negative cpucap which describes the *absence* of
FP/SIMD rather than *presence* of FP/SIMD. This largely works, but is
somewhat awkward relative to other cpucaps that describe the presence of
a feature, and it would be nicer to have a cpucap which describes the
presence of FP/SIMD:
* This will allow the cpucap to be treated as a standard
ARM64_CPUCAP_SYSTEM_FEATURE, which can be detected with the standard
has_cpuid_feature() function and ARM64_CPUID_FIELDS() description.
* This ensures that the cpucap will only transition from not-present to
present, reducing the risk of unintentional and/or unsafe usage of
FP/SIMD before cpucaps are finalized.
* This will allow using arm64_cpu_capabilities::cpu_enable() to enable
the use of FP/SIMD later, with FP/SIMD being disabled at boot time
otherwise. This will ensure that any unintentional and/or unsafe usage
of FP/SIMD prior to this is trapped, and will ensure that FP/SIMD is
never unintentionally enabled for userspace in mismatched big.LITTLE
systems.
This patch replaces the negative ARM64_HAS_NO_FPSIMD cpucap with a
positive ARM64_HAS_FPSIMD cpucap, making changes as described above.
Note that as FP/SIMD will now be trapped when not supported system-wide,
do_fpsimd_acc() must handle these traps in the same way as for SVE and
SME. The commentary in fpsimd_restore_current_state() is updated to
describe the new scheme.
No users of system_supports_fpsimd() need to know that FP/SIMD is
available prior to alternatives being patched, so this is updated to
use alternative_has_cap_likely() to check for the ARM64_HAS_FPSIMD
cpucap, without generating code to test the system_cpucaps bitmap.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2023-10-16 10:24:36 +00:00
|
|
|
extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
|
2023-10-16 10:24:35 +00:00
|
|
|
extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
|
|
|
|
|
extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
|
|
|
|
|
extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
|
|
|
|
|
extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
|
2024-03-06 23:14:48 +00:00
|
|
|
extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
|
2017-10-31 15:51:01 +00:00
|
|
|
|
2022-04-19 11:22:17 +00:00
|
|
|
extern u64 read_smcr_features(void);
|
2018-04-12 15:47:20 +00:00
|
|
|
|
2018-09-28 13:39:21 +00:00
|
|
|
/*
|
|
|
|
|
* Helpers to translate bit indices in sve_vq_map to VQ values (and
|
|
|
|
|
* vice versa). This allows find_next_bit() to be used to find the
|
|
|
|
|
* _maximum_ VQ not exceeding a certain value.
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned int __vq_to_bit(unsigned int vq)
|
|
|
|
|
{
|
|
|
|
|
return SVE_VQ_MAX - vq;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline unsigned int __bit_to_vq(unsigned int bit)
|
|
|
|
|
{
|
|
|
|
|
return SVE_VQ_MAX - bit;
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-19 17:22:12 +00:00
|
|
|
|
|
|
|
|
struct vl_info {
|
|
|
|
|
enum vec_type type;
|
|
|
|
|
const char *name; /* For display purposes */
|
|
|
|
|
|
|
|
|
|
/* Minimum supported vector length across all CPUs */
|
|
|
|
|
int min_vl;
|
|
|
|
|
|
|
|
|
|
/* Maximum supported vector length across all CPUs */
|
|
|
|
|
int max_vl;
|
|
|
|
|
int max_virtualisable_vl;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Set of available vector lengths,
|
|
|
|
|
* where length vq encoded as bit __vq_to_bit(vq):
|
|
|
|
|
*/
|
|
|
|
|
DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
|
|
|
|
|
|
|
|
|
|
/* Set of vector lengths present on at least one cpu: */
|
|
|
|
|
DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
|
|
|
|
|
};
|
2017-10-31 15:51:08 +00:00
|
|
|
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
|
|
|
|
2022-08-17 18:23:23 +00:00
|
|
|
extern void sve_alloc(struct task_struct *task, bool flush);
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
extern void fpsimd_release_task(struct task_struct *task);
|
arm64/sve: ptrace and ELF coredump support
This patch defines and implements a new regset NT_ARM_SVE, which
describes a thread's SVE register state. This allows a debugger to
manipulate the SVE state, as well as being included in ELF
coredumps for post-mortem debugging.
Because the regset size and layout are dependent on the thread's
current vector length, it is not possible to define a C struct to
describe the regset contents as is done for existing regsets.
Instead, and for the same reasons, NT_ARM_SVE is based on the
freeform variable-layout approach used for the SVE signal frame.
Additionally, to reduce debug overhead when debugging threads that
might or might not have live SVE register state, NT_ARM_SVE may be
presented in one of two different formats: the old struct
user_fpsimd_state format is embedded for describing the state of a
thread with no live SVE state, whereas a new variable-layout
structure is embedded for describing live SVE state. This avoids a
debugger needing to poll NT_PRFPREG in addition to NT_ARM_SVE, and
allows existing userspace code to handle the non-SVE case without
too much modification.
For this to work, NT_ARM_SVE is defined with a fixed-format header
of type struct user_sve_header, which the recipient can use to
figure out the content, size and layout of the reset of the regset.
Accessor macros are defined to allow the vector-length-dependent
parts of the regset to be manipulated.
Signed-off-by: Alan Hayward <alan.hayward@arm.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Alex Bennée <alex.bennee@linaro.org>
Cc: Okamoto Takayuki <tokamoto@jp.fujitsu.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:13 +00:00
|
|
|
extern void fpsimd_sync_to_sve(struct task_struct *task);
|
2022-04-19 11:22:28 +00:00
|
|
|
extern void fpsimd_force_sync_to_sve(struct task_struct *task);
|
arm64/sve: ptrace and ELF coredump support
This patch defines and implements a new regset NT_ARM_SVE, which
describes a thread's SVE register state. This allows a debugger to
manipulate the SVE state, as well as being included in ELF
coredumps for post-mortem debugging.
Because the regset size and layout are dependent on the thread's
current vector length, it is not possible to define a C struct to
describe the regset contents as is done for existing regsets.
Instead, and for the same reasons, NT_ARM_SVE is based on the
freeform variable-layout approach used for the SVE signal frame.
Additionally, to reduce debug overhead when debugging threads that
might or might not have live SVE register state, NT_ARM_SVE may be
presented in one of two different formats: the old struct
user_fpsimd_state format is embedded for describing the state of a
thread with no live SVE state, whereas a new variable-layout
structure is embedded for describing live SVE state. This avoids a
debugger needing to poll NT_PRFPREG in addition to NT_ARM_SVE, and
allows existing userspace code to handle the non-SVE case without
too much modification.
For this to work, NT_ARM_SVE is defined with a fixed-format header
of type struct user_sve_header, which the recipient can use to
figure out the content, size and layout of the reset of the regset.
Accessor macros are defined to allow the vector-length-dependent
parts of the regset to be manipulated.
Signed-off-by: Alan Hayward <alan.hayward@arm.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Alex Bennée <alex.bennee@linaro.org>
Cc: Okamoto Takayuki <tokamoto@jp.fujitsu.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:13 +00:00
|
|
|
extern void sve_sync_to_fpsimd(struct task_struct *task);
|
|
|
|
|
extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
|
|
|
|
|
|
2021-12-10 18:40:58 +00:00
|
|
|
extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
|
2017-10-31 15:51:08 +00:00
|
|
|
unsigned long vl, unsigned long flags);
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
|
2017-10-31 15:51:14 +00:00
|
|
|
extern int sve_set_current_vl(unsigned long arg);
|
|
|
|
|
extern int sve_get_current_vl(void);
|
|
|
|
|
|
2018-07-11 13:56:40 +00:00
|
|
|
static inline void sve_user_disable(void)
|
|
|
|
|
{
|
|
|
|
|
sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sve_user_enable(void)
|
|
|
|
|
{
|
|
|
|
|
sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-18 09:43:03 +00:00
|
|
|
#define sve_cond_update_zcr_vq(val, reg) \
|
|
|
|
|
do { \
|
|
|
|
|
u64 __zcr = read_sysreg_s((reg)); \
|
|
|
|
|
u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
|
|
|
|
|
__new |= (val) & ZCR_ELx_LEN_MASK; \
|
|
|
|
|
if (__zcr != __new) \
|
|
|
|
|
write_sysreg_s(__new, (reg)); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
2017-10-31 15:51:10 +00:00
|
|
|
/*
|
|
|
|
|
* Probing and setup functions.
|
|
|
|
|
* Calls to these functions must be serialised with one another.
|
|
|
|
|
*/
|
2021-10-19 17:22:12 +00:00
|
|
|
enum vec_type;
|
|
|
|
|
|
|
|
|
|
extern void __init vec_init_vq_map(enum vec_type type);
|
|
|
|
|
extern void vec_update_vq_map(enum vec_type type);
|
|
|
|
|
extern int vec_verify_vq_map(enum vec_type type);
|
2017-10-31 15:51:10 +00:00
|
|
|
extern void __init sve_setup(void);
|
|
|
|
|
|
2021-10-19 17:22:12 +00:00
|
|
|
extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
|
|
|
|
|
|
|
|
|
|
static inline void write_vl(enum vec_type type, u64 val)
|
|
|
|
|
{
|
|
|
|
|
u64 tmp;
|
|
|
|
|
|
|
|
|
|
switch (type) {
|
|
|
|
|
#ifdef CONFIG_ARM64_SVE
|
|
|
|
|
case ARM64_VEC_SVE:
|
|
|
|
|
tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
|
|
|
|
|
write_sysreg_s(tmp | val, SYS_ZCR_EL1);
|
|
|
|
|
break;
|
2022-04-19 11:22:17 +00:00
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_ARM64_SME
|
|
|
|
|
case ARM64_VEC_SME:
|
|
|
|
|
tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
|
|
|
|
|
write_sysreg_s(tmp | val, SYS_SMCR_EL1);
|
|
|
|
|
break;
|
2021-10-19 17:22:12 +00:00
|
|
|
#endif
|
|
|
|
|
default:
|
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int vec_max_vl(enum vec_type type)
|
|
|
|
|
{
|
|
|
|
|
return vl_info[type].max_vl;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int vec_max_virtualisable_vl(enum vec_type type)
|
|
|
|
|
{
|
|
|
|
|
return vl_info[type].max_virtualisable_vl;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int sve_max_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return vec_max_vl(ARM64_VEC_SVE);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int sve_max_virtualisable_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return vec_max_virtualisable_vl(ARM64_VEC_SVE);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
|
|
|
|
|
static inline bool vq_available(enum vec_type type, unsigned int vq)
|
|
|
|
|
{
|
|
|
|
|
return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool sve_vq_available(unsigned int vq)
|
|
|
|
|
{
|
|
|
|
|
return vq_available(ARM64_VEC_SVE, vq);
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:22:24 +00:00
|
|
|
size_t sve_state_size(struct task_struct const *task);
|
|
|
|
|
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
#else /* ! CONFIG_ARM64_SVE */
|
|
|
|
|
|
2022-08-17 18:23:23 +00:00
|
|
|
static inline void sve_alloc(struct task_struct *task, bool flush) { }
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
static inline void fpsimd_release_task(struct task_struct *task) { }
|
arm64/sve: ptrace and ELF coredump support
This patch defines and implements a new regset NT_ARM_SVE, which
describes a thread's SVE register state. This allows a debugger to
manipulate the SVE state, as well as being included in ELF
coredumps for post-mortem debugging.
Because the regset size and layout are dependent on the thread's
current vector length, it is not possible to define a C struct to
describe the regset contents as is done for existing regsets.
Instead, and for the same reasons, NT_ARM_SVE is based on the
freeform variable-layout approach used for the SVE signal frame.
Additionally, to reduce debug overhead when debugging threads that
might or might not have live SVE register state, NT_ARM_SVE may be
presented in one of two different formats: the old struct
user_fpsimd_state format is embedded for describing the state of a
thread with no live SVE state, whereas a new variable-layout
structure is embedded for describing live SVE state. This avoids a
debugger needing to poll NT_PRFPREG in addition to NT_ARM_SVE, and
allows existing userspace code to handle the non-SVE case without
too much modification.
For this to work, NT_ARM_SVE is defined with a fixed-format header
of type struct user_sve_header, which the recipient can use to
figure out the content, size and layout of the reset of the regset.
Accessor macros are defined to allow the vector-length-dependent
parts of the regset to be manipulated.
Signed-off-by: Alan Hayward <alan.hayward@arm.com>
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Alex Bennée <alex.bennee@linaro.org>
Cc: Okamoto Takayuki <tokamoto@jp.fujitsu.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:13 +00:00
|
|
|
static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
|
|
|
|
|
static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
|
|
|
|
|
|
2021-10-22 14:16:34 +00:00
|
|
|
static inline int sve_max_virtualisable_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-31 15:51:14 +00:00
|
|
|
static inline int sve_set_current_vl(unsigned long arg)
|
|
|
|
|
{
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int sve_get_current_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
2021-10-19 17:22:12 +00:00
|
|
|
static inline int sve_max_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool sve_vq_available(unsigned int vq) { return false; }
|
|
|
|
|
|
2018-07-11 13:56:40 +00:00
|
|
|
static inline void sve_user_disable(void) { BUILD_BUG(); }
|
|
|
|
|
static inline void sve_user_enable(void) { BUILD_BUG(); }
|
|
|
|
|
|
2021-03-31 09:44:39 +00:00
|
|
|
#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
|
|
|
|
|
|
2021-10-19 17:22:12 +00:00
|
|
|
static inline void vec_init_vq_map(enum vec_type t) { }
|
|
|
|
|
static inline void vec_update_vq_map(enum vec_type t) { }
|
|
|
|
|
static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
|
2017-10-31 15:51:10 +00:00
|
|
|
static inline void sve_setup(void) { }
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
|
2022-04-19 11:22:24 +00:00
|
|
|
static inline size_t sve_state_size(struct task_struct const *task)
|
|
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
#endif /* ! CONFIG_ARM64_SVE */
|
|
|
|
|
|
2022-04-19 11:22:14 +00:00
|
|
|
#ifdef CONFIG_ARM64_SME
|
|
|
|
|
|
2022-04-19 11:22:24 +00:00
|
|
|
static inline void sme_user_disable(void)
|
|
|
|
|
{
|
|
|
|
|
sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sme_user_enable(void)
|
|
|
|
|
{
|
|
|
|
|
sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:22:14 +00:00
|
|
|
static inline void sme_smstart_sm(void)
|
|
|
|
|
{
|
|
|
|
|
asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sme_smstop_sm(void)
|
|
|
|
|
{
|
|
|
|
|
asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void sme_smstop(void)
|
|
|
|
|
{
|
|
|
|
|
asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:22:17 +00:00
|
|
|
extern void __init sme_setup(void);
|
|
|
|
|
|
|
|
|
|
static inline int sme_max_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return vec_max_vl(ARM64_VEC_SME);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int sme_max_virtualisable_vl(void)
|
|
|
|
|
{
|
|
|
|
|
return vec_max_virtualisable_vl(ARM64_VEC_SME);
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-10 11:28:19 +00:00
|
|
|
extern void sme_alloc(struct task_struct *task, bool flush);
|
2022-04-19 11:22:17 +00:00
|
|
|
extern unsigned int sme_get_vl(void);
|
2022-04-19 11:22:19 +00:00
|
|
|
extern int sme_set_current_vl(unsigned long arg);
|
|
|
|
|
extern int sme_get_current_vl(void);
|
2024-02-13 23:06:32 +00:00
|
|
|
extern void sme_suspend_exit(void);
|
2022-04-19 11:22:17 +00:00
|
|
|
|
2022-04-19 11:22:24 +00:00
|
|
|
/*
|
|
|
|
|
* Return how many bytes of memory are required to store the full SME
|
2023-01-16 16:04:36 +00:00
|
|
|
* specific state for task, given task's currently configured vector
|
|
|
|
|
* length.
|
2022-04-19 11:22:24 +00:00
|
|
|
*/
|
2023-01-16 16:04:36 +00:00
|
|
|
static inline size_t sme_state_size(struct task_struct const *task)
|
2022-04-19 11:22:24 +00:00
|
|
|
{
|
|
|
|
|
unsigned int vl = task_get_sme_vl(task);
|
2023-01-16 16:04:36 +00:00
|
|
|
size_t size;
|
2022-04-19 11:22:24 +00:00
|
|
|
|
2023-01-16 16:04:36 +00:00
|
|
|
size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
|
|
|
|
|
|
2023-01-16 16:04:44 +00:00
|
|
|
if (system_supports_sme2())
|
|
|
|
|
size += ZT_SIG_REG_SIZE;
|
|
|
|
|
|
2023-01-16 16:04:36 +00:00
|
|
|
return size;
|
2022-04-19 11:22:24 +00:00
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:22:14 +00:00
|
|
|
#else
|
|
|
|
|
|
2022-04-19 11:22:24 +00:00
|
|
|
static inline void sme_user_disable(void) { BUILD_BUG(); }
|
|
|
|
|
static inline void sme_user_enable(void) { BUILD_BUG(); }
|
|
|
|
|
|
2022-04-19 11:22:14 +00:00
|
|
|
static inline void sme_smstart_sm(void) { }
|
|
|
|
|
static inline void sme_smstop_sm(void) { }
|
|
|
|
|
static inline void sme_smstop(void) { }
|
|
|
|
|
|
2023-08-10 11:28:19 +00:00
|
|
|
static inline void sme_alloc(struct task_struct *task, bool flush) { }
|
2022-04-19 11:22:17 +00:00
|
|
|
static inline void sme_setup(void) { }
|
|
|
|
|
static inline unsigned int sme_get_vl(void) { return 0; }
|
|
|
|
|
static inline int sme_max_vl(void) { return 0; }
|
|
|
|
|
static inline int sme_max_virtualisable_vl(void) { return 0; }
|
2022-04-19 11:22:19 +00:00
|
|
|
static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
|
|
|
|
|
static inline int sme_get_current_vl(void) { return -EINVAL; }
|
2024-02-13 23:06:32 +00:00
|
|
|
static inline void sme_suspend_exit(void) { }
|
2022-04-19 11:22:17 +00:00
|
|
|
|
2023-01-16 16:04:36 +00:00
|
|
|
static inline size_t sme_state_size(struct task_struct const *task)
|
2022-04-19 11:22:24 +00:00
|
|
|
{
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:22:14 +00:00
|
|
|
#endif /* ! CONFIG_ARM64_SME */
|
|
|
|
|
|
2017-08-03 16:23:22 +00:00
|
|
|
/* For use by EFI runtime services calls only */
|
|
|
|
|
extern void __efi_fpsimd_begin(void);
|
|
|
|
|
extern void __efi_fpsimd_end(void);
|
|
|
|
|
|
2012-03-05 11:49:32 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#endif
|