2012-03-05 11:49:28 +00:00
|
|
|
/*
|
|
|
|
|
* Based on arch/arm/include/asm/processor.h
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 1995-1999 Russell King
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_PROCESSOR_H
|
|
|
|
|
#define __ASM_PROCESSOR_H
|
|
|
|
|
|
2017-08-31 08:30:50 +00:00
|
|
|
#define TASK_SIZE_64 (UL(1) << VA_BITS)
|
|
|
|
|
|
2018-02-05 15:34:18 +00:00
|
|
|
#define KERNEL_DS UL(-1)
|
|
|
|
|
#define USER_DS (TASK_SIZE_64 - 1)
|
|
|
|
|
|
2017-08-31 08:30:50 +00:00
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
2012-03-05 11:49:28 +00:00
|
|
|
/*
|
|
|
|
|
* Default implementation of macro that returns current
|
|
|
|
|
* instruction pointer ("program counter").
|
|
|
|
|
*/
|
|
|
|
|
#define current_text_addr() ({ __label__ _l; _l: &&_l;})
|
|
|
|
|
|
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
|
|
|
|
|
|
#include <linux/string.h>
|
|
|
|
|
|
2016-02-02 12:46:23 +00:00
|
|
|
#include <asm/alternative.h>
|
2012-03-05 11:49:28 +00:00
|
|
|
#include <asm/fpsimd.h>
|
|
|
|
|
#include <asm/hw_breakpoint.h>
|
2016-02-10 10:07:30 +00:00
|
|
|
#include <asm/lse.h>
|
2015-01-06 00:38:41 +00:00
|
|
|
#include <asm/pgtable-hwdef.h>
|
2012-03-05 11:49:28 +00:00
|
|
|
#include <asm/ptrace.h>
|
|
|
|
|
#include <asm/types.h>
|
|
|
|
|
|
2017-08-31 08:30:50 +00:00
|
|
|
/*
|
|
|
|
|
* TASK_SIZE - the maximum size of a user space task.
|
|
|
|
|
* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
#define TASK_SIZE_32 UL(0x100000000)
|
|
|
|
|
#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
|
|
|
|
|
TASK_SIZE_32 : TASK_SIZE_64)
|
|
|
|
|
#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
|
|
|
|
|
TASK_SIZE_32 : TASK_SIZE_64)
|
|
|
|
|
#else
|
|
|
|
|
#define TASK_SIZE TASK_SIZE_64
|
|
|
|
|
#endif /* CONFIG_COMPAT */
|
|
|
|
|
|
|
|
|
|
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
|
|
|
|
|
|
2012-03-05 11:49:28 +00:00
|
|
|
#define STACK_TOP_MAX TASK_SIZE_64
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
#define AARCH32_VECTORS_BASE 0xffff0000
|
|
|
|
|
#define STACK_TOP (test_thread_flag(TIF_32BIT) ? \
|
|
|
|
|
AARCH32_VECTORS_BASE : STACK_TOP_MAX)
|
|
|
|
|
#else
|
|
|
|
|
#define STACK_TOP STACK_TOP_MAX
|
|
|
|
|
#endif /* CONFIG_COMPAT */
|
2012-11-08 16:00:16 +00:00
|
|
|
|
2015-02-05 18:01:53 +00:00
|
|
|
extern phys_addr_t arm64_dma_phys_limit;
|
|
|
|
|
#define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1)
|
2012-03-05 11:49:28 +00:00
|
|
|
|
|
|
|
|
struct debug_info {
|
2017-03-16 22:10:43 +00:00
|
|
|
#ifdef CONFIG_HAVE_HW_BREAKPOINT
|
2012-03-05 11:49:28 +00:00
|
|
|
/* Have we suspended stepping by a debugger? */
|
|
|
|
|
int suspended_step;
|
|
|
|
|
/* Allow breakpoints and watchpoints to be disabled for this thread. */
|
|
|
|
|
int bps_disabled;
|
|
|
|
|
int wps_disabled;
|
|
|
|
|
/* Hardware breakpoints pinned to this task. */
|
|
|
|
|
struct perf_event *hbp_break[ARM_MAX_BRP];
|
|
|
|
|
struct perf_event *hbp_watch[ARM_MAX_WRP];
|
2017-03-16 22:10:43 +00:00
|
|
|
#endif
|
2012-03-05 11:49:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct cpu_context {
|
|
|
|
|
unsigned long x19;
|
|
|
|
|
unsigned long x20;
|
|
|
|
|
unsigned long x21;
|
|
|
|
|
unsigned long x22;
|
|
|
|
|
unsigned long x23;
|
|
|
|
|
unsigned long x24;
|
|
|
|
|
unsigned long x25;
|
|
|
|
|
unsigned long x26;
|
|
|
|
|
unsigned long x27;
|
|
|
|
|
unsigned long x28;
|
|
|
|
|
unsigned long fp;
|
|
|
|
|
unsigned long sp;
|
|
|
|
|
unsigned long pc;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct thread_struct {
|
|
|
|
|
struct cpu_context cpu_context; /* cpu context */
|
2015-05-27 14:39:40 +00:00
|
|
|
unsigned long tp_value; /* TLS register */
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
unsigned long tp2_value;
|
|
|
|
|
#endif
|
2012-03-05 11:49:28 +00:00
|
|
|
struct fpsimd_state fpsimd_state;
|
arm64/sve: Core task context handling
This patch adds the core support for switching and managing the SVE
architectural state of user tasks.
Calls to the existing FPSIMD low-level save/restore functions are
factored out as new functions task_fpsimd_{save,load}(), since SVE
now dynamically may or may not need to be handled at these points
depending on the kernel configuration, hardware features discovered
at boot, and the runtime state of the task. To make these
decisions as fast as possible, const cpucaps are used where
feasible, via the system_supports_sve() helper.
The SVE registers are only tracked for threads that have explicitly
used SVE, indicated by the new thread flag TIF_SVE. Otherwise, the
FPSIMD view of the architectural state is stored in
thread.fpsimd_state as usual.
When in use, the SVE registers are not stored directly in
thread_struct due to their potentially large and variable size.
Because the task_struct slab allocator must be configured very
early during kernel boot, it is also tricky to configure it
correctly to match the maximum vector length provided by the
hardware, since this depends on examining secondary CPUs as well as
the primary. Instead, a pointer sve_state in thread_struct points
to a dynamically allocated buffer containing the SVE register data,
and code is added to allocate and free this buffer at appropriate
times.
TIF_SVE is set when taking an SVE access trap from userspace, if
suitable hardware support has been detected. This enables SVE for
the thread: a subsequent return to userspace will disable the trap
accordingly. If such a trap is taken without sufficient system-
wide hardware support, SIGILL is sent to the thread instead as if
an undefined instruction had been executed: this may happen if
userspace tries to use SVE in a system where not all CPUs support
it for example.
The kernel will clear TIF_SVE and disable SVE for the thread
whenever an explicit syscall is made by userspace. For backwards
compatibility reasons and conformance with the spirit of the base
AArch64 procedure call standard, the subset of the SVE register
state that aliases the FPSIMD registers is still preserved across a
syscall even if this happens. The remainder of the SVE register
state logically becomes zero at syscall entry, though the actual
zeroing work is currently deferred until the thread next tries to
use SVE, causing another trap to the kernel. This implementation
is suboptimal: in the future, the fastpath case may be optimised
to zero the registers in-place and leave SVE enabled for the task,
where beneficial.
TIF_SVE is also cleared in the following slowpath cases, which are
taken as reasonable hints that the task may no longer use SVE:
* exec
* fork and clone
Code is added to sync data between thread.fpsimd_state and
thread.sve_state whenever enabling/disabling SVE, in a manner
consistent with the SVE architectural programmer's model.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Alex Bennée <alex.bennee@linaro.org>
[will: added #include to fix allnoconfig build]
[will: use enable_daif in do_sve_acc]
Signed-off-by: Will Deacon <will.deacon@arm.com>
2017-10-31 15:51:05 +00:00
|
|
|
void *sve_state; /* SVE registers, if any */
|
|
|
|
|
unsigned int sve_vl; /* SVE vector length */
|
2017-10-31 15:51:06 +00:00
|
|
|
unsigned int sve_vl_onexec; /* SVE vl after next exec */
|
2012-03-05 11:49:28 +00:00
|
|
|
unsigned long fault_address; /* fault info */
|
2014-04-06 22:04:12 +00:00
|
|
|
unsigned long fault_code; /* ESR_EL1 value */
|
2012-03-05 11:49:28 +00:00
|
|
|
struct debug_info debug; /* debugging */
|
|
|
|
|
};
|
|
|
|
|
|
2017-08-16 21:05:09 +00:00
|
|
|
/*
|
|
|
|
|
* Everything usercopied to/from thread_struct is statically-sized, so
|
|
|
|
|
* no hardened usercopy whitelist is needed.
|
|
|
|
|
*/
|
|
|
|
|
static inline void arch_thread_struct_whitelist(unsigned long *offset,
|
|
|
|
|
unsigned long *size)
|
|
|
|
|
{
|
|
|
|
|
*offset = *size = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2015-05-27 14:39:40 +00:00
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
#define task_user_tls(t) \
|
|
|
|
|
({ \
|
|
|
|
|
unsigned long *__tls; \
|
|
|
|
|
if (is_compat_thread(task_thread_info(t))) \
|
|
|
|
|
__tls = &(t)->thread.tp2_value; \
|
|
|
|
|
else \
|
|
|
|
|
__tls = &(t)->thread.tp_value; \
|
|
|
|
|
__tls; \
|
|
|
|
|
})
|
|
|
|
|
#else
|
|
|
|
|
#define task_user_tls(t) (&(t)->thread.tp_value)
|
|
|
|
|
#endif
|
|
|
|
|
|
2017-06-21 15:00:44 +00:00
|
|
|
/* Sync TPIDR_EL0 back to thread_struct for current */
|
|
|
|
|
void tls_preserve_current_state(void);
|
|
|
|
|
|
2012-03-05 11:49:28 +00:00
|
|
|
#define INIT_THREAD { }
|
|
|
|
|
|
|
|
|
|
static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
|
|
|
|
|
{
|
|
|
|
|
memset(regs, 0, sizeof(*regs));
|
2017-08-01 14:35:54 +00:00
|
|
|
forget_syscall(regs);
|
2012-03-05 11:49:28 +00:00
|
|
|
regs->pc = pc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void start_thread(struct pt_regs *regs, unsigned long pc,
|
|
|
|
|
unsigned long sp)
|
|
|
|
|
{
|
|
|
|
|
start_thread_common(regs, pc);
|
|
|
|
|
regs->pstate = PSR_MODE_EL0t;
|
|
|
|
|
regs->sp = sp;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
|
|
|
static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc,
|
|
|
|
|
unsigned long sp)
|
|
|
|
|
{
|
|
|
|
|
start_thread_common(regs, pc);
|
|
|
|
|
regs->pstate = COMPAT_PSR_MODE_USR;
|
|
|
|
|
if (pc & 1)
|
|
|
|
|
regs->pstate |= COMPAT_PSR_T_BIT;
|
2013-10-11 13:52:12 +00:00
|
|
|
|
|
|
|
|
#ifdef __AARCH64EB__
|
|
|
|
|
regs->pstate |= COMPAT_PSR_E_BIT;
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-03-05 11:49:28 +00:00
|
|
|
regs->compat_sp = sp;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* Forward declaration, a strange C thing */
|
|
|
|
|
struct task_struct;
|
|
|
|
|
|
|
|
|
|
/* Free all resources held by a thread. */
|
|
|
|
|
extern void release_thread(struct task_struct *);
|
|
|
|
|
|
|
|
|
|
unsigned long get_wchan(struct task_struct *p);
|
|
|
|
|
|
2015-03-02 19:19:14 +00:00
|
|
|
static inline void cpu_relax(void)
|
|
|
|
|
{
|
|
|
|
|
asm volatile("yield" ::: "memory");
|
|
|
|
|
}
|
|
|
|
|
|
2012-03-05 11:49:28 +00:00
|
|
|
/* Thread switching */
|
|
|
|
|
extern struct task_struct *cpu_switch_to(struct task_struct *prev,
|
|
|
|
|
struct task_struct *next);
|
|
|
|
|
|
|
|
|
|
#define task_pt_regs(p) \
|
arm64: kernel: remove {THREAD,IRQ_STACK}_START_SP
For historical reasons, we leave the top 16 bytes of our task and IRQ
stacks unused, a practice used to ensure that the SP can always be
masked to find the base of the current stack (historically, where
thread_info could be found).
However, this is not necessary, as:
* When an exception is taken from a task stack, we decrement the SP by
S_FRAME_SIZE and stash the exception registers before we compare the
SP against the task stack. In such cases, the SP must be at least
S_FRAME_SIZE below the limit, and can be safely masked to determine
whether the task stack is in use.
* When transitioning to an IRQ stack, we'll place a dummy frame onto the
IRQ stack before enabling asynchronous exceptions, or executing code
we expect to trigger faults. Thus, if an exception is taken from the
IRQ stack, the SP must be at least 16 bytes below the limit.
* We no longer mask the SP to find the thread_info, which is now found
via sp_el0. Note that historically, the offset was critical to ensure
that cpu_switch_to() found the correct stack for new threads that
hadn't yet executed ret_from_fork().
Given that, this initial offset serves no purpose, and can be removed.
This brings us in-line with other architectures (e.g. x86) which do not
rely on this masking.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
[Mark: rebase, kill THREAD_START_SP, commit msg additions]
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Will Deacon <will.deacon@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
2017-07-20 16:15:45 +00:00
|
|
|
((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
|
2012-03-05 11:49:28 +00:00
|
|
|
|
2014-07-10 10:37:40 +00:00
|
|
|
#define KSTK_EIP(tsk) ((unsigned long)task_pt_regs(tsk)->pc)
|
2014-08-29 15:11:10 +00:00
|
|
|
#define KSTK_ESP(tsk) user_stack_pointer(task_pt_regs(tsk))
|
2012-03-05 11:49:28 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Prefetching support
|
|
|
|
|
*/
|
|
|
|
|
#define ARCH_HAS_PREFETCH
|
|
|
|
|
static inline void prefetch(const void *ptr)
|
|
|
|
|
{
|
|
|
|
|
asm volatile("prfm pldl1keep, %a0\n" : : "p" (ptr));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define ARCH_HAS_PREFETCHW
|
|
|
|
|
static inline void prefetchw(const void *ptr)
|
|
|
|
|
{
|
|
|
|
|
asm volatile("prfm pstl1keep, %a0\n" : : "p" (ptr));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define ARCH_HAS_SPINLOCK_PREFETCH
|
2016-02-02 12:46:23 +00:00
|
|
|
static inline void spin_lock_prefetch(const void *ptr)
|
2012-03-05 11:49:28 +00:00
|
|
|
{
|
2016-02-02 12:46:23 +00:00
|
|
|
asm volatile(ARM64_LSE_ATOMIC_INSN(
|
|
|
|
|
"prfm pstl1strm, %a0",
|
|
|
|
|
"nop") : : "p" (ptr));
|
2012-03-05 11:49:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#define HAVE_ARCH_PICK_MMAP_LAYOUT
|
|
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
2016-10-18 10:27:46 +00:00
|
|
|
int cpu_enable_pan(void *__unused);
|
|
|
|
|
int cpu_enable_cache_maint_trap(void *__unused);
|
2018-01-15 19:38:59 +00:00
|
|
|
int cpu_clear_disr(void *__unused);
|
2015-07-22 18:05:54 +00:00
|
|
|
|
2017-10-31 15:51:14 +00:00
|
|
|
/* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
|
|
|
|
|
#define SVE_SET_VL(arg) sve_set_current_vl(arg)
|
|
|
|
|
#define SVE_GET_VL() sve_get_current_vl()
|
|
|
|
|
|
2017-08-31 08:30:50 +00:00
|
|
|
#endif /* __ASSEMBLY__ */
|
2012-03-05 11:49:28 +00:00
|
|
|
#endif /* __ASM_PROCESSOR_H */
|