mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-06 10:57:46 +00:00
ARM: percpu: add SMP_ON_UP support
Permit the use of the TPIDRPRW system register for carrying the per-CPU offset in generic SMP configurations that also target non-SMP capable ARMv6 cores. This uses the SMP_ON_UP code patching framework to turn all TPIDRPRW accesses into reads/writes of entry #0 in the __per_cpu_offset array. While at it, switch over some existing direct TPIDRPRW accesses in asm code to invocations of a new helper that is patched in the same way when necessary. Note that CPU_V6+SMP without SMP_ON_UP results in a kernel that does not boot on v6 CPUs without SMP extensions, so add this dependency to Kconfig as well. Acked-by: Linus Walleij <linus.walleij@linaro.org> Acked-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Tested-by: Marc Zyngier <maz@kernel.org> Tested-by: Vladimir Murzin <vladimir.murzin@arm.com> # ARMv7M
This commit is contained in:
parent
4e918ab13e
commit
7b9896c352
6 changed files with 107 additions and 22 deletions
|
@ -220,9 +220,7 @@ THUMB( fpreg .req r7 )
|
||||||
|
|
||||||
.macro reload_current, t1:req, t2:req
|
.macro reload_current, t1:req, t2:req
|
||||||
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
|
#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO
|
||||||
adr_l \t1, __entry_task @ get __entry_task base address
|
ldr_this_cpu \t1, __entry_task, \t1, \t2
|
||||||
mrc p15, 0, \t2, c13, c0, 4 @ get per-CPU offset
|
|
||||||
ldr \t1, [\t1, \t2] @ load variable
|
|
||||||
mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
|
mcr p15, 0, \t1, c13, c0, 3 @ store in TPIDRURO
|
||||||
#endif
|
#endif
|
||||||
.endm
|
.endm
|
||||||
|
@ -312,6 +310,26 @@ THUMB( fpreg .req r7 )
|
||||||
#define ALT_UP_B(label) b label
|
#define ALT_UP_B(label) b label
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
/*
|
||||||
|
* this_cpu_offset - load the per-CPU offset of this CPU into
|
||||||
|
* register 'rd'
|
||||||
|
*/
|
||||||
|
.macro this_cpu_offset, rd:req
|
||||||
|
#ifdef CONFIG_SMP
|
||||||
|
ALT_SMP(mrc p15, 0, \rd, c13, c0, 4)
|
||||||
|
#ifdef CONFIG_CPU_V6
|
||||||
|
ALT_UP_B(.L1_\@)
|
||||||
|
.L0_\@:
|
||||||
|
.subsection 1
|
||||||
|
.L1_\@: ldr_va \rd, __per_cpu_offset
|
||||||
|
b .L0_\@
|
||||||
|
.previous
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
mov \rd, #0
|
||||||
|
#endif
|
||||||
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Instruction barrier
|
* Instruction barrier
|
||||||
*/
|
*/
|
||||||
|
@ -648,6 +666,41 @@ THUMB( orr \reg , \reg , #PSR_T_BIT )
|
||||||
__ldst_va str, \rn, \tmp, \sym, \cond
|
__ldst_va str, \rn, \tmp, \sym, \cond
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
|
/*
|
||||||
|
* ldr_this_cpu_armv6 - Load a 32-bit word from the per-CPU variable 'sym',
|
||||||
|
* without using a temp register. Supported in ARM mode
|
||||||
|
* only.
|
||||||
|
*/
|
||||||
|
.macro ldr_this_cpu_armv6, rd:req, sym:req
|
||||||
|
this_cpu_offset \rd
|
||||||
|
.globl \sym
|
||||||
|
.reloc .L0_\@, R_ARM_ALU_PC_G0_NC, \sym
|
||||||
|
.reloc .L1_\@, R_ARM_ALU_PC_G1_NC, \sym
|
||||||
|
.reloc .L2_\@, R_ARM_LDR_PC_G2, \sym
|
||||||
|
add \rd, \rd, pc
|
||||||
|
.L0_\@: sub \rd, \rd, #4
|
||||||
|
.L1_\@: sub \rd, \rd, #0
|
||||||
|
.L2_\@: ldr \rd, [\rd, #4]
|
||||||
|
.endm
|
||||||
|
|
||||||
|
/*
|
||||||
|
* ldr_this_cpu - Load a 32-bit word from the per-CPU variable 'sym'
|
||||||
|
* into register 'rd', which may be the stack pointer,
|
||||||
|
* using 't1' and 't2' as general temp registers. These
|
||||||
|
* are permitted to overlap with 'rd' if != sp
|
||||||
|
*/
|
||||||
|
.macro ldr_this_cpu, rd:req, sym:req, t1:req, t2:req
|
||||||
|
#if __LINUX_ARM_ARCH__ >= 7 || \
|
||||||
|
(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) || \
|
||||||
|
(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000)
|
||||||
|
this_cpu_offset \t1
|
||||||
|
mov_l \t2, \sym
|
||||||
|
ldr \rd, [\t1, \t2]
|
||||||
|
#else
|
||||||
|
ldr_this_cpu_armv6 \rd, \sym
|
||||||
|
#endif
|
||||||
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* rev_l - byte-swap a 32-bit value
|
* rev_l - byte-swap a 32-bit value
|
||||||
*
|
*
|
||||||
|
|
|
@ -2,6 +2,30 @@
|
||||||
#ifndef __ASM_ARM_INSN_H
|
#ifndef __ASM_ARM_INSN_H
|
||||||
#define __ASM_ARM_INSN_H
|
#define __ASM_ARM_INSN_H
|
||||||
|
|
||||||
|
#include <linux/types.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Avoid a literal load by emitting a sequence of ADD/LDR instructions with the
|
||||||
|
* appropriate relocations. The combined sequence has a range of -/+ 256 MiB,
|
||||||
|
* which should be sufficient for the core kernel as well as modules loaded
|
||||||
|
* into the module region. (Not supported by LLD before release 14)
|
||||||
|
*/
|
||||||
|
#if !(defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) && \
|
||||||
|
!(defined(CONFIG_LD_IS_LLD) && CONFIG_LLD_VERSION < 140000)
|
||||||
|
#define LOAD_SYM_ARMV6(reg, sym) \
|
||||||
|
" .globl " #sym " \n\t" \
|
||||||
|
" .reloc 10f, R_ARM_ALU_PC_G0_NC, " #sym " \n\t" \
|
||||||
|
" .reloc 11f, R_ARM_ALU_PC_G1_NC, " #sym " \n\t" \
|
||||||
|
" .reloc 12f, R_ARM_LDR_PC_G2, " #sym " \n\t" \
|
||||||
|
"10: sub " #reg ", pc, #8 \n\t" \
|
||||||
|
"11: sub " #reg ", " #reg ", #4 \n\t" \
|
||||||
|
"12: ldr " #reg ", [" #reg ", #0] \n\t"
|
||||||
|
#else
|
||||||
|
#define LOAD_SYM_ARMV6(reg, sym) \
|
||||||
|
" ldr " #reg ", =" #sym " \n\t" \
|
||||||
|
" ldr " #reg ", [" #reg "] \n\t"
|
||||||
|
#endif
|
||||||
|
|
||||||
static inline unsigned long
|
static inline unsigned long
|
||||||
arm_gen_nop(void)
|
arm_gen_nop(void)
|
||||||
{
|
{
|
||||||
|
|
|
@ -5,15 +5,22 @@
|
||||||
#ifndef _ASM_ARM_PERCPU_H_
|
#ifndef _ASM_ARM_PERCPU_H_
|
||||||
#define _ASM_ARM_PERCPU_H_
|
#define _ASM_ARM_PERCPU_H_
|
||||||
|
|
||||||
|
#include <asm/insn.h>
|
||||||
|
|
||||||
register unsigned long current_stack_pointer asm ("sp");
|
register unsigned long current_stack_pointer asm ("sp");
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Same as asm-generic/percpu.h, except that we store the per cpu offset
|
* Same as asm-generic/percpu.h, except that we store the per cpu offset
|
||||||
* in the TPIDRPRW. TPIDRPRW only exists on V6K and V7
|
* in the TPIDRPRW. TPIDRPRW only exists on V6K and V7
|
||||||
*/
|
*/
|
||||||
#if defined(CONFIG_SMP) && !defined(CONFIG_CPU_V6)
|
#ifdef CONFIG_SMP
|
||||||
static inline void set_my_cpu_offset(unsigned long off)
|
static inline void set_my_cpu_offset(unsigned long off)
|
||||||
{
|
{
|
||||||
|
extern unsigned int smp_on_up;
|
||||||
|
|
||||||
|
if (IS_ENABLED(CONFIG_CPU_V6) && !smp_on_up)
|
||||||
|
return;
|
||||||
|
|
||||||
/* Set TPIDRPRW */
|
/* Set TPIDRPRW */
|
||||||
asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory");
|
asm volatile("mcr p15, 0, %0, c13, c0, 4" : : "r" (off) : "memory");
|
||||||
}
|
}
|
||||||
|
@ -27,7 +34,19 @@ static inline unsigned long __my_cpu_offset(void)
|
||||||
* We want to allow caching the value, so avoid using volatile and
|
* We want to allow caching the value, so avoid using volatile and
|
||||||
* instead use a fake stack read to hazard against barrier().
|
* instead use a fake stack read to hazard against barrier().
|
||||||
*/
|
*/
|
||||||
asm("mrc p15, 0, %0, c13, c0, 4" : "=r" (off)
|
asm("0: mrc p15, 0, %0, c13, c0, 4 \n\t"
|
||||||
|
#ifdef CONFIG_CPU_V6
|
||||||
|
"1: \n\t"
|
||||||
|
" .subsection 1 \n\t"
|
||||||
|
"2: " LOAD_SYM_ARMV6(%0, __per_cpu_offset) " \n\t"
|
||||||
|
" b 1b \n\t"
|
||||||
|
" .previous \n\t"
|
||||||
|
" .pushsection \".alt.smp.init\", \"a\" \n\t"
|
||||||
|
" .long 0b - . \n\t"
|
||||||
|
" b . + (2b - 0b) \n\t"
|
||||||
|
" .popsection \n\t"
|
||||||
|
#endif
|
||||||
|
: "=r" (off)
|
||||||
: "Q" (*(const unsigned long *)current_stack_pointer));
|
: "Q" (*(const unsigned long *)current_stack_pointer));
|
||||||
|
|
||||||
return off;
|
return off;
|
||||||
|
|
|
@ -35,15 +35,14 @@
|
||||||
.macro irq_handler, from_user:req
|
.macro irq_handler, from_user:req
|
||||||
mov r0, sp
|
mov r0, sp
|
||||||
#ifdef CONFIG_IRQSTACKS
|
#ifdef CONFIG_IRQSTACKS
|
||||||
mov_l r2, irq_stack_ptr @ Take base address
|
|
||||||
mrc p15, 0, r3, c13, c0, 4 @ Get CPU offset
|
|
||||||
#ifdef CONFIG_UNWINDER_ARM
|
#ifdef CONFIG_UNWINDER_ARM
|
||||||
mov fpreg, sp @ Preserve original SP
|
mov fpreg, sp @ Preserve original SP
|
||||||
#else
|
#else
|
||||||
mov r8, fp @ Preserve original FP
|
mov r8, fp @ Preserve original FP
|
||||||
mov r9, sp @ Preserve original SP
|
mov r9, sp @ Preserve original SP
|
||||||
#endif
|
#endif
|
||||||
ldr sp, [r2, r3] @ Load SP from per-CPU var
|
ldr_this_cpu sp, irq_stack_ptr, r2, r3
|
||||||
|
|
||||||
.if \from_user == 0
|
.if \from_user == 0
|
||||||
UNWIND( .setfp fpreg, sp )
|
UNWIND( .setfp fpreg, sp )
|
||||||
@
|
@
|
||||||
|
@ -876,16 +875,7 @@ __bad_stack:
|
||||||
THUMB( bx pc )
|
THUMB( bx pc )
|
||||||
THUMB( nop )
|
THUMB( nop )
|
||||||
THUMB( .arm )
|
THUMB( .arm )
|
||||||
mrc p15, 0, ip, c13, c0, 4 @ Get per-CPU offset
|
ldr_this_cpu_armv6 ip, overflow_stack_ptr
|
||||||
|
|
||||||
.globl overflow_stack_ptr
|
|
||||||
.reloc 0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr
|
|
||||||
.reloc 1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr
|
|
||||||
.reloc 2f, R_ARM_LDR_PC_G2, overflow_stack_ptr
|
|
||||||
add ip, ip, pc
|
|
||||||
0: add ip, ip, #-4
|
|
||||||
1: add ip, ip, #0
|
|
||||||
2: ldr ip, [ip, #4]
|
|
||||||
|
|
||||||
str sp, [ip, #-4]! @ Preserve original SP value
|
str sp, [ip, #-4]! @ Preserve original SP value
|
||||||
mov sp, ip @ Switch to overflow stack
|
mov sp, ip @ Switch to overflow stack
|
||||||
|
|
|
@ -71,9 +71,7 @@ ENTRY(__cpu_suspend)
|
||||||
@ Run the suspend code from the overflow stack so we don't have to rely
|
@ Run the suspend code from the overflow stack so we don't have to rely
|
||||||
@ on vmalloc-to-phys conversions anywhere in the arch suspend code.
|
@ on vmalloc-to-phys conversions anywhere in the arch suspend code.
|
||||||
@ The original SP value captured in R5 will be restored on the way out.
|
@ The original SP value captured in R5 will be restored on the way out.
|
||||||
mov_l r6, overflow_stack_ptr @ Base pointer
|
ldr_this_cpu sp, overflow_stack_ptr, r6, r7
|
||||||
mrc p15, 0, r7, c13, c0, 4 @ Get per-CPU offset
|
|
||||||
ldr sp, [r6, r7] @ Address of this CPU's overflow stack
|
|
||||||
#endif
|
#endif
|
||||||
add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn
|
add r4, r4, #12 @ Space for pgd, virt sp, phys resume fn
|
||||||
sub sp, sp, r4 @ allocate CPU state on stack
|
sub sp, sp, r4 @ allocate CPU state on stack
|
||||||
|
|
|
@ -386,6 +386,7 @@ config CPU_V6
|
||||||
select CPU_PABRT_V6
|
select CPU_PABRT_V6
|
||||||
select CPU_THUMB_CAPABLE
|
select CPU_THUMB_CAPABLE
|
||||||
select CPU_TLB_V6 if MMU
|
select CPU_TLB_V6 if MMU
|
||||||
|
select SMP_ON_UP if SMP
|
||||||
|
|
||||||
# ARMv6k
|
# ARMv6k
|
||||||
config CPU_V6K
|
config CPU_V6K
|
||||||
|
|
Loading…
Reference in a new issue