diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 838613ac15b8..1506a22a4fb6 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -29,7 +29,7 @@ VDSO32-$(CONFIG_IA32_EMULATION) := y # files to link into the vdso vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o -vobjs32-y += vdso32/vclock_gettime.o +vobjs32-y += vdso32/vclock_gettime.o vdso32/vgetcpu.o vobjs-$(CONFIG_X86_SGX) += vsgx.o # files to link into kernel @@ -104,6 +104,7 @@ $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO CFLAGS_REMOVE_vclock_gettime.o = -pg CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg CFLAGS_REMOVE_vgetcpu.o = -pg +CFLAGS_REMOVE_vdso32/vgetcpu.o = -pg CFLAGS_REMOVE_vsgx.o = -pg # diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 5264daa8859f..67b3e37576a6 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -179,6 +179,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "/* AUTOMATICALLY GENERATED -- DO NOT EDIT */\n\n"); fprintf(outfile, "#include \n"); + fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "#include \n"); fprintf(outfile, "\n"); @@ -218,5 +219,10 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", required_syms[i].name, (int64_t)syms[i]); } + fprintf(outfile, "};\n\n"); + fprintf(outfile, "static __init int init_%s(void) {\n", image_name); + fprintf(outfile, "\treturn init_vdso_image(&%s);\n", image_name); fprintf(outfile, "};\n"); + fprintf(outfile, "subsys_initcall(init_%s);\n", image_name); + } diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c index 43842fade8fa..3b300a773c7e 100644 --- a/arch/x86/entry/vdso/vdso32-setup.c +++ b/arch/x86/entry/vdso/vdso32-setup.c @@ -51,17 +51,8 @@ __setup("vdso32=", vdso32_setup); __setup_param("vdso=", vdso_setup, vdso32_setup, 0); #endif -int __init sysenter_setup(void) -{ - init_vdso_image(&vdso_image_32); - - return 0; -} - #ifdef CONFIG_X86_64 -subsys_initcall(sysenter_setup); - #ifdef CONFIG_SYSCTL /* Register vsyscall32 into the ABI table */ #include diff --git a/arch/x86/entry/vdso/vdso32/fake_32bit_build.h b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h new file mode 100644 index 000000000000..db1b15f686e3 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/fake_32bit_build.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef CONFIG_X86_64 + +/* + * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel + * configuration + */ +#undef CONFIG_64BIT +#undef CONFIG_X86_64 +#undef CONFIG_COMPAT +#undef CONFIG_PGTABLE_LEVELS +#undef CONFIG_ILLEGAL_POINTER_VALUE +#undef CONFIG_SPARSEMEM_VMEMMAP +#undef CONFIG_NR_CPUS +#undef CONFIG_PARAVIRT_XXL + +#define CONFIG_X86_32 1 +#define CONFIG_PGTABLE_LEVELS 2 +#define CONFIG_PAGE_OFFSET 0 +#define CONFIG_ILLEGAL_POINTER_VALUE 0 +#define CONFIG_NR_CPUS 1 + +#define BUILD_VDSO32_64 + +#endif diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c index 283ed9d00426..86981decfea8 100644 --- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c +++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c @@ -1,29 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 #define BUILD_VDSO32 - -#ifdef CONFIG_X86_64 - -/* - * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel - * configuration - */ -#undef CONFIG_64BIT -#undef CONFIG_X86_64 -#undef CONFIG_COMPAT -#undef CONFIG_PGTABLE_LEVELS -#undef CONFIG_ILLEGAL_POINTER_VALUE -#undef CONFIG_SPARSEMEM_VMEMMAP -#undef CONFIG_NR_CPUS -#undef CONFIG_PARAVIRT_XXL - -#define CONFIG_X86_32 1 -#define CONFIG_PGTABLE_LEVELS 2 -#define CONFIG_PAGE_OFFSET 0 -#define CONFIG_ILLEGAL_POINTER_VALUE 0 -#define CONFIG_NR_CPUS 1 - -#define BUILD_VDSO32_64 - -#endif - +#include "fake_32bit_build.h" #include "../vclock_gettime.c" diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S index c7720995ab1a..8a3be07006bb 100644 --- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S +++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S @@ -28,6 +28,7 @@ VERSION __vdso_time; __vdso_clock_getres; __vdso_clock_gettime64; + __vdso_getcpu; }; LINUX_2.5 { diff --git a/arch/x86/entry/vdso/vdso32/vgetcpu.c b/arch/x86/entry/vdso/vdso32/vgetcpu.c new file mode 100644 index 000000000000..3a9791f5e998 --- /dev/null +++ b/arch/x86/entry/vdso/vdso32/vgetcpu.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "fake_32bit_build.h" +#include "../vgetcpu.c" diff --git a/arch/x86/entry/vdso/vgetcpu.c b/arch/x86/entry/vdso/vgetcpu.c index b88a82bbc359..0a9007c24056 100644 --- a/arch/x86/entry/vdso/vgetcpu.c +++ b/arch/x86/entry/vdso/vgetcpu.c @@ -7,8 +7,7 @@ #include #include -#include -#include +#include notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index b8f3f9b9e53c..2738eb28cb2e 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -44,13 +44,16 @@ unsigned int vclocks_used __read_mostly; unsigned int __read_mostly vdso64_enabled = 1; #endif -void __init init_vdso_image(const struct vdso_image *image) +int __init init_vdso_image(const struct vdso_image *image) { + BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); BUG_ON(image->size % PAGE_SIZE != 0); apply_alternatives((struct alt_instr *)(image->data + image->alt), (struct alt_instr *)(image->data + image->alt + image->alt_len)); + + return 0; } static const struct vm_special_mapping vvar_mapping; @@ -418,18 +421,4 @@ static __init int vdso_setup(char *s) return 1; } __setup("vdso=", vdso_setup); - -static int __init init_vdso(void) -{ - BUILD_BUG_ON(VDSO_CLOCKMODE_MAX >= 32); - - init_vdso_image(&vdso_image_64); - -#ifdef CONFIG_X86_X32_ABI - init_vdso_image(&vdso_image_x32); -#endif - - return 0; -} -subsys_initcall(init_vdso); #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index a77dee6a2bf2..8d73004e4cac 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -542,7 +542,6 @@ enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL}; extern void enable_sep_cpu(void); -extern int sysenter_setup(void); /* Defined in head.S */ diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h index c390a672d560..794f69625780 100644 --- a/arch/x86/include/asm/segment.h +++ b/arch/x86/include/asm/segment.h @@ -96,7 +96,7 @@ * * 26 - ESPFIX small SS * 27 - per-cpu [ offset to per-cpu data area ] - * 28 - unused + * 28 - VDSO getcpu * 29 - unused * 30 - unused * 31 - TSS for double fault handler @@ -119,6 +119,7 @@ #define GDT_ENTRY_ESPFIX_SS 26 #define GDT_ENTRY_PERCPU 27 +#define GDT_ENTRY_CPUNODE 28 #define GDT_ENTRY_DOUBLEFAULT_TSS 31 @@ -159,6 +160,8 @@ # define __KERNEL_PERCPU 0 #endif +#define __CPUNODE_SEG (GDT_ENTRY_CPUNODE*8 + 3) + #else /* 64-bit: */ #include @@ -226,8 +229,6 @@ #define GDT_ENTRY_TLS_ENTRIES 3 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES* 8) -#ifdef CONFIG_X86_64 - /* Bit size and mask of CPU number stored in the per CPU data (and TSC_AUX) */ #define VDSO_CPUNODE_BITS 12 #define VDSO_CPUNODE_MASK 0xfff @@ -265,7 +266,6 @@ static inline void vdso_read_cpunode(unsigned *cpu, unsigned *node) } #endif /* !__ASSEMBLY__ */ -#endif /* CONFIG_X86_64 */ #ifdef __KERNEL__ diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index 2963a2f5dbc4..d7f6592b74a9 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -45,7 +45,7 @@ extern const struct vdso_image vdso_image_x32; extern const struct vdso_image vdso_image_32; #endif -extern void __init init_vdso_image(const struct vdso_image *image); +extern int __init init_vdso_image(const struct vdso_image *image); extern int map_vdso_once(const struct vdso_image *image, unsigned long addr); diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 1936f21ed8cd..4cf6794f9d68 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -318,6 +318,8 @@ u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult) } #define vdso_calc_delta vdso_calc_delta +int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts); + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h index 57b1a7034c64..2cbce97d29ea 100644 --- a/arch/x86/include/asm/vdso/processor.h +++ b/arch/x86/include/asm/vdso/processor.h @@ -18,6 +18,10 @@ static __always_inline void cpu_relax(void) rep_nop(); } +struct getcpu_cache; + +notrace long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); + #endif /* __ASSEMBLY__ */ #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 6a25e93f2a87..a539c8fbe334 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1958,7 +1958,6 @@ void __init identify_boot_cpu(void) if (HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT)) pr_info("CET detected: Indirect Branch Tracking enabled\n"); #ifdef CONFIG_X86_32 - sysenter_setup(); enable_sep_cpu(); #endif cpu_detect_tlb(&boot_cpu_data); @@ -2130,7 +2129,6 @@ static void wait_for_master_cpu(int cpu) #endif } -#ifdef CONFIG_X86_64 static inline void setup_getcpu(int cpu) { unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu)); @@ -2152,6 +2150,7 @@ static inline void setup_getcpu(int cpu) write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S); } +#ifdef CONFIG_X86_64 static inline void ucode_cpu_init(int cpu) { if (cpu) @@ -2171,8 +2170,6 @@ static inline void tss_setup_ist(struct tss_struct *tss) #else /* CONFIG_X86_64 */ -static inline void setup_getcpu(int cpu) { } - static inline void ucode_cpu_init(int cpu) { show_ucode_info_early(); diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c index 5b45e6986aea..47cab972807c 100644 --- a/tools/testing/selftests/x86/test_vsyscall.c +++ b/tools/testing/selftests/x86/test_vsyscall.c @@ -92,11 +92,8 @@ static void init_vdso(void) printf("[WARN]\tfailed to find time in vDSO\n"); vdso_getcpu = (getcpu_t)dlsym(vdso, "__vdso_getcpu"); - if (!vdso_getcpu) { - /* getcpu() was never wired up in the 32-bit vDSO. */ - printf("[%s]\tfailed to find getcpu in vDSO\n", - sizeof(long) == 8 ? "WARN" : "NOTE"); - } + if (!vdso_getcpu) + printf("[WARN]\tfailed to find getcpu in vDSO\n"); } static int init_vsys(void)