From e232333be69ee9ccd4281032af0d2416940cb98d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 8 Nov 2021 11:42:20 +0000 Subject: [PATCH 01/13] scripts/sorttable: Unify arm64 & x86 sort functions The format of the arm64 and x86 exception table entries is essentially the same as of commits: 46d28947d9876fc0 ("x86/extable: Rework the exception table mechanics") d6e2cc5647753825 ("arm64: extable: add `type` and `data` fields") Both use a 12-byte entry consisting of two 32-bit relative offsets and 32 bits of (absolute) data, and their sort functions are identical aside from commentary, with arm64 saying: /* Don't touch the fixup type or data */ ... and x86 saying: /* Don't touch the fixup type */ Unify the two behind a common sort_relative_table_with_data() function, retaining the arm64 commentary. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Cc: Ard Biesheuvel Cc: Borislav Petkov Cc: Catalin Marinas Cc: Thomas Gleixner Cc: Will Deacon Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Palmer Dabbelt --- scripts/sorttable.c | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index b7c2ad71f9cf..ca9db62bf766 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -231,7 +231,7 @@ static void sort_relative_table(char *extab_image, int image_size) } } -static void arm64_sort_relative_table(char *extab_image, int image_size) +static void sort_relative_table_with_data(char *extab_image, int image_size) { int i = 0; @@ -259,34 +259,6 @@ static void arm64_sort_relative_table(char *extab_image, int image_size) } } -static void x86_sort_relative_table(char *extab_image, int image_size) -{ - int i = 0; - - while (i < image_size) { - uint32_t *loc = (uint32_t *)(extab_image + i); - - w(r(loc) + i, loc); - w(r(loc + 1) + i + 4, loc + 1); - /* Don't touch the fixup type */ - - i += sizeof(uint32_t) * 3; - } - - qsort(extab_image, image_size / 12, 12, compare_relative_table); - - i = 0; - while (i < image_size) { - uint32_t *loc = (uint32_t *)(extab_image + i); - - w(r(loc) - i, loc); - w(r(loc + 1) - (i + 4), loc + 1); - /* Don't touch the fixup type */ - - i += sizeof(uint32_t) * 3; - } -} - static void s390_sort_relative_table(char *extab_image, int image_size) { int i; @@ -364,15 +336,13 @@ static int do_file(char const *const fname, void *addr) switch (r2(&ehdr->e_machine)) { case EM_386: + case EM_AARCH64: case EM_X86_64: - custom_sort = x86_sort_relative_table; + custom_sort = sort_relative_table_with_data; break; case EM_S390: custom_sort = s390_sort_relative_table; break; - case EM_AARCH64: - custom_sort = arm64_sort_relative_table; - break; case EM_PARISC: case EM_PPC: case EM_PPC64: From ddad0b88d5032e1c4a2a94b46aeadada4e4d8a72 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:22:00 +0800 Subject: [PATCH 02/13] riscv: remove unused __cmpxchg_user() macro This macro is defined but not used, remove it. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/uaccess.h | 75 -------------------------------- 1 file changed, 75 deletions(-) diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index f314ff44c48d..9f9219545e59 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -388,81 +388,6 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) __clear_user(to, n) : n; } -/* - * Atomic compare-and-exchange, but with a fixup for userspace faults. Faults - * will set "err" to -EFAULT, while successful accesses return the previous - * value. - */ -#define __cmpxchg_user(ptr, old, new, err, size, lrb, scb) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - __typeof__(*(ptr)) __old = (old); \ - __typeof__(*(ptr)) __new = (new); \ - __typeof__(*(ptr)) __ret; \ - __typeof__(err) __err = 0; \ - register unsigned int __rc; \ - __enable_user_access(); \ - switch (size) { \ - case 4: \ - __asm__ __volatile__ ( \ - "0:\n" \ - " lr.w" #scb " %[ret], %[ptr]\n" \ - " bne %[ret], %z[old], 1f\n" \ - " sc.w" #lrb " %[rc], %z[new], %[ptr]\n" \ - " bnez %[rc], 0b\n" \ - "1:\n" \ - ".section .fixup,\"ax\"\n" \ - ".balign 4\n" \ - "2:\n" \ - " li %[err], %[efault]\n" \ - " jump 1b, %[rc]\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - ".balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 2b\n" \ - ".previous\n" \ - : [ret] "=&r" (__ret), \ - [rc] "=&r" (__rc), \ - [ptr] "+A" (*__ptr), \ - [err] "=&r" (__err) \ - : [old] "rJ" (__old), \ - [new] "rJ" (__new), \ - [efault] "i" (-EFAULT)); \ - break; \ - case 8: \ - __asm__ __volatile__ ( \ - "0:\n" \ - " lr.d" #scb " %[ret], %[ptr]\n" \ - " bne %[ret], %z[old], 1f\n" \ - " sc.d" #lrb " %[rc], %z[new], %[ptr]\n" \ - " bnez %[rc], 0b\n" \ - "1:\n" \ - ".section .fixup,\"ax\"\n" \ - ".balign 4\n" \ - "2:\n" \ - " li %[err], %[efault]\n" \ - " jump 1b, %[rc]\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - ".balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 2b\n" \ - ".previous\n" \ - : [ret] "=&r" (__ret), \ - [rc] "=&r" (__rc), \ - [ptr] "+A" (*__ptr), \ - [err] "=&r" (__err) \ - : [old] "rJ" (__old), \ - [new] "rJ" (__new), \ - [efault] "i" (-EFAULT)); \ - break; \ - default: \ - BUILD_BUG(); \ - } \ - __disable_user_access(); \ - (err) = __err; \ - __ret; \ -}) - #define HAVE_GET_KERNEL_NOFAULT #define __get_kernel_nofault(dst, src, type, err_label) \ From f8f2ad02ee438b7dbceaac947077eaf77c823646 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:22:27 +0800 Subject: [PATCH 03/13] riscv: consolidate __ex_table construction Consolidate all the __ex_table constuction code with a _ASM_EXTABLE helper. There should be no functional change as a result of this patch. Signed-off-by: Jisheng Zhang Reviewed-by: Kefeng Wang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/futex.h | 12 +++--------- arch/riscv/include/asm/uaccess.h | 30 ++++++++++++------------------ 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 1b00badb9f87..3191574e135c 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -30,10 +30,7 @@ "3: li %[r],%[e] \n" \ " jump 2b,%[t] \n" \ " .previous \n" \ - " .section __ex_table,\"a\" \n" \ - " .balign " RISCV_SZPTR " \n" \ - " " RISCV_PTR " 1b, 3b \n" \ - " .previous \n" \ + _ASM_EXTABLE(1b, 3b) \ : [r] "+r" (ret), [ov] "=&r" (oldval), \ [u] "+m" (*uaddr), [t] "=&r" (tmp) \ : [op] "Jr" (oparg), [e] "i" (-EFAULT) \ @@ -103,11 +100,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, "4: li %[r],%[e] \n" " jump 3b,%[t] \n" " .previous \n" - " .section __ex_table,\"a\" \n" - " .balign " RISCV_SZPTR " \n" - " " RISCV_PTR " 1b, 4b \n" - " " RISCV_PTR " 2b, 4b \n" - " .previous \n" + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), [t] "=&r" (tmp) : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "i" (-EFAULT) : "memory"); diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 9f9219545e59..714cd311d9f1 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -10,6 +10,12 @@ #include /* for TASK_SIZE */ +#define _ASM_EXTABLE(from, to) \ + " .pushsection __ex_table, \"a\"\n" \ + " .balign " RISCV_SZPTR " \n" \ + " " RISCV_PTR "(" #from "), (" #to ")\n" \ + " .popsection\n" + /* * User space memory access functions */ @@ -93,10 +99,7 @@ do { \ " li %1, 0\n" \ " jump 2b, %2\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 3b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 3b) \ : "+r" (err), "=&r" (__x), "=r" (__tmp) \ : "m" (*(ptr)), "i" (-EFAULT)); \ (x) = __x; \ @@ -125,11 +128,8 @@ do { \ " li %2, 0\n" \ " jump 3b, %3\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 4b\n" \ - " " RISCV_PTR " 2b, 4b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ : "+r" (err), "=&r" (__lo), "=r" (__hi), \ "=r" (__tmp) \ : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]), \ @@ -233,10 +233,7 @@ do { \ " li %0, %4\n" \ " jump 2b, %1\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 3b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 3b) \ : "+r" (err), "=r" (__tmp), "=m" (*(ptr)) \ : "rJ" (__x), "i" (-EFAULT)); \ } while (0) @@ -262,11 +259,8 @@ do { \ " li %0, %6\n" \ " jump 3b, %1\n" \ " .previous\n" \ - " .section __ex_table,\"a\"\n" \ - " .balign " RISCV_SZPTR "\n" \ - " " RISCV_PTR " 1b, 4b\n" \ - " " RISCV_PTR " 2b, 4b\n" \ - " .previous" \ + _ASM_EXTABLE(1b, 4b) \ + _ASM_EXTABLE(2b, 4b) \ : "+r" (err), "=r" (__tmp), \ "=m" (__ptr[__LSW]), \ "=m" (__ptr[__MSW]) \ From bb1f85d6046f0db757ac52ed60a5eba5df394819 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:22:51 +0800 Subject: [PATCH 04/13] riscv: switch to relative exception tables Similar as other architectures such as arm64, x86 and so on, use offsets relative to the exception table entry values rather than absolute addresses for both the exception locationand the fixup. However, RISCV label difference will actually produce two relocations, a pair of R_RISCV_ADD32 and R_RISCV_SUB32. Take below simple code for example: $ cat test.S .section .text 1: nop .section __ex_table,"a" .balign 4 .long (1b - .) .previous $ riscv64-linux-gnu-gcc -c test.S $ riscv64-linux-gnu-readelf -r test.o Relocation section '.rela__ex_table' at offset 0x100 contains 2 entries: Offset Info Type Sym. Value Sym. Name + Addend 000000000000 000600000023 R_RISCV_ADD32 0000000000000000 .L1^B1 + 0 000000000000 000500000027 R_RISCV_SUB32 0000000000000000 .L0 + 0 The modpost will complain the R_RISCV_SUB32 relocation, so we need to patch modpost.c to skip this relocation for .rela__ex_table section. After this patch, the __ex_table section size of defconfig vmlinux is reduced from 7072 Bytes to 3536 Bytes. Signed-off-by: Jisheng Zhang Reviewed-by: Kefeng Wang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/Kbuild | 1 - arch/riscv/include/asm/extable.h | 25 +++++++++++++++++++++++++ arch/riscv/include/asm/uaccess.h | 4 ++-- arch/riscv/lib/uaccess.S | 4 ++-- arch/riscv/mm/extable.c | 2 +- scripts/mod/modpost.c | 15 +++++++++++++++ scripts/sorttable.c | 2 +- 7 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 arch/riscv/include/asm/extable.h diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 445ccc97305a..57b86fd9916c 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += early_ioremap.h -generic-y += extable.h generic-y += flat.h generic-y += kvm_para.h generic-y += user.h diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h new file mode 100644 index 000000000000..84760392fc69 --- /dev/null +++ b/arch/riscv/include/asm/extable.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_RISCV_EXTABLE_H +#define _ASM_RISCV_EXTABLE_H + +/* + * The exception table consists of pairs of relative offsets: the first + * is the relative offset to an instruction that is allowed to fault, + * and the second is the relative offset at which the program should + * continue. No registers are modified, so it is entirely up to the + * continuation code to figure out what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + int insn, fixup; +}; + +#define ARCH_HAS_RELATIVE_EXTABLE + +int fixup_exception(struct pt_regs *regs); +#endif diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 714cd311d9f1..0f2c5b9d2e8f 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -12,8 +12,8 @@ #define _ASM_EXTABLE(from, to) \ " .pushsection __ex_table, \"a\"\n" \ - " .balign " RISCV_SZPTR " \n" \ - " " RISCV_PTR "(" #from "), (" #to ")\n" \ + " .balign 4\n" \ + " .long (" #from " - .), (" #to " - .)\n" \ " .popsection\n" /* diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 63bc691cff91..55f80f84e23f 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -7,8 +7,8 @@ 100: \op \reg, \addr .section __ex_table,"a" - .balign RISCV_SZPTR - RISCV_PTR 100b, \lbl + .balign 4 + .long (100b - .), (\lbl - .) .previous .endm diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index ddb7d3b99e89..d8d239c2c1bd 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -28,6 +28,6 @@ int fixup_exception(struct pt_regs *regs) return rv_bpf_fixup_exception(fixup, regs); #endif - regs->epc = fixup->fixup; + regs->epc = (unsigned long)&fixup->fixup + fixup->fixup; return 1; } diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index cb8ab7d91d30..6bfa33217914 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1830,6 +1830,14 @@ static int addend_mips_rel(struct elf_info *elf, Elf_Shdr *sechdr, Elf_Rela *r) return 0; } +#ifndef EM_RISCV +#define EM_RISCV 243 +#endif + +#ifndef R_RISCV_SUB32 +#define R_RISCV_SUB32 39 +#endif + static void section_rela(const char *modname, struct elf_info *elf, Elf_Shdr *sechdr) { @@ -1866,6 +1874,13 @@ static void section_rela(const char *modname, struct elf_info *elf, r_sym = ELF_R_SYM(r.r_info); #endif r.r_addend = TO_NATIVE(rela->r_addend); + switch (elf->hdr->e_machine) { + case EM_RISCV: + if (!strcmp("__ex_table", fromsec) && + ELF_R_TYPE(r.r_info) == R_RISCV_SUB32) + continue; + break; + } sym = elf->symtab_start + r_sym; /* Skip special sections */ if (is_shndx_special(sym->st_shndx)) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index ca9db62bf766..f4a8255036b5 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -346,6 +346,7 @@ static int do_file(char const *const fname, void *addr) case EM_PARISC: case EM_PPC: case EM_PPC64: + case EM_RISCV: custom_sort = sort_relative_table; break; case EM_ARCOMPACT: @@ -353,7 +354,6 @@ static int do_file(char const *const fname, void *addr) case EM_ARM: case EM_MICROBLAZE: case EM_MIPS: - case EM_RISCV: case EM_XTENSA: break; default: From c07935cb3ccf37acc5df079074ba20a720716f7a Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:23:21 +0800 Subject: [PATCH 05/13] riscv: bpf: move rv_bpf_fixup_exception signature to extable.h This is to group riscv related extable related functions signature into one file. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/extable.h | 12 ++++++++++++ arch/riscv/mm/extable.c | 6 ------ arch/riscv/net/bpf_jit_comp64.c | 2 -- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h index 84760392fc69..c48c020fcf4d 100644 --- a/arch/riscv/include/asm/extable.h +++ b/arch/riscv/include/asm/extable.h @@ -22,4 +22,16 @@ struct exception_table_entry { #define ARCH_HAS_RELATIVE_EXTABLE int fixup_exception(struct pt_regs *regs); + +#if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) +int rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); +#else +static inline int +rv_bpf_fixup_exception(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + return 0; +} +#endif + #endif diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index d8d239c2c1bd..cbb0db11b28f 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -11,10 +11,6 @@ #include #include -#if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) -int rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); -#endif - int fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; @@ -23,10 +19,8 @@ int fixup_exception(struct pt_regs *regs) if (!fixup) return 0; -#if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) if (regs->epc >= BPF_JIT_REGION_START && regs->epc < BPF_JIT_REGION_END) return rv_bpf_fixup_exception(fixup, regs); -#endif regs->epc = (unsigned long)&fixup->fixup + fixup->fixup; return 1; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f2a779c7e225..2ca345c7b0bf 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -459,8 +459,6 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) -int rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs); int rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs) { From ef127bca1129d3d15f909f9215b9431a2f67555a Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:24:14 +0800 Subject: [PATCH 06/13] riscv: extable: make fixup_exception() return bool The return values of fixup_exception() and riscv_bpf_fixup_exception() represent a boolean condition rather than an error code, so it's better to return `bool` rather than `int`. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/extable.h | 8 ++++---- arch/riscv/mm/extable.c | 6 +++--- arch/riscv/net/bpf_jit_comp64.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h index c48c020fcf4d..e4374dde02b4 100644 --- a/arch/riscv/include/asm/extable.h +++ b/arch/riscv/include/asm/extable.h @@ -21,16 +21,16 @@ struct exception_table_entry { #define ARCH_HAS_RELATIVE_EXTABLE -int fixup_exception(struct pt_regs *regs); +bool fixup_exception(struct pt_regs *regs); #if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) -int rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); +bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); #else -static inline int +static inline bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs) { - return 0; + return false; } #endif diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index cbb0db11b28f..d41bf38e37e9 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -11,17 +11,17 @@ #include #include -int fixup_exception(struct pt_regs *regs) +bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *fixup; fixup = search_exception_tables(regs->epc); if (!fixup) - return 0; + return false; if (regs->epc >= BPF_JIT_REGION_START && regs->epc < BPF_JIT_REGION_END) return rv_bpf_fixup_exception(fixup, regs); regs->epc = (unsigned long)&fixup->fixup + fixup->fixup; - return 1; + return true; } diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 2ca345c7b0bf..7714081cbb64 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -459,8 +459,8 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) -int rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, + struct pt_regs *regs) { off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); @@ -468,7 +468,7 @@ int rv_bpf_fixup_exception(const struct exception_table_entry *ex, *(unsigned long *)((void *)regs + pt_regmap[regs_offset]) = 0; regs->epc = (unsigned long)&ex->fixup - offset; - return 1; + return true; } /* For accesses to BTF pointers, add an entry to the exception table */ From 4c2e7ce8b9864fa1c5f9fc5ea19af11205730cd6 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:24:42 +0800 Subject: [PATCH 07/13] riscv: extable: use `ex` for `exception_table_entry` The var name "fixup" is a bit confusing, since this is a exception_table_entry. Use "ex" instead to refer to an entire entry. In subsequent patches we'll use `fixup` to refer to the fixup field specifically. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/extable.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index d41bf38e37e9..3c561f1d0115 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -13,15 +13,15 @@ bool fixup_exception(struct pt_regs *regs) { - const struct exception_table_entry *fixup; + const struct exception_table_entry *ex; - fixup = search_exception_tables(regs->epc); - if (!fixup) + ex = search_exception_tables(regs->epc); + if (!ex) return false; if (regs->epc >= BPF_JIT_REGION_START && regs->epc < BPF_JIT_REGION_END) - return rv_bpf_fixup_exception(fixup, regs); + return rv_bpf_fixup_exception(ex, regs); - regs->epc = (unsigned long)&fixup->fixup + fixup->fixup; + regs->epc = (unsigned long)&ex->fixup + ex->fixup; return true; } From 9d504f9aa5c1b76673018da9503e76b351a24b8c Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:25:14 +0800 Subject: [PATCH 08/13] riscv: lib: uaccess: fold fixups into body uaccess functions such __asm_copy_to_user(), __arch_copy_from_user() and __clear_user() place their exception fixups in the `.fixup` section without any clear association with themselves. If we backtrace the fixup code, it will be symbolized as an offset from the nearest prior symbol. Similar as arm64 does, we must move fixups into the body of the functions themselves, after the usual fast-path returns. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/lib/uaccess.S | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 55f80f84e23f..047f517ac780 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -173,6 +173,13 @@ ENTRY(__asm_copy_from_user) csrc CSR_STATUS, t6 li a0, 0 ret + + /* Exception fixup code */ +10: + /* Disable access to user memory */ + csrs CSR_STATUS, t6 + mv a0, t5 + ret ENDPROC(__asm_copy_to_user) ENDPROC(__asm_copy_from_user) EXPORT_SYMBOL(__asm_copy_to_user) @@ -218,19 +225,12 @@ ENTRY(__clear_user) addi a0, a0, 1 bltu a0, a3, 5b j 3b -ENDPROC(__clear_user) -EXPORT_SYMBOL(__clear_user) - .section .fixup,"ax" - .balign 4 - /* Fixup code for __copy_user(10) and __clear_user(11) */ -10: - /* Disable access to user memory */ - csrs CSR_STATUS, t6 - mv a0, t5 - ret + /* Exception fixup code */ 11: + /* Disable access to user memory */ csrs CSR_STATUS, t6 mv a0, a1 ret - .previous +ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) From 6dd10d9166a0c06260e0ac6b1fac454117c8024a Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:25:45 +0800 Subject: [PATCH 09/13] riscv: extable: consolidate definitions This is a riscv port of commit 819771cc2892 ("arm64: extable: consolidate definitions"). Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/asm-extable.h | 33 ++++++++++++++++++++++++++++ arch/riscv/include/asm/futex.h | 1 + arch/riscv/include/asm/uaccess.h | 7 +----- arch/riscv/lib/uaccess.S | 6 ++--- 4 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 arch/riscv/include/asm/asm-extable.h diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h new file mode 100644 index 000000000000..b790c02dbdda --- /dev/null +++ b/arch/riscv/include/asm/asm-extable.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_ASM_EXTABLE_H +#define __ASM_ASM_EXTABLE_H + +#ifdef __ASSEMBLY__ + +#define __ASM_EXTABLE_RAW(insn, fixup) \ + .pushsection __ex_table, "a"; \ + .balign 4; \ + .long ((insn) - .); \ + .long ((fixup) - .); \ + .popsection; + + .macro _asm_extable, insn, fixup + __ASM_EXTABLE_RAW(\insn, \fixup) + .endm + +#else /* __ASSEMBLY__ */ + +#include + +#define __ASM_EXTABLE_RAW(insn, fixup) \ + ".pushsection __ex_table, \"a\"\n" \ + ".balign 4\n" \ + ".long ((" insn ") - .)\n" \ + ".long ((" fixup ") - .)\n" \ + ".popsection\n" + +#define _ASM_EXTABLE(insn, fixup) __ASM_EXTABLE_RAW(#insn, #fixup) + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_ASM_EXTABLE_H */ diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 3191574e135c..2e15e8e89502 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -11,6 +11,7 @@ #include #include #include +#include /* We don't even really need the extable code, but for now keep it simple */ #ifndef CONFIG_MMU diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 0f2c5b9d2e8f..40e6099af488 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -8,14 +8,9 @@ #ifndef _ASM_RISCV_UACCESS_H #define _ASM_RISCV_UACCESS_H +#include #include /* for TASK_SIZE */ -#define _ASM_EXTABLE(from, to) \ - " .pushsection __ex_table, \"a\"\n" \ - " .balign 4\n" \ - " .long (" #from " - .), (" #to " - .)\n" \ - " .popsection\n" - /* * User space memory access functions */ diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index 047f517ac780..8c475f4da308 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -1,15 +1,13 @@ #include #include #include +#include #include .macro fixup op reg addr lbl 100: \op \reg, \addr - .section __ex_table,"a" - .balign 4 - .long (100b - .), (\lbl - .) - .previous + _asm_extable 100b, \lbl .endm ENTRY(__asm_copy_to_user) From 2bf847db0c7437c28b10fba2981b9a7db4b4e0e2 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:26:05 +0800 Subject: [PATCH 10/13] riscv: extable: add `type` and `data` fields This is a riscv port of commit d6e2cc564775 ("arm64: extable: add `type` and `data` fields"). Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/asm-extable.h | 25 +++++++++++++++++-------- arch/riscv/include/asm/extable.h | 17 ++++++++++++++--- arch/riscv/kernel/vmlinux.lds.S | 2 +- arch/riscv/mm/extable.c | 25 +++++++++++++++++++++---- arch/riscv/net/bpf_jit_comp64.c | 5 +++-- scripts/sorttable.c | 2 +- 6 files changed, 57 insertions(+), 19 deletions(-) diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h index b790c02dbdda..1b1f4ffd8d37 100644 --- a/arch/riscv/include/asm/asm-extable.h +++ b/arch/riscv/include/asm/asm-extable.h @@ -2,31 +2,40 @@ #ifndef __ASM_ASM_EXTABLE_H #define __ASM_ASM_EXTABLE_H +#define EX_TYPE_NONE 0 +#define EX_TYPE_FIXUP 1 +#define EX_TYPE_BPF 2 + #ifdef __ASSEMBLY__ -#define __ASM_EXTABLE_RAW(insn, fixup) \ - .pushsection __ex_table, "a"; \ - .balign 4; \ - .long ((insn) - .); \ - .long ((fixup) - .); \ +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ + .pushsection __ex_table, "a"; \ + .balign 4; \ + .long ((insn) - .); \ + .long ((fixup) - .); \ + .short (type); \ + .short (data); \ .popsection; .macro _asm_extable, insn, fixup - __ASM_EXTABLE_RAW(\insn, \fixup) + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0) .endm #else /* __ASSEMBLY__ */ #include -#define __ASM_EXTABLE_RAW(insn, fixup) \ +#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ ".pushsection __ex_table, \"a\"\n" \ ".balign 4\n" \ ".long ((" insn ") - .)\n" \ ".long ((" fixup ") - .)\n" \ + ".short (" type ")\n" \ + ".short (" data ")\n" \ ".popsection\n" -#define _ASM_EXTABLE(insn, fixup) __ASM_EXTABLE_RAW(#insn, #fixup) +#define _ASM_EXTABLE(insn, fixup) \ + __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0") #endif /* __ASSEMBLY__ */ diff --git a/arch/riscv/include/asm/extable.h b/arch/riscv/include/asm/extable.h index e4374dde02b4..512012d193dc 100644 --- a/arch/riscv/include/asm/extable.h +++ b/arch/riscv/include/asm/extable.h @@ -17,18 +17,29 @@ struct exception_table_entry { int insn, fixup; + short type, data; }; #define ARCH_HAS_RELATIVE_EXTABLE +#define swap_ex_entry_fixup(a, b, tmp, delta) \ +do { \ + (a)->fixup = (b)->fixup + (delta); \ + (b)->fixup = (tmp).fixup - (delta); \ + (a)->type = (b)->type; \ + (b)->type = (tmp).type; \ + (a)->data = (b)->data; \ + (b)->data = (tmp).data; \ +} while (0) + bool fixup_exception(struct pt_regs *regs); #if defined(CONFIG_BPF_JIT) && defined(CONFIG_ARCH_RV64I) -bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, struct pt_regs *regs); +bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs); #else static inline bool -rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { return false; } diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 5104f3a871e3..0e5ae851929e 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define RO_EXCEPTION_TABLE_ALIGN 16 +#define RO_EXCEPTION_TABLE_ALIGN 4 #ifdef CONFIG_XIP_KERNEL #include "vmlinux-xip.lds.S" diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 3c561f1d0115..91e52c4bb33a 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -10,6 +10,20 @@ #include #include #include +#include + +static inline unsigned long +get_ex_fixup(const struct exception_table_entry *ex) +{ + return ((unsigned long)&ex->fixup + ex->fixup); +} + +static bool ex_handler_fixup(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + regs->epc = get_ex_fixup(ex); + return true; +} bool fixup_exception(struct pt_regs *regs) { @@ -19,9 +33,12 @@ bool fixup_exception(struct pt_regs *regs) if (!ex) return false; - if (regs->epc >= BPF_JIT_REGION_START && regs->epc < BPF_JIT_REGION_END) - return rv_bpf_fixup_exception(ex, regs); + switch (ex->type) { + case EX_TYPE_FIXUP: + return ex_handler_fixup(ex, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(ex, regs); + } - regs->epc = (unsigned long)&ex->fixup + ex->fixup; - return true; + BUG(); } diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 7714081cbb64..69bab7e28f91 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -459,8 +459,8 @@ static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0) #define BPF_FIXUP_REG_MASK GENMASK(31, 27) -bool rv_bpf_fixup_exception(const struct exception_table_entry *ex, - struct pt_regs *regs) +bool ex_handler_bpf(const struct exception_table_entry *ex, + struct pt_regs *regs) { off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); @@ -514,6 +514,7 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) | FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); + ex->type = EX_TYPE_BPF; ctx->nexentries++; return 0; diff --git a/scripts/sorttable.c b/scripts/sorttable.c index f4a8255036b5..82b162b3941b 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -337,6 +337,7 @@ static int do_file(char const *const fname, void *addr) switch (r2(&ehdr->e_machine)) { case EM_386: case EM_AARCH64: + case EM_RISCV: case EM_X86_64: custom_sort = sort_relative_table_with_data; break; @@ -346,7 +347,6 @@ static int do_file(char const *const fname, void *addr) case EM_PARISC: case EM_PPC: case EM_PPC64: - case EM_RISCV: custom_sort = sort_relative_table; break; case EM_ARCOMPACT: From ff4b8cad3a81b3e55b143c689686134d134e2416 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:26:29 +0800 Subject: [PATCH 11/13] riscv: add gpr-num.h This defines the mapping from ABI names to X registers. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/gpr-num.h | 77 ++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 arch/riscv/include/asm/gpr-num.h diff --git a/arch/riscv/include/asm/gpr-num.h b/arch/riscv/include/asm/gpr-num.h new file mode 100644 index 000000000000..dfee2829fc7c --- /dev/null +++ b/arch/riscv/include/asm/gpr-num.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_GPR_NUM_H +#define __ASM_GPR_NUM_H + +#ifdef __ASSEMBLY__ + .equ .L__gpr_num_zero, 0 + .equ .L__gpr_num_ra, 1 + .equ .L__gpr_num_sp, 2 + .equ .L__gpr_num_gp, 3 + .equ .L__gpr_num_tp, 4 + .equ .L__gpr_num_t0, 5 + .equ .L__gpr_num_t1, 6 + .equ .L__gpr_num_t2, 7 + .equ .L__gpr_num_s0, 8 + .equ .L__gpr_num_s1, 9 + .equ .L__gpr_num_a0, 10 + .equ .L__gpr_num_a1, 11 + .equ .L__gpr_num_a2, 12 + .equ .L__gpr_num_a3, 13 + .equ .L__gpr_num_a4, 14 + .equ .L__gpr_num_a5, 15 + .equ .L__gpr_num_a6, 16 + .equ .L__gpr_num_a7, 17 + .equ .L__gpr_num_s2, 18 + .equ .L__gpr_num_s3, 19 + .equ .L__gpr_num_s4, 20 + .equ .L__gpr_num_s5, 21 + .equ .L__gpr_num_s6, 22 + .equ .L__gpr_num_s7, 23 + .equ .L__gpr_num_s8, 24 + .equ .L__gpr_num_s9, 25 + .equ .L__gpr_num_s10, 26 + .equ .L__gpr_num_s11, 27 + .equ .L__gpr_num_t3, 28 + .equ .L__gpr_num_t4, 29 + .equ .L__gpr_num_t5, 30 + .equ .L__gpr_num_t6, 31 + +#else /* __ASSEMBLY__ */ + +#define __DEFINE_ASM_GPR_NUMS \ +" .equ .L__gpr_num_zero, 0\n" \ +" .equ .L__gpr_num_ra, 1\n" \ +" .equ .L__gpr_num_sp, 2\n" \ +" .equ .L__gpr_num_gp, 3\n" \ +" .equ .L__gpr_num_tp, 4\n" \ +" .equ .L__gpr_num_t0, 5\n" \ +" .equ .L__gpr_num_t1, 6\n" \ +" .equ .L__gpr_num_t2, 7\n" \ +" .equ .L__gpr_num_s0, 8\n" \ +" .equ .L__gpr_num_s1, 9\n" \ +" .equ .L__gpr_num_a0, 10\n" \ +" .equ .L__gpr_num_a1, 11\n" \ +" .equ .L__gpr_num_a2, 12\n" \ +" .equ .L__gpr_num_a3, 13\n" \ +" .equ .L__gpr_num_a4, 14\n" \ +" .equ .L__gpr_num_a5, 15\n" \ +" .equ .L__gpr_num_a6, 16\n" \ +" .equ .L__gpr_num_a7, 17\n" \ +" .equ .L__gpr_num_s2, 18\n" \ +" .equ .L__gpr_num_s3, 19\n" \ +" .equ .L__gpr_num_s4, 20\n" \ +" .equ .L__gpr_num_s5, 21\n" \ +" .equ .L__gpr_num_s6, 22\n" \ +" .equ .L__gpr_num_s7, 23\n" \ +" .equ .L__gpr_num_s8, 24\n" \ +" .equ .L__gpr_num_s9, 25\n" \ +" .equ .L__gpr_num_s10, 26\n" \ +" .equ .L__gpr_num_s11, 27\n" \ +" .equ .L__gpr_num_t3, 28\n" \ +" .equ .L__gpr_num_t4, 29\n" \ +" .equ .L__gpr_num_t5, 30\n" \ +" .equ .L__gpr_num_t6, 31\n" + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_GPR_NUM_H */ From 20802d8d477d5771368fdac8d476285adc713af0 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:26:51 +0800 Subject: [PATCH 12/13] riscv: extable: add a dedicated uaccess handler Inspired by commit 2e77a62cb3a6 ("arm64: extable: add a dedicated uaccess handler"), do similar to riscv to add a dedicated uaccess exception handler to update registers in exception context and subsequently return back into the function which faulted, so we remove the need for fixups specialized to each faulting instruction. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/asm-extable.h | 23 +++++++++ arch/riscv/include/asm/futex.h | 23 +++------ arch/riscv/include/asm/uaccess.h | 75 ++++++++-------------------- arch/riscv/mm/extable.c | 27 ++++++++++ 4 files changed, 78 insertions(+), 70 deletions(-) diff --git a/arch/riscv/include/asm/asm-extable.h b/arch/riscv/include/asm/asm-extable.h index 1b1f4ffd8d37..14be0673f5b5 100644 --- a/arch/riscv/include/asm/asm-extable.h +++ b/arch/riscv/include/asm/asm-extable.h @@ -5,6 +5,7 @@ #define EX_TYPE_NONE 0 #define EX_TYPE_FIXUP 1 #define EX_TYPE_BPF 2 +#define EX_TYPE_UACCESS_ERR_ZERO 3 #ifdef __ASSEMBLY__ @@ -23,7 +24,9 @@ #else /* __ASSEMBLY__ */ +#include #include +#include #define __ASM_EXTABLE_RAW(insn, fixup, type, data) \ ".pushsection __ex_table, \"a\"\n" \ @@ -37,6 +40,26 @@ #define _ASM_EXTABLE(insn, fixup) \ __ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0") +#define EX_DATA_REG_ERR_SHIFT 0 +#define EX_DATA_REG_ERR GENMASK(4, 0) +#define EX_DATA_REG_ZERO_SHIFT 5 +#define EX_DATA_REG_ZERO GENMASK(9, 5) + +#define EX_DATA_REG(reg, gpr) \ + "((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")" + +#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) \ + __DEFINE_ASM_GPR_NUMS \ + __ASM_EXTABLE_RAW(#insn, #fixup, \ + __stringify(EX_TYPE_UACCESS_ERR_ZERO), \ + "(" \ + EX_DATA_REG(ERR, err) " | " \ + EX_DATA_REG(ZERO, zero) \ + ")") + +#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) + #endif /* __ASSEMBLY__ */ #endif /* __ASM_ASM_EXTABLE_H */ diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 2e15e8e89502..fc8130f995c1 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -21,20 +21,14 @@ #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ { \ - uintptr_t tmp; \ __enable_user_access(); \ __asm__ __volatile__ ( \ "1: " insn " \n" \ "2: \n" \ - " .section .fixup,\"ax\" \n" \ - " .balign 4 \n" \ - "3: li %[r],%[e] \n" \ - " jump 2b,%[t] \n" \ - " .previous \n" \ - _ASM_EXTABLE(1b, 3b) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %[r]) \ : [r] "+r" (ret), [ov] "=&r" (oldval), \ - [u] "+m" (*uaddr), [t] "=&r" (tmp) \ - : [op] "Jr" (oparg), [e] "i" (-EFAULT) \ + [u] "+m" (*uaddr) \ + : [op] "Jr" (oparg) \ : "memory"); \ __disable_user_access(); \ } @@ -96,15 +90,10 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, "2: sc.w.aqrl %[t],%z[nv],%[u] \n" " bnez %[t],1b \n" "3: \n" - " .section .fixup,\"ax\" \n" - " .balign 4 \n" - "4: li %[r],%[e] \n" - " jump 3b,%[t] \n" - " .previous \n" - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %[r]) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %[r]) \ : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), [t] "=&r" (tmp) - : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "i" (-EFAULT) + : [ov] "Jr" (oldval), [nv] "Jr" (newval) : "memory"); __disable_user_access(); diff --git a/arch/riscv/include/asm/uaccess.h b/arch/riscv/include/asm/uaccess.h index 40e6099af488..c701a5e57a2b 100644 --- a/arch/riscv/include/asm/uaccess.h +++ b/arch/riscv/include/asm/uaccess.h @@ -81,22 +81,14 @@ static inline int __access_ok(unsigned long addr, unsigned long size) #define __get_user_asm(insn, x, ptr, err) \ do { \ - uintptr_t __tmp; \ __typeof__(x) __x; \ __asm__ __volatile__ ( \ "1:\n" \ - " " insn " %1, %3\n" \ + " " insn " %1, %2\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "3:\n" \ - " li %0, %4\n" \ - " li %1, 0\n" \ - " jump 2b, %2\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "+r" (err), "=&r" (__x), "=r" (__tmp) \ - : "m" (*(ptr)), "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %0, %1) \ + : "+r" (err), "=&r" (__x) \ + : "m" (*(ptr))); \ (x) = __x; \ } while (0) @@ -108,27 +100,18 @@ do { \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u32 __lo, __hi; \ - uintptr_t __tmp; \ __asm__ __volatile__ ( \ "1:\n" \ - " lw %1, %4\n" \ + " lw %1, %3\n" \ "2:\n" \ - " lw %2, %5\n" \ + " lw %2, %4\n" \ "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "4:\n" \ - " li %0, %6\n" \ - " li %1, 0\n" \ - " li %2, 0\n" \ - " jump 3b, %3\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "+r" (err), "=&r" (__lo), "=r" (__hi), \ - "=r" (__tmp) \ - : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]), \ - "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 3b, %0, %1) \ + _ASM_EXTABLE_UACCESS_ERR_ZERO(2b, 3b, %0, %1) \ + : "+r" (err), "=&r" (__lo), "=r" (__hi) \ + : "m" (__ptr[__LSW]), "m" (__ptr[__MSW])); \ + if (err) \ + __hi = 0; \ (x) = (__typeof__(x))((__typeof__((x)-(x)))( \ (((u64)__hi << 32) | __lo))); \ } while (0) @@ -216,21 +199,14 @@ do { \ #define __put_user_asm(insn, x, ptr, err) \ do { \ - uintptr_t __tmp; \ __typeof__(*(ptr)) __x = x; \ __asm__ __volatile__ ( \ "1:\n" \ - " " insn " %z3, %2\n" \ + " " insn " %z2, %1\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "3:\n" \ - " li %0, %4\n" \ - " jump 2b, %1\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "+r" (err), "=r" (__tmp), "=m" (*(ptr)) \ - : "rJ" (__x), "i" (-EFAULT)); \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %0) \ + : "+r" (err), "=m" (*(ptr)) \ + : "rJ" (__x)); \ } while (0) #ifdef CONFIG_64BIT @@ -241,25 +217,18 @@ do { \ do { \ u32 __user *__ptr = (u32 __user *)(ptr); \ u64 __x = (__typeof__((x)-(x)))(x); \ - uintptr_t __tmp; \ __asm__ __volatile__ ( \ "1:\n" \ - " sw %z4, %2\n" \ + " sw %z3, %1\n" \ "2:\n" \ - " sw %z5, %3\n" \ + " sw %z4, %2\n" \ "3:\n" \ - " .section .fixup,\"ax\"\n" \ - " .balign 4\n" \ - "4:\n" \ - " li %0, %6\n" \ - " jump 3b, %1\n" \ - " .previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "+r" (err), "=r" (__tmp), \ + _ASM_EXTABLE_UACCESS_ERR(1b, 3b, %0) \ + _ASM_EXTABLE_UACCESS_ERR(2b, 3b, %0) \ + : "+r" (err), \ "=m" (__ptr[__LSW]), \ "=m" (__ptr[__MSW]) \ - : "rJ" (__x), "rJ" (__x >> 32), "i" (-EFAULT)); \ + : "rJ" (__x), "rJ" (__x >> 32)); \ } while (0) #endif /* CONFIG_64BIT */ diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index 91e52c4bb33a..05978f78579f 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -7,10 +7,12 @@ */ +#include #include #include #include #include +#include static inline unsigned long get_ex_fixup(const struct exception_table_entry *ex) @@ -25,6 +27,29 @@ static bool ex_handler_fixup(const struct exception_table_entry *ex, return true; } +static inline void regs_set_gpr(struct pt_regs *regs, unsigned int offset, + unsigned long val) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return; + + if (!offset) + *(unsigned long *)((unsigned long)regs + offset) = val; +} + +static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex, + struct pt_regs *regs) +{ + int reg_err = FIELD_GET(EX_DATA_REG_ERR, ex->data); + int reg_zero = FIELD_GET(EX_DATA_REG_ZERO, ex->data); + + regs_set_gpr(regs, reg_err, -EFAULT); + regs_set_gpr(regs, reg_zero, 0); + + regs->epc = get_ex_fixup(ex); + return true; +} + bool fixup_exception(struct pt_regs *regs) { const struct exception_table_entry *ex; @@ -38,6 +63,8 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_fixup(ex, regs); case EX_TYPE_BPF: return ex_handler_bpf(ex, regs); + case EX_TYPE_UACCESS_ERR_ZERO: + return ex_handler_uaccess_err_zero(ex, regs); } BUG(); From a2ceb8c4efce97a9392084f45c072b0ec8e36701 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Thu, 18 Nov 2021 19:27:21 +0800 Subject: [PATCH 13/13] riscv: vmlinux.lds.S|vmlinux-xip.lds.S: remove `.fixup` section These are no longer necessary now that we have a more standard extable mechanism. Signed-off-by: Jisheng Zhang Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vmlinux-xip.lds.S | 1 - arch/riscv/kernel/vmlinux.lds.S | 1 - 2 files changed, 2 deletions(-) diff --git a/arch/riscv/kernel/vmlinux-xip.lds.S b/arch/riscv/kernel/vmlinux-xip.lds.S index f5ed08262139..75e0fa8a700a 100644 --- a/arch/riscv/kernel/vmlinux-xip.lds.S +++ b/arch/riscv/kernel/vmlinux-xip.lds.S @@ -45,7 +45,6 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT - *(.fixup) _etext = .; } RO_DATA(L1_CACHE_BYTES) diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 0e5ae851929e..4e6c88aa4d87 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -48,7 +48,6 @@ SECTIONS ENTRY_TEXT IRQENTRY_TEXT SOFTIRQENTRY_TEXT - *(.fixup) _etext = .; }