mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 21:57:43 +00:00
Merge branch 'for-next/extable' into for-next/core
* for-next/extable: arm64: extable: cleanup redundant extable type EX_TYPE_FIXUP arm64: extable: move _cond_extable to _cond_uaccess_extable arm64: extable: make uaaccess helper use extable type EX_TYPE_UACCESS_ERR_ZERO arm64: asm-extable: add asm uacess helpers arm64: asm-extable: move data fields arm64: extable: add new extable type EX_TYPE_KACCESS_ERR_ZERO support
This commit is contained in:
commit
ee8b00a956
5 changed files with 111 additions and 88 deletions
|
@ -2,12 +2,27 @@
|
||||||
#ifndef __ASM_ASM_EXTABLE_H
|
#ifndef __ASM_ASM_EXTABLE_H
|
||||||
#define __ASM_ASM_EXTABLE_H
|
#define __ASM_ASM_EXTABLE_H
|
||||||
|
|
||||||
|
#include <linux/bits.h>
|
||||||
|
#include <asm/gpr-num.h>
|
||||||
|
|
||||||
#define EX_TYPE_NONE 0
|
#define EX_TYPE_NONE 0
|
||||||
#define EX_TYPE_FIXUP 1
|
#define EX_TYPE_BPF 1
|
||||||
#define EX_TYPE_BPF 2
|
#define EX_TYPE_UACCESS_ERR_ZERO 2
|
||||||
#define EX_TYPE_UACCESS_ERR_ZERO 3
|
#define EX_TYPE_KACCESS_ERR_ZERO 3
|
||||||
#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
|
#define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4
|
||||||
|
|
||||||
|
/* Data fields for EX_TYPE_UACCESS_ERR_ZERO */
|
||||||
|
#define EX_DATA_REG_ERR_SHIFT 0
|
||||||
|
#define EX_DATA_REG_ERR GENMASK(4, 0)
|
||||||
|
#define EX_DATA_REG_ZERO_SHIFT 5
|
||||||
|
#define EX_DATA_REG_ZERO GENMASK(9, 5)
|
||||||
|
|
||||||
|
/* Data fields for EX_TYPE_LOAD_UNALIGNED_ZEROPAD */
|
||||||
|
#define EX_DATA_REG_DATA_SHIFT 0
|
||||||
|
#define EX_DATA_REG_DATA GENMASK(4, 0)
|
||||||
|
#define EX_DATA_REG_ADDR_SHIFT 5
|
||||||
|
#define EX_DATA_REG_ADDR GENMASK(9, 5)
|
||||||
|
|
||||||
#ifdef __ASSEMBLY__
|
#ifdef __ASSEMBLY__
|
||||||
|
|
||||||
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
|
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
|
||||||
|
@ -19,31 +34,45 @@
|
||||||
.short (data); \
|
.short (data); \
|
||||||
.popsection;
|
.popsection;
|
||||||
|
|
||||||
|
#define EX_DATA_REG(reg, gpr) \
|
||||||
|
(.L__gpr_num_##gpr << EX_DATA_REG_##reg##_SHIFT)
|
||||||
|
|
||||||
|
#define _ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, zero) \
|
||||||
|
__ASM_EXTABLE_RAW(insn, fixup, \
|
||||||
|
EX_TYPE_UACCESS_ERR_ZERO, \
|
||||||
|
( \
|
||||||
|
EX_DATA_REG(ERR, err) | \
|
||||||
|
EX_DATA_REG(ZERO, zero) \
|
||||||
|
))
|
||||||
|
|
||||||
|
#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \
|
||||||
|
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)
|
||||||
|
|
||||||
|
#define _ASM_EXTABLE_UACCESS(insn, fixup) \
|
||||||
|
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Create an exception table entry for `insn`, which will branch to `fixup`
|
* Create an exception table entry for uaccess `insn`, which will branch to `fixup`
|
||||||
* when an unhandled fault is taken.
|
* when an unhandled fault is taken.
|
||||||
*/
|
*/
|
||||||
.macro _asm_extable, insn, fixup
|
.macro _asm_extable_uaccess, insn, fixup
|
||||||
__ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_FIXUP, 0)
|
_ASM_EXTABLE_UACCESS(\insn, \fixup)
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
|
* Create an exception table entry for `insn` if `fixup` is provided. Otherwise
|
||||||
* do nothing.
|
* do nothing.
|
||||||
*/
|
*/
|
||||||
.macro _cond_extable, insn, fixup
|
.macro _cond_uaccess_extable, insn, fixup
|
||||||
.ifnc \fixup,
|
.ifnc \fixup,
|
||||||
_asm_extable \insn, \fixup
|
_asm_extable_uaccess \insn, \fixup
|
||||||
.endif
|
.endif
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
#else /* __ASSEMBLY__ */
|
#else /* __ASSEMBLY__ */
|
||||||
|
|
||||||
#include <linux/bits.h>
|
|
||||||
#include <linux/stringify.h>
|
#include <linux/stringify.h>
|
||||||
|
|
||||||
#include <asm/gpr-num.h>
|
|
||||||
|
|
||||||
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
|
#define __ASM_EXTABLE_RAW(insn, fixup, type, data) \
|
||||||
".pushsection __ex_table, \"a\"\n" \
|
".pushsection __ex_table, \"a\"\n" \
|
||||||
".align 2\n" \
|
".align 2\n" \
|
||||||
|
@ -53,14 +82,6 @@
|
||||||
".short (" data ")\n" \
|
".short (" data ")\n" \
|
||||||
".popsection\n"
|
".popsection\n"
|
||||||
|
|
||||||
#define _ASM_EXTABLE(insn, fixup) \
|
|
||||||
__ASM_EXTABLE_RAW(#insn, #fixup, __stringify(EX_TYPE_FIXUP), "0")
|
|
||||||
|
|
||||||
#define EX_DATA_REG_ERR_SHIFT 0
|
|
||||||
#define EX_DATA_REG_ERR GENMASK(4, 0)
|
|
||||||
#define EX_DATA_REG_ZERO_SHIFT 5
|
|
||||||
#define EX_DATA_REG_ZERO GENMASK(9, 5)
|
|
||||||
|
|
||||||
#define EX_DATA_REG(reg, gpr) \
|
#define EX_DATA_REG(reg, gpr) \
|
||||||
"((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"
|
"((.L__gpr_num_" #gpr ") << " __stringify(EX_DATA_REG_##reg##_SHIFT) ")"
|
||||||
|
|
||||||
|
@ -73,13 +94,23 @@
|
||||||
EX_DATA_REG(ZERO, zero) \
|
EX_DATA_REG(ZERO, zero) \
|
||||||
")")
|
")")
|
||||||
|
|
||||||
|
#define _ASM_EXTABLE_KACCESS_ERR_ZERO(insn, fixup, err, zero) \
|
||||||
|
__DEFINE_ASM_GPR_NUMS \
|
||||||
|
__ASM_EXTABLE_RAW(#insn, #fixup, \
|
||||||
|
__stringify(EX_TYPE_KACCESS_ERR_ZERO), \
|
||||||
|
"(" \
|
||||||
|
EX_DATA_REG(ERR, err) " | " \
|
||||||
|
EX_DATA_REG(ZERO, zero) \
|
||||||
|
")")
|
||||||
|
|
||||||
#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \
|
#define _ASM_EXTABLE_UACCESS_ERR(insn, fixup, err) \
|
||||||
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)
|
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, err, wzr)
|
||||||
|
|
||||||
#define EX_DATA_REG_DATA_SHIFT 0
|
#define _ASM_EXTABLE_UACCESS(insn, fixup) \
|
||||||
#define EX_DATA_REG_DATA GENMASK(4, 0)
|
_ASM_EXTABLE_UACCESS_ERR_ZERO(insn, fixup, wzr, wzr)
|
||||||
#define EX_DATA_REG_ADDR_SHIFT 5
|
|
||||||
#define EX_DATA_REG_ADDR GENMASK(9, 5)
|
#define _ASM_EXTABLE_KACCESS_ERR(insn, fixup, err) \
|
||||||
|
_ASM_EXTABLE_KACCESS_ERR_ZERO(insn, fixup, err, wzr)
|
||||||
|
|
||||||
#define _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(insn, fixup, data, addr) \
|
#define _ASM_EXTABLE_LOAD_UNALIGNED_ZEROPAD(insn, fixup, data, addr) \
|
||||||
__DEFINE_ASM_GPR_NUMS \
|
__DEFINE_ASM_GPR_NUMS \
|
||||||
|
|
|
@ -61,7 +61,7 @@ alternative_else_nop_endif
|
||||||
|
|
||||||
#define USER(l, x...) \
|
#define USER(l, x...) \
|
||||||
9999: x; \
|
9999: x; \
|
||||||
_asm_extable 9999b, l
|
_asm_extable_uaccess 9999b, l
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Generate the assembly for LDTR/STTR with exception table entries.
|
* Generate the assembly for LDTR/STTR with exception table entries.
|
||||||
|
@ -73,8 +73,8 @@ alternative_else_nop_endif
|
||||||
8889: ldtr \reg2, [\addr, #8];
|
8889: ldtr \reg2, [\addr, #8];
|
||||||
add \addr, \addr, \post_inc;
|
add \addr, \addr, \post_inc;
|
||||||
|
|
||||||
_asm_extable 8888b,\l;
|
_asm_extable_uaccess 8888b, \l;
|
||||||
_asm_extable 8889b,\l;
|
_asm_extable_uaccess 8889b, \l;
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro user_stp l, reg1, reg2, addr, post_inc
|
.macro user_stp l, reg1, reg2, addr, post_inc
|
||||||
|
@ -82,14 +82,14 @@ alternative_else_nop_endif
|
||||||
8889: sttr \reg2, [\addr, #8];
|
8889: sttr \reg2, [\addr, #8];
|
||||||
add \addr, \addr, \post_inc;
|
add \addr, \addr, \post_inc;
|
||||||
|
|
||||||
_asm_extable 8888b,\l;
|
_asm_extable_uaccess 8888b,\l;
|
||||||
_asm_extable 8889b,\l;
|
_asm_extable_uaccess 8889b,\l;
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
.macro user_ldst l, inst, reg, addr, post_inc
|
.macro user_ldst l, inst, reg, addr, post_inc
|
||||||
8888: \inst \reg, [\addr];
|
8888: \inst \reg, [\addr];
|
||||||
add \addr, \addr, \post_inc;
|
add \addr, \addr, \post_inc;
|
||||||
|
|
||||||
_asm_extable 8888b,\l;
|
_asm_extable_uaccess 8888b, \l;
|
||||||
.endm
|
.endm
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -423,7 +423,7 @@ alternative_endif
|
||||||
b.lo .Ldcache_op\@
|
b.lo .Ldcache_op\@
|
||||||
dsb \domain
|
dsb \domain
|
||||||
|
|
||||||
_cond_extable .Ldcache_op\@, \fixup
|
_cond_uaccess_extable .Ldcache_op\@, \fixup
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -462,7 +462,7 @@ alternative_endif
|
||||||
dsb ish
|
dsb ish
|
||||||
isb
|
isb
|
||||||
|
|
||||||
_cond_extable .Licache_op\@, \fixup
|
_cond_uaccess_extable .Licache_op\@, \fixup
|
||||||
.endm
|
.endm
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -232,34 +232,34 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
|
||||||
* The "__xxx_error" versions set the third argument to -EFAULT if an error
|
* The "__xxx_error" versions set the third argument to -EFAULT if an error
|
||||||
* occurs, and leave it unchanged on success.
|
* occurs, and leave it unchanged on success.
|
||||||
*/
|
*/
|
||||||
#define __get_mem_asm(load, reg, x, addr, err) \
|
#define __get_mem_asm(load, reg, x, addr, err, type) \
|
||||||
asm volatile( \
|
asm volatile( \
|
||||||
"1: " load " " reg "1, [%2]\n" \
|
"1: " load " " reg "1, [%2]\n" \
|
||||||
"2:\n" \
|
"2:\n" \
|
||||||
_ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \
|
_ASM_EXTABLE_##type##ACCESS_ERR_ZERO(1b, 2b, %w0, %w1) \
|
||||||
: "+r" (err), "=&r" (x) \
|
: "+r" (err), "=&r" (x) \
|
||||||
: "r" (addr))
|
: "r" (addr))
|
||||||
|
|
||||||
#define __raw_get_mem(ldr, x, ptr, err) \
|
#define __raw_get_mem(ldr, x, ptr, err, type) \
|
||||||
do { \
|
do { \
|
||||||
unsigned long __gu_val; \
|
unsigned long __gu_val; \
|
||||||
switch (sizeof(*(ptr))) { \
|
switch (sizeof(*(ptr))) { \
|
||||||
case 1: \
|
case 1: \
|
||||||
__get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err)); \
|
__get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 2: \
|
case 2: \
|
||||||
__get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err)); \
|
__get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 4: \
|
case 4: \
|
||||||
__get_mem_asm(ldr, "%w", __gu_val, (ptr), (err)); \
|
__get_mem_asm(ldr, "%w", __gu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 8: \
|
case 8: \
|
||||||
__get_mem_asm(ldr, "%x", __gu_val, (ptr), (err)); \
|
__get_mem_asm(ldr, "%x", __gu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
default: \
|
default: \
|
||||||
BUILD_BUG(); \
|
BUILD_BUG(); \
|
||||||
} \
|
} \
|
||||||
(x) = (__force __typeof__(*(ptr)))__gu_val; \
|
(x) = (__force __typeof__(*(ptr)))__gu_val; \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -274,7 +274,7 @@ do { \
|
||||||
__chk_user_ptr(ptr); \
|
__chk_user_ptr(ptr); \
|
||||||
\
|
\
|
||||||
uaccess_ttbr0_enable(); \
|
uaccess_ttbr0_enable(); \
|
||||||
__raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err); \
|
__raw_get_mem("ldtr", __rgu_val, __rgu_ptr, err, U); \
|
||||||
uaccess_ttbr0_disable(); \
|
uaccess_ttbr0_disable(); \
|
||||||
\
|
\
|
||||||
(x) = __rgu_val; \
|
(x) = __rgu_val; \
|
||||||
|
@ -314,40 +314,40 @@ do { \
|
||||||
\
|
\
|
||||||
__uaccess_enable_tco_async(); \
|
__uaccess_enable_tco_async(); \
|
||||||
__raw_get_mem("ldr", *((type *)(__gkn_dst)), \
|
__raw_get_mem("ldr", *((type *)(__gkn_dst)), \
|
||||||
(__force type *)(__gkn_src), __gkn_err); \
|
(__force type *)(__gkn_src), __gkn_err, K); \
|
||||||
__uaccess_disable_tco_async(); \
|
__uaccess_disable_tco_async(); \
|
||||||
\
|
\
|
||||||
if (unlikely(__gkn_err)) \
|
if (unlikely(__gkn_err)) \
|
||||||
goto err_label; \
|
goto err_label; \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
#define __put_mem_asm(store, reg, x, addr, err) \
|
#define __put_mem_asm(store, reg, x, addr, err, type) \
|
||||||
asm volatile( \
|
asm volatile( \
|
||||||
"1: " store " " reg "1, [%2]\n" \
|
"1: " store " " reg "1, [%2]\n" \
|
||||||
"2:\n" \
|
"2:\n" \
|
||||||
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w0) \
|
_ASM_EXTABLE_##type##ACCESS_ERR(1b, 2b, %w0) \
|
||||||
: "+r" (err) \
|
: "+r" (err) \
|
||||||
: "r" (x), "r" (addr))
|
: "r" (x), "r" (addr))
|
||||||
|
|
||||||
#define __raw_put_mem(str, x, ptr, err) \
|
#define __raw_put_mem(str, x, ptr, err, type) \
|
||||||
do { \
|
do { \
|
||||||
__typeof__(*(ptr)) __pu_val = (x); \
|
__typeof__(*(ptr)) __pu_val = (x); \
|
||||||
switch (sizeof(*(ptr))) { \
|
switch (sizeof(*(ptr))) { \
|
||||||
case 1: \
|
case 1: \
|
||||||
__put_mem_asm(str "b", "%w", __pu_val, (ptr), (err)); \
|
__put_mem_asm(str "b", "%w", __pu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 2: \
|
case 2: \
|
||||||
__put_mem_asm(str "h", "%w", __pu_val, (ptr), (err)); \
|
__put_mem_asm(str "h", "%w", __pu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 4: \
|
case 4: \
|
||||||
__put_mem_asm(str, "%w", __pu_val, (ptr), (err)); \
|
__put_mem_asm(str, "%w", __pu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
case 8: \
|
case 8: \
|
||||||
__put_mem_asm(str, "%x", __pu_val, (ptr), (err)); \
|
__put_mem_asm(str, "%x", __pu_val, (ptr), (err), type); \
|
||||||
break; \
|
break; \
|
||||||
default: \
|
default: \
|
||||||
BUILD_BUG(); \
|
BUILD_BUG(); \
|
||||||
} \
|
} \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -362,7 +362,7 @@ do { \
|
||||||
__chk_user_ptr(__rpu_ptr); \
|
__chk_user_ptr(__rpu_ptr); \
|
||||||
\
|
\
|
||||||
uaccess_ttbr0_enable(); \
|
uaccess_ttbr0_enable(); \
|
||||||
__raw_put_mem("sttr", __rpu_val, __rpu_ptr, err); \
|
__raw_put_mem("sttr", __rpu_val, __rpu_ptr, err, U); \
|
||||||
uaccess_ttbr0_disable(); \
|
uaccess_ttbr0_disable(); \
|
||||||
} while (0)
|
} while (0)
|
||||||
|
|
||||||
|
@ -400,7 +400,7 @@ do { \
|
||||||
\
|
\
|
||||||
__uaccess_enable_tco_async(); \
|
__uaccess_enable_tco_async(); \
|
||||||
__raw_put_mem("str", *((type *)(__pkn_src)), \
|
__raw_put_mem("str", *((type *)(__pkn_src)), \
|
||||||
(__force type *)(__pkn_dst), __pkn_err); \
|
(__force type *)(__pkn_dst), __pkn_err, K); \
|
||||||
__uaccess_disable_tco_async(); \
|
__uaccess_disable_tco_async(); \
|
||||||
\
|
\
|
||||||
if (unlikely(__pkn_err)) \
|
if (unlikely(__pkn_err)) \
|
||||||
|
|
|
@ -16,13 +16,6 @@ get_ex_fixup(const struct exception_table_entry *ex)
|
||||||
return ((unsigned long)&ex->fixup + ex->fixup);
|
return ((unsigned long)&ex->fixup + ex->fixup);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool ex_handler_fixup(const struct exception_table_entry *ex,
|
|
||||||
struct pt_regs *regs)
|
|
||||||
{
|
|
||||||
regs->pc = get_ex_fixup(ex);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
|
static bool ex_handler_uaccess_err_zero(const struct exception_table_entry *ex,
|
||||||
struct pt_regs *regs)
|
struct pt_regs *regs)
|
||||||
{
|
{
|
||||||
|
@ -72,11 +65,10 @@ bool fixup_exception(struct pt_regs *regs)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
switch (ex->type) {
|
switch (ex->type) {
|
||||||
case EX_TYPE_FIXUP:
|
|
||||||
return ex_handler_fixup(ex, regs);
|
|
||||||
case EX_TYPE_BPF:
|
case EX_TYPE_BPF:
|
||||||
return ex_handler_bpf(ex, regs);
|
return ex_handler_bpf(ex, regs);
|
||||||
case EX_TYPE_UACCESS_ERR_ZERO:
|
case EX_TYPE_UACCESS_ERR_ZERO:
|
||||||
|
case EX_TYPE_KACCESS_ERR_ZERO:
|
||||||
return ex_handler_uaccess_err_zero(ex, regs);
|
return ex_handler_uaccess_err_zero(ex, regs);
|
||||||
case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
|
case EX_TYPE_LOAD_UNALIGNED_ZEROPAD:
|
||||||
return ex_handler_load_unaligned_zeropad(ex, regs);
|
return ex_handler_load_unaligned_zeropad(ex, regs);
|
||||||
|
|
Loading…
Reference in a new issue