kcfi updates for v6.1-rc1
This replaces the prior support for Clang's standard Control Flow
Integrity (CFI) instrumentation, which has required a lot of special
conditions (e.g. LTO) and work-arounds. The current implementation
("Kernel CFI") is specific to C, directly designed for the Linux kernel,
and takes advantage of architectural features like x86's IBT. This
series retains arm64 support and adds x86 support. Additional "generic"
architectural support is expected soon:
https://github.com/samitolvanen/llvm-project/commits/kcfi_generic
- treewide: Remove old CFI support details
- arm64: Replace Clang CFI support with Clang KCFI support
- x86: Introduce Clang KCFI support
-----BEGIN PGP SIGNATURE-----
iQJKBAABCgA0FiEEpcP2jyKd1g9yPm4TiXL039xtwCYFAmM4aAUWHGtlZXNjb29r
QGNocm9taXVtLm9yZwAKCRCJcvTf3G3AJkgWD/4mUgb7xewNIG/+fuipGd620Iao
K0T8q4BNxLNRltOxNc3Q0WMDCggX0qJGCeds7EdFQJQOGxWcbifM8MAS4idAGM0G
fc3Gxl1imC/oF6goCAbQgndA6jYFIWXGsv8LsRjAXRidWLFr3GFAqVqYJyokSySr
8zMQsEDuF4I1gQnOhEWdtPZbV3MQ4ZjfFzpv+33agbq6Gb72vKvDh3G6g2VXlxjt
1qnMtS+eEpbBU65cJkOi4MSLgymWbnIAeTMb0dbsV4kJ08YoTl8uz1B+weeH6GgT
WP73ZJ4nqh1kkkT9EqS9oKozNB9fObhvCokEuAjuQ7i1eCEZsbShvRc0iL7OKTGG
UfuTJa5qQ4h7Z0JS35FCSJETa+fcG0lTyEd133nLXLMZP9K2antf+A6O//fd0J1V
Jg4VN7DQmZ+UNGOzRkL6dTtQUy4PkxhniIloaClfSYXxhNirA+v//sHTnTK3z2Bl
6qceYqmFmns2Laual7+lvnZgt6egMBcmAL/MOdbU74+KIR9Xw76wxQjifktHX+WF
FEUQkUJDB5XcUyKlbvHoqobRMxvEZ8RIlC5DIkgFiPRE3TI0MqfzNSFnQ/6+lFNg
Y0AS9HYJmcj8sVzAJ7ji24WPFCXzsbFn6baJa9usDNbWyQZokYeiv7ZPNPHPDVrv
YEBP6aYko0lVSUS9qw==
=Li4D
-----END PGP SIGNATURE-----
Merge tag 'kcfi-v6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux
Pull kcfi updates from Kees Cook:
"This replaces the prior support for Clang's standard Control Flow
Integrity (CFI) instrumentation, which has required a lot of special
conditions (e.g. LTO) and work-arounds.
The new implementation ("Kernel CFI") is specific to C, directly
designed for the Linux kernel, and takes advantage of architectural
features like x86's IBT. This series retains arm64 support and adds
x86 support.
GCC support is expected in the future[1], and additional "generic"
architectural support is expected soon[2].
Summary:
- treewide: Remove old CFI support details
- arm64: Replace Clang CFI support with Clang KCFI support
- x86: Introduce Clang KCFI support"
Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=107048 [1]
Link: https://github.com/samitolvanen/llvm-project/commits/kcfi_generic [2]
* tag 'kcfi-v6.1-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux: (22 commits)
x86: Add support for CONFIG_CFI_CLANG
x86/purgatory: Disable CFI
x86: Add types to indirectly called assembly functions
x86/tools/relocs: Ignore __kcfi_typeid_ relocations
kallsyms: Drop CONFIG_CFI_CLANG workarounds
objtool: Disable CFI warnings
objtool: Preserve special st_shndx indexes in elf_update_symbol
treewide: Drop __cficanonical
treewide: Drop WARN_ON_FUNCTION_MISMATCH
treewide: Drop function_nocfi
init: Drop __nocfi from __init
arm64: Drop unneeded __nocfi attributes
arm64: Add CFI error handling
arm64: Add types to indirect called assembly functions
psci: Fix the function type for psci_initcall_t
lkdtm: Emit an indirect call for CFI tests
cfi: Add type helper macros
cfi: Switch to -fsanitize=kcfi
cfi: Drop __CFI_ADDRESSABLE
cfi: Remove CONFIG_CFI_CLANG_SHADOW
...
This commit is contained in:
Linus Torvalds
commit
865dad2022
13
Makefile
13
Makefile
|
|
@ -1001,18 +1001,7 @@ export CC_FLAGS_LTO
|
|||
endif
|
||||
|
||||
ifdef CONFIG_CFI_CLANG
|
||||
CC_FLAGS_CFI := -fsanitize=cfi \
|
||||
-fsanitize-cfi-cross-dso \
|
||||
-fno-sanitize-cfi-canonical-jump-tables \
|
||||
-fno-sanitize-trap=cfi \
|
||||
-fno-sanitize-blacklist
|
||||
|
||||
ifdef CONFIG_CFI_PERMISSIVE
|
||||
CC_FLAGS_CFI += -fsanitize-recover=cfi
|
||||
endif
|
||||
|
||||
# If LTO flags are filtered out, we must also filter out CFI.
|
||||
CC_FLAGS_LTO += $(CC_FLAGS_CFI)
|
||||
CC_FLAGS_CFI := -fsanitize=kcfi
|
||||
KBUILD_CFLAGS += $(CC_FLAGS_CFI)
|
||||
export CC_FLAGS_CFI
|
||||
endif
|
||||
|
|
|
|||
18
arch/Kconfig
18
arch/Kconfig
|
|
@ -744,11 +744,13 @@ config ARCH_SUPPORTS_CFI_CLANG
|
|||
An architecture should select this option if it can support Clang's
|
||||
Control-Flow Integrity (CFI) checking.
|
||||
|
||||
config ARCH_USES_CFI_TRAPS
|
||||
bool
|
||||
|
||||
config CFI_CLANG
|
||||
bool "Use Clang's Control Flow Integrity (CFI)"
|
||||
depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG
|
||||
depends on CLANG_VERSION >= 140000
|
||||
select KALLSYMS
|
||||
depends on ARCH_SUPPORTS_CFI_CLANG
|
||||
depends on $(cc-option,-fsanitize=kcfi)
|
||||
help
|
||||
This option enables Clang’s forward-edge Control Flow Integrity
|
||||
(CFI) checking, where the compiler injects a runtime check to each
|
||||
|
|
@ -760,16 +762,6 @@ config CFI_CLANG
|
|||
|
||||
https://clang.llvm.org/docs/ControlFlowIntegrity.html
|
||||
|
||||
config CFI_CLANG_SHADOW
|
||||
bool "Use CFI shadow to speed up cross-module checks"
|
||||
default y
|
||||
depends on CFI_CLANG && MODULES
|
||||
help
|
||||
If you select this option, the kernel builds a fast look-up table of
|
||||
CFI check functions in loaded modules to reduce performance overhead.
|
||||
|
||||
If unsure, say Y.
|
||||
|
||||
config CFI_PERMISSIVE
|
||||
bool "Use CFI in permissive mode"
|
||||
depends on CFI_CLANG
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/assembler.h>
|
||||
|
||||
SHASH .req v0
|
||||
|
|
@ -350,11 +351,11 @@ CPU_LE( rev64 T1.16b, T1.16b )
|
|||
* void pmull_ghash_update(int blocks, u64 dg[], const char *src,
|
||||
* struct ghash_key const *k, const char *head)
|
||||
*/
|
||||
SYM_FUNC_START(pmull_ghash_update_p64)
|
||||
SYM_TYPED_FUNC_START(pmull_ghash_update_p64)
|
||||
__pmull_ghash p64
|
||||
SYM_FUNC_END(pmull_ghash_update_p64)
|
||||
|
||||
SYM_FUNC_START(pmull_ghash_update_p8)
|
||||
SYM_TYPED_FUNC_START(pmull_ghash_update_p8)
|
||||
__pmull_ghash p8
|
||||
SYM_FUNC_END(pmull_ghash_update_p8)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/assembler.h>
|
||||
|
||||
.irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
|
||||
|
|
@ -73,7 +74,7 @@
|
|||
* int blocks)
|
||||
*/
|
||||
.text
|
||||
SYM_FUNC_START(sm3_ce_transform)
|
||||
SYM_TYPED_FUNC_START(sm3_ce_transform)
|
||||
/* load state */
|
||||
ld1 {v8.4s-v9.4s}, [x0]
|
||||
rev64 v8.4s, v8.4s
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
* 0x401: for compile time BRK instruction
|
||||
* 0x800: kernel-mode BUG() and WARN() traps
|
||||
* 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff)
|
||||
* 0x8xxx: Control-Flow Integrity traps
|
||||
*/
|
||||
#define KPROBES_BRK_IMM 0x004
|
||||
#define UPROBES_BRK_IMM 0x005
|
||||
|
|
@ -28,4 +29,9 @@
|
|||
#define KASAN_BRK_IMM 0x900
|
||||
#define KASAN_BRK_MASK 0x0ff
|
||||
|
||||
#define CFI_BRK_IMM_TARGET GENMASK(4, 0)
|
||||
#define CFI_BRK_IMM_TYPE GENMASK(9, 5)
|
||||
#define CFI_BRK_IMM_BASE 0x8000
|
||||
#define CFI_BRK_IMM_MASK (CFI_BRK_IMM_TARGET | CFI_BRK_IMM_TYPE)
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@
|
|||
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
|
||||
#define ARCH_SUPPORTS_FTRACE_OPS 1
|
||||
#else
|
||||
#define MCOUNT_ADDR ((unsigned long)function_nocfi(_mcount))
|
||||
#define MCOUNT_ADDR ((unsigned long)_mcount)
|
||||
#endif
|
||||
|
||||
/* The BL at the callsite's adjusted rec->ip */
|
||||
|
|
|
|||
|
|
@ -39,4 +39,8 @@
|
|||
SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \
|
||||
bti c ;
|
||||
|
||||
#define SYM_TYPED_FUNC_START(name) \
|
||||
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
|
||||
bti c ;
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
|
|||
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
|
||||
* avoiding the possibility of conflicting TLB entries being allocated.
|
||||
*/
|
||||
static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
|
||||
static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
|
||||
{
|
||||
typedef void (ttbr_replace_func)(phys_addr_t);
|
||||
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
|
||||
|
|
@ -168,7 +168,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
|
|||
ttbr1 |= TTBR_CNP_BIT;
|
||||
}
|
||||
|
||||
replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1));
|
||||
replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
|
||||
|
||||
__cpu_install_idmap(idmap);
|
||||
replace_phys(ttbr1);
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu)
|
|||
* that read this address need to convert this address to the
|
||||
* Boot-Loader's endianness before jumping.
|
||||
*/
|
||||
writeq_relaxed(__pa_symbol(function_nocfi(secondary_entry)),
|
||||
writeq_relaxed(__pa_symbol(secondary_entry),
|
||||
&mailbox->entry_point);
|
||||
writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id);
|
||||
|
||||
|
|
|
|||
|
|
@ -133,7 +133,7 @@ static void clean_dcache_range_nopatch(u64 start, u64 end)
|
|||
} while (cur += d_size, cur < end);
|
||||
}
|
||||
|
||||
static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module,
|
||||
static void __apply_alternatives(struct alt_region *region, bool is_module,
|
||||
unsigned long *feature_mask)
|
||||
{
|
||||
struct alt_instr *alt;
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/assembler.h>
|
||||
#include <asm/sysreg.h>
|
||||
#include <asm/virt.h>
|
||||
|
|
@ -28,7 +29,7 @@
|
|||
* branch to what would be the reset vector. It must be executed with the
|
||||
* flat identity mapping.
|
||||
*/
|
||||
SYM_CODE_START(cpu_soft_restart)
|
||||
SYM_TYPED_FUNC_START(cpu_soft_restart)
|
||||
mov_q x12, INIT_SCTLR_EL1_MMU_OFF
|
||||
pre_disable_mmu_workaround
|
||||
/*
|
||||
|
|
@ -47,6 +48,6 @@ SYM_CODE_START(cpu_soft_restart)
|
|||
mov x1, x3 // arg1
|
||||
mov x2, x4 // arg2
|
||||
br x8
|
||||
SYM_CODE_END(cpu_soft_restart)
|
||||
SYM_FUNC_END(cpu_soft_restart)
|
||||
|
||||
.popsection
|
||||
|
|
|
|||
|
|
@ -1685,7 +1685,7 @@ static phys_addr_t kpti_ng_pgd_alloc(int shift)
|
|||
return kpti_ng_temp_alloc;
|
||||
}
|
||||
|
||||
static void __nocfi
|
||||
static void
|
||||
kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
|
||||
{
|
||||
typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
|
||||
|
|
@ -1713,7 +1713,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
|
|||
if (arm64_use_ng_mappings)
|
||||
return;
|
||||
|
||||
remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings));
|
||||
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
|
||||
|
||||
if (!cpu) {
|
||||
alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
|
|||
unsigned long pc;
|
||||
u32 new;
|
||||
|
||||
pc = (unsigned long)function_nocfi(ftrace_call);
|
||||
pc = (unsigned long)ftrace_call;
|
||||
new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
|
||||
AARCH64_INSN_BRANCH_LINK);
|
||||
|
||||
|
|
|
|||
|
|
@ -204,7 +204,7 @@ void machine_kexec(struct kimage *kimage)
|
|||
typeof(cpu_soft_restart) *restart;
|
||||
|
||||
cpu_install_idmap();
|
||||
restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart));
|
||||
restart = (void *)__pa_symbol(cpu_soft_restart);
|
||||
restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
|
||||
0, 0);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu)
|
|||
|
||||
static int cpu_psci_cpu_boot(unsigned int cpu)
|
||||
{
|
||||
phys_addr_t pa_secondary_entry = __pa_symbol(function_nocfi(secondary_entry));
|
||||
phys_addr_t pa_secondary_entry = __pa_symbol(secondary_entry);
|
||||
int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry);
|
||||
if (err)
|
||||
pr_err("failed to boot CPU%d (%d)\n", cpu, err);
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu)
|
|||
static int smp_spin_table_cpu_prepare(unsigned int cpu)
|
||||
{
|
||||
__le64 __iomem *release_addr;
|
||||
phys_addr_t pa_holding_pen = __pa_symbol(function_nocfi(secondary_holding_pen));
|
||||
phys_addr_t pa_holding_pen = __pa_symbol(secondary_holding_pen);
|
||||
|
||||
if (!cpu_release_addr[cpu])
|
||||
return -ENODEV;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
#include <linux/syscalls.h>
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/kasan.h>
|
||||
#include <linux/cfi.h>
|
||||
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/bug.h>
|
||||
|
|
@ -991,6 +992,38 @@ static struct break_hook bug_break_hook = {
|
|||
.imm = BUG_BRK_IMM,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
static int cfi_handler(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
unsigned long target;
|
||||
u32 type;
|
||||
|
||||
target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr));
|
||||
type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr));
|
||||
|
||||
switch (report_cfi_failure(regs, regs->pc, &target, type)) {
|
||||
case BUG_TRAP_TYPE_BUG:
|
||||
die("Oops - CFI", regs, 0);
|
||||
break;
|
||||
|
||||
case BUG_TRAP_TYPE_WARN:
|
||||
break;
|
||||
|
||||
default:
|
||||
return DBG_HOOK_ERROR;
|
||||
}
|
||||
|
||||
arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
|
||||
return DBG_HOOK_HANDLED;
|
||||
}
|
||||
|
||||
static struct break_hook cfi_break_hook = {
|
||||
.fn = cfi_handler,
|
||||
.imm = CFI_BRK_IMM_BASE,
|
||||
.mask = CFI_BRK_IMM_MASK,
|
||||
};
|
||||
#endif /* CONFIG_CFI_CLANG */
|
||||
|
||||
static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr)
|
||||
{
|
||||
pr_err("%s generated an invalid instruction at %pS!\n",
|
||||
|
|
@ -1052,6 +1085,9 @@ static struct break_hook kasan_break_hook = {
|
|||
};
|
||||
#endif
|
||||
|
||||
|
||||
#define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK)
|
||||
|
||||
/*
|
||||
* Initial handler for AArch64 BRK exceptions
|
||||
* This handler only used until debug_traps_init().
|
||||
|
|
@ -1059,10 +1095,12 @@ static struct break_hook kasan_break_hook = {
|
|||
int __init early_brk64(unsigned long addr, unsigned long esr,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE)
|
||||
return cfi_handler(regs, esr) != DBG_HOOK_HANDLED;
|
||||
#endif
|
||||
#ifdef CONFIG_KASAN_SW_TAGS
|
||||
unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
|
||||
|
||||
if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
|
||||
if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
|
||||
return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
|
||||
#endif
|
||||
return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
|
||||
|
|
@ -1071,6 +1109,9 @@ int __init early_brk64(unsigned long addr, unsigned long esr,
|
|||
void __init trap_init(void)
|
||||
{
|
||||
register_kernel_break_hook(&bug_break_hook);
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
register_kernel_break_hook(&cfi_break_hook);
|
||||
#endif
|
||||
register_kernel_break_hook(&fault_break_hook);
|
||||
#ifdef CONFIG_KASAN_SW_TAGS
|
||||
register_kernel_break_hook(&kasan_break_hook);
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
|
|||
# kernel with CONFIG_WERROR enabled.
|
||||
CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
|
||||
$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
|
||||
$(CC_FLAGS_LTO) -Wmissing-prototypes -Wmissing-declarations
|
||||
$(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
|
||||
-Wmissing-prototypes -Wmissing-declarations
|
||||
KASAN_SANITIZE := n
|
||||
KCSAN_SANITIZE := n
|
||||
UBSAN_SANITIZE := n
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <linux/init.h>
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/pgtable.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/assembler.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/asm_pointer_auth.h>
|
||||
|
|
@ -185,7 +186,7 @@ SYM_FUNC_END(cpu_do_resume)
|
|||
* This is the low-level counterpart to cpu_replace_ttbr1, and should not be
|
||||
* called by anything else. It can only be executed from a TTBR0 mapping.
|
||||
*/
|
||||
SYM_FUNC_START(idmap_cpu_replace_ttbr1)
|
||||
SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
|
||||
save_and_disable_daif flags=x2
|
||||
|
||||
__idmap_cpu_set_reserved_ttbr1 x1, x3
|
||||
|
|
@ -253,7 +254,7 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1)
|
|||
SYM_DATA(__idmap_kpti_flag, .long 1)
|
||||
.popsection
|
||||
|
||||
SYM_FUNC_START(idmap_kpti_install_ng_mappings)
|
||||
SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
|
||||
cpu .req w0
|
||||
temp_pte .req x0
|
||||
num_cpus .req w1
|
||||
|
|
|
|||
|
|
@ -107,6 +107,8 @@ config X86
|
|||
select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64
|
||||
select ARCH_SUPPORTS_NUMA_BALANCING if X86_64
|
||||
select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096
|
||||
select ARCH_SUPPORTS_CFI_CLANG if X86_64
|
||||
select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG
|
||||
select ARCH_SUPPORTS_LTO_CLANG
|
||||
select ARCH_SUPPORTS_LTO_CLANG_THIN
|
||||
select ARCH_USE_BUILTIN_BSWAP
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
|
||||
.file "blowfish-x86_64-asm.S"
|
||||
.text
|
||||
|
|
@ -141,7 +142,7 @@ SYM_FUNC_START(__blowfish_enc_blk)
|
|||
RET;
|
||||
SYM_FUNC_END(__blowfish_enc_blk)
|
||||
|
||||
SYM_FUNC_START(blowfish_dec_blk)
|
||||
SYM_TYPED_FUNC_START(blowfish_dec_blk)
|
||||
/* input:
|
||||
* %rdi: ctx
|
||||
* %rsi: dst
|
||||
|
|
@ -332,7 +333,7 @@ SYM_FUNC_START(__blowfish_enc_blk_4way)
|
|||
RET;
|
||||
SYM_FUNC_END(__blowfish_enc_blk_4way)
|
||||
|
||||
SYM_FUNC_START(blowfish_dec_blk_4way)
|
||||
SYM_TYPED_FUNC_START(blowfish_dec_blk_4way)
|
||||
/* input:
|
||||
* %rdi: ctx
|
||||
* %rsi: dst
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),)
|
|||
endif
|
||||
endif
|
||||
|
||||
$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
|
||||
$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
|
||||
$(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
|
||||
|
||||
#
|
||||
|
|
@ -153,6 +153,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32))
|
|||
KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32))
|
||||
KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
|
||||
KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
|
||||
KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))
|
||||
KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic
|
||||
KBUILD_CFLAGS_32 += -fno-stack-protector
|
||||
KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _ASM_X86_CFI_H
|
||||
#define _ASM_X86_CFI_H
|
||||
|
||||
/*
|
||||
* Clang Control Flow Integrity (CFI) support.
|
||||
*
|
||||
* Copyright (C) 2022 Google LLC
|
||||
*/
|
||||
|
||||
#include <linux/cfi.h>
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
|
||||
#else
|
||||
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
|
||||
{
|
||||
return BUG_TRAP_TYPE_NONE;
|
||||
}
|
||||
#endif /* CONFIG_CFI_CLANG */
|
||||
|
||||
#endif /* _ASM_X86_CFI_H */
|
||||
|
|
@ -43,6 +43,18 @@
|
|||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#define __CFI_TYPE(name) \
|
||||
SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \
|
||||
.fill 11, 1, 0x90 ASM_NL \
|
||||
.byte 0xb8 ASM_NL \
|
||||
.long __kcfi_typeid_##name ASM_NL \
|
||||
SYM_FUNC_END(__cfi_##name)
|
||||
|
||||
/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
|
||||
#define SYM_TYPED_FUNC_START(name) \
|
||||
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
|
||||
ENDBR
|
||||
|
||||
/* SYM_FUNC_START -- use for global functions */
|
||||
#define SYM_FUNC_START(name) \
|
||||
SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \
|
||||
|
|
|
|||
|
|
@ -139,6 +139,8 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o
|
|||
|
||||
obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o
|
||||
|
||||
obj-$(CONFIG_CFI_CLANG) += cfi.o
|
||||
|
||||
###
|
||||
# 64 bit specific files
|
||||
ifeq ($(CONFIG_X86_64),y)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Clang Control Flow Integrity (CFI) support.
|
||||
*
|
||||
* Copyright (C) 2022 Google LLC
|
||||
*/
|
||||
#include <asm/cfi.h>
|
||||
#include <asm/insn.h>
|
||||
#include <asm/insn-eval.h>
|
||||
#include <linux/string.h>
|
||||
|
||||
/*
|
||||
* Returns the target address and the expected type when regs->ip points
|
||||
* to a compiler-generated CFI trap.
|
||||
*/
|
||||
static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
|
||||
u32 *type)
|
||||
{
|
||||
char buffer[MAX_INSN_SIZE];
|
||||
struct insn insn;
|
||||
int offset = 0;
|
||||
|
||||
*target = *type = 0;
|
||||
|
||||
/*
|
||||
* The compiler generates the following instruction sequence
|
||||
* for indirect call checks:
|
||||
*
|
||||
* movl -<id>, %r10d ; 6 bytes
|
||||
* addl -4(%reg), %r10d ; 4 bytes
|
||||
* je .Ltmp1 ; 2 bytes
|
||||
* ud2 ; <- regs->ip
|
||||
* .Ltmp1:
|
||||
*
|
||||
* We can decode the expected type and the target address from the
|
||||
* movl/addl instructions.
|
||||
*/
|
||||
if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 12, MAX_INSN_SIZE))
|
||||
return false;
|
||||
if (insn_decode_kernel(&insn, &buffer[offset]))
|
||||
return false;
|
||||
if (insn.opcode.value != 0xBA)
|
||||
return false;
|
||||
|
||||
*type = -(u32)insn.immediate.value;
|
||||
|
||||
if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 6, MAX_INSN_SIZE))
|
||||
return false;
|
||||
if (insn_decode_kernel(&insn, &buffer[offset]))
|
||||
return false;
|
||||
if (insn.opcode.value != 0x3)
|
||||
return false;
|
||||
|
||||
/* Read the target address from the register. */
|
||||
offset = insn_get_modrm_rm_off(&insn, regs);
|
||||
if (offset < 0)
|
||||
return false;
|
||||
|
||||
*target = *(unsigned long *)((void *)regs + offset);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks if a ud2 trap is because of a CFI failure, and handles the trap
|
||||
* if needed. Returns a bug_trap_type value similarly to report_bug.
|
||||
*/
|
||||
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long target;
|
||||
u32 type;
|
||||
|
||||
if (!is_cfi_trap(regs->ip))
|
||||
return BUG_TRAP_TYPE_NONE;
|
||||
|
||||
if (!decode_cfi_insn(regs, &target, &type))
|
||||
return report_cfi_failure_noaddr(regs, regs->ip);
|
||||
|
||||
return report_cfi_failure(regs, regs->ip, &target, type);
|
||||
}
|
||||
|
||||
/*
|
||||
* Ensure that __kcfi_typeid_ symbols are emitted for functions that may
|
||||
* not be indirectly called with all configurations.
|
||||
*/
|
||||
__ADDRESSABLE(__memcpy)
|
||||
|
|
@ -63,6 +63,7 @@
|
|||
#include <asm/insn-eval.h>
|
||||
#include <asm/vdso.h>
|
||||
#include <asm/tdx.h>
|
||||
#include <asm/cfi.h>
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
#include <asm/x86_init.h>
|
||||
|
|
@ -313,7 +314,8 @@ static noinstr bool handle_bug(struct pt_regs *regs)
|
|||
*/
|
||||
if (regs->flags & X86_EFLAGS_IF)
|
||||
raw_local_irq_enable();
|
||||
if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
|
||||
if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||
|
||||
handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
|
||||
regs->ip += LEN_UD2;
|
||||
handled = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
/* Copyright 2002 Andi Kleen */
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/errno.h>
|
||||
#include <asm/cpufeatures.h>
|
||||
#include <asm/alternative.h>
|
||||
|
|
@ -27,7 +28,7 @@
|
|||
* Output:
|
||||
* rax original destination
|
||||
*/
|
||||
SYM_FUNC_START(__memcpy)
|
||||
SYM_TYPED_FUNC_START(__memcpy)
|
||||
ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
|
||||
"jmp memcpy_erms", X86_FEATURE_ERMS
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE
|
|||
PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS)
|
||||
endif
|
||||
|
||||
ifdef CONFIG_CFI_CLANG
|
||||
PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI)
|
||||
endif
|
||||
|
||||
CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE)
|
||||
CFLAGS_purgatory.o += $(PURGATORY_CFLAGS)
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
|
|||
"^(xen_irq_disable_direct_reloc$|"
|
||||
"xen_save_fl_direct_reloc$|"
|
||||
"VDSO|"
|
||||
"__kcfi_typeid_|"
|
||||
"__crc_)",
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@ KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS), $(KBUILD_CFLAGS))
|
|||
|
||||
# remove SCS flags from all objects in this directory
|
||||
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
|
||||
# disable CFI
|
||||
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS))
|
||||
# disable LTO
|
||||
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS))
|
||||
|
||||
|
|
|
|||
|
|
@ -274,7 +274,7 @@ static void set_conduit(enum arm_smccc_conduit conduit)
|
|||
psci_conduit = conduit;
|
||||
}
|
||||
|
||||
static int get_set_conduit_method(struct device_node *np)
|
||||
static int get_set_conduit_method(const struct device_node *np)
|
||||
{
|
||||
const char *method;
|
||||
|
||||
|
|
@ -334,7 +334,7 @@ static int __init psci_features(u32 psci_func_id)
|
|||
static int psci_suspend_finisher(unsigned long state)
|
||||
{
|
||||
u32 power_state = state;
|
||||
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
|
||||
phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
|
||||
|
||||
return psci_ops.cpu_suspend(power_state, pa_cpu_resume);
|
||||
}
|
||||
|
|
@ -359,7 +359,7 @@ int psci_cpu_suspend_enter(u32 state)
|
|||
|
||||
static int psci_system_suspend(unsigned long unused)
|
||||
{
|
||||
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
|
||||
phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
|
||||
|
||||
return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND),
|
||||
pa_cpu_resume, 0, 0);
|
||||
|
|
@ -528,7 +528,7 @@ typedef int (*psci_initcall_t)(const struct device_node *);
|
|||
*
|
||||
* Probe based on PSCI PSCI_VERSION function
|
||||
*/
|
||||
static int __init psci_0_2_init(struct device_node *np)
|
||||
static int __init psci_0_2_init(const struct device_node *np)
|
||||
{
|
||||
int err;
|
||||
|
||||
|
|
@ -549,7 +549,7 @@ static int __init psci_0_2_init(struct device_node *np)
|
|||
/*
|
||||
* PSCI < v0.2 get PSCI Function IDs via DT.
|
||||
*/
|
||||
static int __init psci_0_1_init(struct device_node *np)
|
||||
static int __init psci_0_1_init(const struct device_node *np)
|
||||
{
|
||||
u32 id;
|
||||
int err;
|
||||
|
|
@ -585,7 +585,7 @@ static int __init psci_0_1_init(struct device_node *np)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int __init psci_1_0_init(struct device_node *np)
|
||||
static int __init psci_1_0_init(const struct device_node *np)
|
||||
{
|
||||
int err;
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,13 @@ static noinline int lkdtm_increment_int(int *counter)
|
|||
|
||||
return *counter;
|
||||
}
|
||||
|
||||
/* Don't allow the compiler to inline the calls. */
|
||||
static noinline void lkdtm_indirect_call(void (*func)(int *))
|
||||
{
|
||||
func(&called_count);
|
||||
}
|
||||
|
||||
/*
|
||||
* This tries to call an indirect function with a mismatched prototype.
|
||||
*/
|
||||
|
|
@ -29,15 +36,11 @@ static void lkdtm_CFI_FORWARD_PROTO(void)
|
|||
* Matches lkdtm_increment_void()'s prototype, but not
|
||||
* lkdtm_increment_int()'s prototype.
|
||||
*/
|
||||
void (*func)(int *);
|
||||
|
||||
pr_info("Calling matched prototype ...\n");
|
||||
func = lkdtm_increment_void;
|
||||
func(&called_count);
|
||||
lkdtm_indirect_call(lkdtm_increment_void);
|
||||
|
||||
pr_info("Calling mismatched prototype ...\n");
|
||||
func = (void *)lkdtm_increment_int;
|
||||
func(&called_count);
|
||||
lkdtm_indirect_call((void *)lkdtm_increment_int);
|
||||
|
||||
pr_err("FAIL: survived mismatched prototype function call!\n");
|
||||
pr_expected_config(CONFIG_CFI_CLANG);
|
||||
|
|
|
|||
|
|
@ -330,7 +330,7 @@ static void lkdtm_USERCOPY_KERNEL(void)
|
|||
|
||||
pr_info("attempting bad copy_to_user from kernel text: %px\n",
|
||||
vm_mmap);
|
||||
if (copy_to_user((void __user *)user_addr, function_nocfi(vm_mmap),
|
||||
if (copy_to_user((void __user *)user_addr, vm_mmap,
|
||||
unconst + PAGE_SIZE)) {
|
||||
pr_warn("copy_to_user failed, but lacked Oops\n");
|
||||
goto free_user;
|
||||
|
|
|
|||
|
|
@ -220,22 +220,6 @@ extern __printf(1, 2) void __warn_printk(const char *fmt, ...);
|
|||
# define WARN_ON_SMP(x) ({0;})
|
||||
#endif
|
||||
|
||||
/*
|
||||
* WARN_ON_FUNCTION_MISMATCH() warns if a value doesn't match a
|
||||
* function address, and can be useful for catching issues with
|
||||
* callback functions, for example.
|
||||
*
|
||||
* With CONFIG_CFI_CLANG, the warning is disabled because the
|
||||
* compiler replaces function addresses taken in C code with
|
||||
* local jump table addresses, which breaks cross-module function
|
||||
* address equality.
|
||||
*/
|
||||
#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_MODULES)
|
||||
# define WARN_ON_FUNCTION_MISMATCH(x, fn) ({ 0; })
|
||||
#else
|
||||
# define WARN_ON_FUNCTION_MISMATCH(x, fn) WARN_ON_ONCE((x) != (fn))
|
||||
#endif
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -421,6 +421,22 @@
|
|||
__end_ro_after_init = .;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* .kcfi_traps contains a list KCFI trap locations.
|
||||
*/
|
||||
#ifndef KCFI_TRAPS
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
#define KCFI_TRAPS \
|
||||
__kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \
|
||||
__start___kcfi_traps = .; \
|
||||
KEEP(*(.kcfi_traps)) \
|
||||
__stop___kcfi_traps = .; \
|
||||
}
|
||||
#else
|
||||
#define KCFI_TRAPS
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Read only Data
|
||||
*/
|
||||
|
|
@ -529,6 +545,8 @@
|
|||
__stop___modver = .; \
|
||||
} \
|
||||
\
|
||||
KCFI_TRAPS \
|
||||
\
|
||||
RO_EXCEPTION_TABLE \
|
||||
NOTES \
|
||||
BTF \
|
||||
|
|
@ -537,20 +555,6 @@
|
|||
__end_rodata = .;
|
||||
|
||||
|
||||
/*
|
||||
* .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI)
|
||||
* jump table entries.
|
||||
*/
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
#define TEXT_CFI_JT \
|
||||
ALIGN_FUNCTION(); \
|
||||
__cfi_jt_start = .; \
|
||||
*(.text..L.cfi.jumptable .text..L.cfi.jumptable.*) \
|
||||
__cfi_jt_end = .;
|
||||
#else
|
||||
#define TEXT_CFI_JT
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Non-instrumentable text section
|
||||
*/
|
||||
|
|
@ -578,7 +582,6 @@
|
|||
*(.text..refcount) \
|
||||
*(.ref.text) \
|
||||
*(.text.asan.* .text.tsan.*) \
|
||||
TEXT_CFI_JT \
|
||||
MEM_KEEP(init.text*) \
|
||||
MEM_KEEP(exit.text*) \
|
||||
|
||||
|
|
@ -1007,8 +1010,7 @@
|
|||
* keep any .init_array.* sections.
|
||||
* https://bugs.llvm.org/show_bug.cgi?id=46478
|
||||
*/
|
||||
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \
|
||||
defined(CONFIG_CFI_CLANG)
|
||||
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
|
||||
# ifdef CONFIG_CONSTRUCTORS
|
||||
# define SANITIZER_DISCARDS \
|
||||
*(.eh_frame)
|
||||
|
|
|
|||
|
|
@ -2,49 +2,38 @@
|
|||
/*
|
||||
* Clang Control Flow Integrity (CFI) support.
|
||||
*
|
||||
* Copyright (C) 2021 Google LLC
|
||||
* Copyright (C) 2022 Google LLC
|
||||
*/
|
||||
#ifndef _LINUX_CFI_H
|
||||
#define _LINUX_CFI_H
|
||||
|
||||
#include <linux/bug.h>
|
||||
#include <linux/module.h>
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag);
|
||||
enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
|
||||
unsigned long *target, u32 type);
|
||||
|
||||
/* Compiler-generated function in each module, and the kernel */
|
||||
extern void __cfi_check(uint64_t id, void *ptr, void *diag);
|
||||
|
||||
/*
|
||||
* Force the compiler to generate a CFI jump table entry for a function
|
||||
* and store the jump table address to __cfi_jt_<function>.
|
||||
*/
|
||||
#define __CFI_ADDRESSABLE(fn, __attr) \
|
||||
const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG_SHADOW
|
||||
|
||||
extern void cfi_module_add(struct module *mod, unsigned long base_addr);
|
||||
extern void cfi_module_remove(struct module *mod, unsigned long base_addr);
|
||||
|
||||
#else
|
||||
|
||||
static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {}
|
||||
static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {}
|
||||
|
||||
#endif /* CONFIG_CFI_CLANG_SHADOW */
|
||||
|
||||
#else /* !CONFIG_CFI_CLANG */
|
||||
|
||||
#ifdef CONFIG_X86_KERNEL_IBT
|
||||
|
||||
#define __CFI_ADDRESSABLE(fn, __attr) \
|
||||
const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
|
||||
|
||||
#endif /* CONFIG_X86_KERNEL_IBT */
|
||||
static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs,
|
||||
unsigned long addr)
|
||||
{
|
||||
return report_cfi_failure(regs, addr, NULL, 0);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
bool is_cfi_trap(unsigned long addr);
|
||||
#endif
|
||||
#endif /* CONFIG_CFI_CLANG */
|
||||
|
||||
#ifndef __CFI_ADDRESSABLE
|
||||
#define __CFI_ADDRESSABLE(fn, __attr)
|
||||
#endif
|
||||
#ifdef CONFIG_MODULES
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
|
||||
struct module *mod);
|
||||
#else
|
||||
static inline void module_cfi_finalize(const Elf_Ehdr *hdr,
|
||||
const Elf_Shdr *sechdrs,
|
||||
struct module *mod) {}
|
||||
#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
|
||||
#endif /* CONFIG_MODULES */
|
||||
|
||||
#endif /* _LINUX_CFI_H */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/*
|
||||
* Clang Control Flow Integrity (CFI) type definitions.
|
||||
*/
|
||||
#ifndef _LINUX_CFI_TYPES_H
|
||||
#define _LINUX_CFI_TYPES_H
|
||||
|
||||
#ifdef __ASSEMBLY__
|
||||
#include <linux/linkage.h>
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
/*
|
||||
* Use the __kcfi_typeid_<function> type identifier symbol to
|
||||
* annotate indirectly called assembly functions. The compiler emits
|
||||
* these symbols for all address-taken function declarations in C
|
||||
* code.
|
||||
*/
|
||||
#ifndef __CFI_TYPE
|
||||
#define __CFI_TYPE(name) \
|
||||
.4byte __kcfi_typeid_##name
|
||||
#endif
|
||||
|
||||
#define SYM_TYPED_ENTRY(name, linkage, align...) \
|
||||
linkage(name) ASM_NL \
|
||||
align ASM_NL \
|
||||
__CFI_TYPE(name) ASM_NL \
|
||||
name:
|
||||
|
||||
#define SYM_TYPED_START(name, linkage, align...) \
|
||||
SYM_TYPED_ENTRY(name, linkage, align)
|
||||
|
||||
#else /* CONFIG_CFI_CLANG */
|
||||
|
||||
#define SYM_TYPED_START(name, linkage, align...) \
|
||||
SYM_START(name, linkage, align)
|
||||
|
||||
#endif /* CONFIG_CFI_CLANG */
|
||||
|
||||
#ifndef SYM_TYPED_FUNC_START
|
||||
#define SYM_TYPED_FUNC_START(name) \
|
||||
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
|
||||
#endif
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* _LINUX_CFI_TYPES_H */
|
||||
|
|
@ -66,17 +66,9 @@
|
|||
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
|
||||
#endif
|
||||
|
||||
#define __nocfi __attribute__((__no_sanitize__("cfi")))
|
||||
#define __cficanonical __attribute__((__cfi_canonical_jump_table__))
|
||||
|
||||
#if defined(CONFIG_CFI_CLANG)
|
||||
/*
|
||||
* With CONFIG_CFI_CLANG, the compiler replaces function address
|
||||
* references with the address of the function's CFI jump table
|
||||
* entry. The function_nocfi macro always returns the address of the
|
||||
* actual function instead.
|
||||
*/
|
||||
#define function_nocfi(x) __builtin_function_start(x)
|
||||
#if __has_feature(kcfi)
|
||||
/* Disable CFI checking inside a function. */
|
||||
#define __nocfi __attribute__((__no_sanitize__("kcfi")))
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -203,16 +203,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
|
|||
__v; \
|
||||
})
|
||||
|
||||
/*
|
||||
* With CONFIG_CFI_CLANG, the compiler replaces function addresses in
|
||||
* instrumented C code with jump table addresses. Architectures that
|
||||
* support CFI can define this macro to return the actual function address
|
||||
* when needed.
|
||||
*/
|
||||
#ifndef function_nocfi
|
||||
#define function_nocfi(x) (x)
|
||||
#endif
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
|
||||
/*
|
||||
|
|
@ -221,9 +211,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
|
|||
* otherwise, or eliminated entirely due to lack of references that are
|
||||
* visible to the compiler.
|
||||
*/
|
||||
#define __ADDRESSABLE(sym) \
|
||||
static void * __section(".discard.addressable") __used \
|
||||
#define ___ADDRESSABLE(sym, __attrs) \
|
||||
static void * __used __attrs \
|
||||
__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
|
||||
#define __ADDRESSABLE(sym) \
|
||||
___ADDRESSABLE(sym, __section(".discard.addressable"))
|
||||
|
||||
/**
|
||||
* offset_to_ptr - convert a relative memory offset to an absolute pointer
|
||||
|
|
|
|||
|
|
@ -269,10 +269,6 @@ struct ftrace_likely_data {
|
|||
# define __nocfi
|
||||
#endif
|
||||
|
||||
#ifndef __cficanonical
|
||||
# define __cficanonical
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Any place that could be marked with the "alloc_size" attribute is also
|
||||
* a place to be marked with the "malloc" attribute. Do this as part of the
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@
|
|||
|
||||
/* These are for everybody (although not all archs will actually
|
||||
discard it in modules) */
|
||||
#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline __nocfi
|
||||
#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline
|
||||
#define __initdata __section(".init.data")
|
||||
#define __initconst __section(".init.rodata")
|
||||
#define __exitdata __section(".exit.data")
|
||||
|
|
@ -220,8 +220,8 @@ extern bool initcall_debug;
|
|||
__initcall_name(initstub, __iid, id)
|
||||
|
||||
#define __define_initcall_stub(__stub, fn) \
|
||||
int __init __cficanonical __stub(void); \
|
||||
int __init __cficanonical __stub(void) \
|
||||
int __init __stub(void); \
|
||||
int __init __stub(void) \
|
||||
{ \
|
||||
return fn(); \
|
||||
} \
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@
|
|||
#include <linux/tracepoint-defs.h>
|
||||
#include <linux/srcu.h>
|
||||
#include <linux/static_call_types.h>
|
||||
#include <linux/cfi.h>
|
||||
|
||||
#include <linux/percpu.h>
|
||||
#include <asm/module.h>
|
||||
|
|
@ -132,7 +131,7 @@ extern void cleanup_module(void);
|
|||
{ return initfn; } \
|
||||
int init_module(void) __copy(initfn) \
|
||||
__attribute__((alias(#initfn))); \
|
||||
__CFI_ADDRESSABLE(init_module, __initdata);
|
||||
___ADDRESSABLE(init_module, __initdata);
|
||||
|
||||
/* This is only required if you want to be unloadable. */
|
||||
#define module_exit(exitfn) \
|
||||
|
|
@ -140,7 +139,7 @@ extern void cleanup_module(void);
|
|||
{ return exitfn; } \
|
||||
void cleanup_module(void) __copy(exitfn) \
|
||||
__attribute__((alias(#exitfn))); \
|
||||
__CFI_ADDRESSABLE(cleanup_module, __exitdata);
|
||||
___ADDRESSABLE(cleanup_module, __exitdata);
|
||||
|
||||
#endif
|
||||
|
||||
|
|
@ -387,8 +386,9 @@ struct module {
|
|||
const s32 *crcs;
|
||||
unsigned int num_syms;
|
||||
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
cfi_check_fn cfi_check;
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
s32 *kcfi_traps;
|
||||
s32 *kcfi_traps_end;
|
||||
#endif
|
||||
|
||||
/* Kernel parameters. */
|
||||
|
|
|
|||
|
|
@ -2019,8 +2019,8 @@ enum pci_fixup_pass {
|
|||
#ifdef CONFIG_LTO_CLANG
|
||||
#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \
|
||||
class_shift, hook, stub) \
|
||||
void __cficanonical stub(struct pci_dev *dev); \
|
||||
void __cficanonical stub(struct pci_dev *dev) \
|
||||
void stub(struct pci_dev *dev); \
|
||||
void stub(struct pci_dev *dev) \
|
||||
{ \
|
||||
hook(dev); \
|
||||
} \
|
||||
|
|
|
|||
372
kernel/cfi.c
372
kernel/cfi.c
|
|
@ -1,339 +1,101 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Clang Control Flow Integrity (CFI) error and slowpath handling.
|
||||
* Clang Control Flow Integrity (CFI) error handling.
|
||||
*
|
||||
* Copyright (C) 2021 Google LLC
|
||||
* Copyright (C) 2022 Google LLC
|
||||
*/
|
||||
|
||||
#include <linux/hardirq.h>
|
||||
#include <linux/kallsyms.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/printk.h>
|
||||
#include <linux/ratelimit.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/set_memory.h>
|
||||
#include <linux/cfi.h>
|
||||
|
||||
/* Compiler-defined handler names */
|
||||
#ifdef CONFIG_CFI_PERMISSIVE
|
||||
#define cfi_failure_handler __ubsan_handle_cfi_check_fail
|
||||
#else
|
||||
#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
|
||||
#endif
|
||||
|
||||
static inline void handle_cfi_failure(void *ptr)
|
||||
enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
|
||||
unsigned long *target, u32 type)
|
||||
{
|
||||
if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
|
||||
WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
|
||||
if (target)
|
||||
pr_err("CFI failure at %pS (target: %pS; expected type: 0x%08x)\n",
|
||||
(void *)addr, (void *)*target, type);
|
||||
else
|
||||
panic("CFI failure (target: %pS)\n", ptr);
|
||||
pr_err("CFI failure at %pS (no target information)\n",
|
||||
(void *)addr);
|
||||
|
||||
if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) {
|
||||
__warn(NULL, 0, (void *)addr, 0, regs, NULL);
|
||||
return BUG_TRAP_TYPE_WARN;
|
||||
}
|
||||
|
||||
return BUG_TRAP_TYPE_BUG;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
static inline unsigned long trap_address(s32 *p)
|
||||
{
|
||||
return (unsigned long)((long)p + (long)*p);
|
||||
}
|
||||
|
||||
static bool is_trap(unsigned long addr, s32 *start, s32 *end)
|
||||
{
|
||||
s32 *p;
|
||||
|
||||
for (p = start; p < end; ++p) {
|
||||
if (trap_address(p) == addr)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_MODULES
|
||||
#ifdef CONFIG_CFI_CLANG_SHADOW
|
||||
/*
|
||||
* Index type. A 16-bit index can address at most (2^16)-2 pages (taking
|
||||
* into account SHADOW_INVALID), i.e. ~256M with 4k pages.
|
||||
*/
|
||||
typedef u16 shadow_t;
|
||||
#define SHADOW_INVALID ((shadow_t)~0UL)
|
||||
|
||||
struct cfi_shadow {
|
||||
/* Page index for the beginning of the shadow */
|
||||
unsigned long base;
|
||||
/* An array of __cfi_check locations (as indices to the shadow) */
|
||||
shadow_t shadow[1];
|
||||
} __packed;
|
||||
|
||||
/*
|
||||
* The shadow covers ~128M from the beginning of the module region. If
|
||||
* the region is larger, we fall back to __module_address for the rest.
|
||||
*/
|
||||
#define __SHADOW_RANGE (_UL(SZ_128M) >> PAGE_SHIFT)
|
||||
|
||||
/* The in-memory size of struct cfi_shadow, always at least one page */
|
||||
#define __SHADOW_PAGES ((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT)
|
||||
#define SHADOW_PAGES max(1UL, __SHADOW_PAGES)
|
||||
#define SHADOW_SIZE (SHADOW_PAGES << PAGE_SHIFT)
|
||||
|
||||
/* The actual size of the shadow array, minus metadata */
|
||||
#define SHADOW_ARR_SIZE (SHADOW_SIZE - offsetof(struct cfi_shadow, shadow))
|
||||
#define SHADOW_ARR_SLOTS (SHADOW_ARR_SIZE / sizeof(shadow_t))
|
||||
|
||||
static DEFINE_MUTEX(shadow_update_lock);
|
||||
static struct cfi_shadow __rcu *cfi_shadow __read_mostly;
|
||||
|
||||
/* Returns the index in the shadow for the given address */
|
||||
static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
|
||||
/* Populates `kcfi_trap(_end)?` fields in `struct module`. */
|
||||
void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
|
||||
struct module *mod)
|
||||
{
|
||||
unsigned long index;
|
||||
unsigned long page = ptr >> PAGE_SHIFT;
|
||||
char *secstrings;
|
||||
unsigned int i;
|
||||
|
||||
if (unlikely(page < s->base))
|
||||
return -1; /* Outside of module area */
|
||||
mod->kcfi_traps = NULL;
|
||||
mod->kcfi_traps_end = NULL;
|
||||
|
||||
index = page - s->base;
|
||||
secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
|
||||
|
||||
if (index >= SHADOW_ARR_SLOTS)
|
||||
return -1; /* Cannot be addressed with shadow */
|
||||
|
||||
return (int)index;
|
||||
}
|
||||
|
||||
/* Returns the page address for an index in the shadow */
|
||||
static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
|
||||
int index)
|
||||
{
|
||||
if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
|
||||
return 0;
|
||||
|
||||
return (s->base + index) << PAGE_SHIFT;
|
||||
}
|
||||
|
||||
/* Returns the __cfi_check function address for the given shadow location */
|
||||
static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s,
|
||||
int index)
|
||||
{
|
||||
if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
|
||||
return 0;
|
||||
|
||||
if (unlikely(s->shadow[index] == SHADOW_INVALID))
|
||||
return 0;
|
||||
|
||||
/* __cfi_check is always page aligned */
|
||||
return (s->base + s->shadow[index]) << PAGE_SHIFT;
|
||||
}
|
||||
|
||||
static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
|
||||
struct cfi_shadow *next)
|
||||
{
|
||||
int i, index, check;
|
||||
|
||||
/* Mark everything invalid */
|
||||
memset(next->shadow, 0xFF, SHADOW_ARR_SIZE);
|
||||
|
||||
if (!prev)
|
||||
return; /* No previous shadow */
|
||||
|
||||
/* If the base address didn't change, an update is not needed */
|
||||
if (prev->base == next->base) {
|
||||
memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Convert the previous shadow to the new address range */
|
||||
for (i = 0; i < SHADOW_ARR_SLOTS; ++i) {
|
||||
if (prev->shadow[i] == SHADOW_INVALID)
|
||||
for (i = 1; i < hdr->e_shnum; i++) {
|
||||
if (strcmp(secstrings + sechdrs[i].sh_name, "__kcfi_traps"))
|
||||
continue;
|
||||
|
||||
index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
|
||||
if (index < 0)
|
||||
continue;
|
||||
|
||||
check = ptr_to_shadow(next,
|
||||
shadow_to_check_fn(prev, prev->shadow[i]));
|
||||
if (check < 0)
|
||||
continue;
|
||||
|
||||
next->shadow[index] = (shadow_t)check;
|
||||
mod->kcfi_traps = (s32 *)sechdrs[i].sh_addr;
|
||||
mod->kcfi_traps_end = (s32 *)(sechdrs[i].sh_addr + sechdrs[i].sh_size);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod,
|
||||
unsigned long min_addr, unsigned long max_addr)
|
||||
static bool is_module_cfi_trap(unsigned long addr)
|
||||
{
|
||||
int check_index;
|
||||
unsigned long check = (unsigned long)mod->cfi_check;
|
||||
unsigned long ptr;
|
||||
|
||||
if (unlikely(!PAGE_ALIGNED(check))) {
|
||||
pr_warn("cfi: not using shadow for module %s\n", mod->name);
|
||||
return;
|
||||
}
|
||||
|
||||
check_index = ptr_to_shadow(s, check);
|
||||
if (check_index < 0)
|
||||
return; /* Module not addressable with shadow */
|
||||
|
||||
/* For each page, store the check function index in the shadow */
|
||||
for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
|
||||
int index = ptr_to_shadow(s, ptr);
|
||||
|
||||
if (index >= 0) {
|
||||
/* Each page must only contain one module */
|
||||
WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID);
|
||||
s->shadow[index] = (shadow_t)check_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod,
|
||||
unsigned long min_addr, unsigned long max_addr)
|
||||
{
|
||||
unsigned long ptr;
|
||||
|
||||
for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
|
||||
int index = ptr_to_shadow(s, ptr);
|
||||
|
||||
if (index >= 0)
|
||||
s->shadow[index] = SHADOW_INVALID;
|
||||
}
|
||||
}
|
||||
|
||||
typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *,
|
||||
unsigned long min_addr, unsigned long max_addr);
|
||||
|
||||
static void update_shadow(struct module *mod, unsigned long base_addr,
|
||||
update_shadow_fn fn)
|
||||
{
|
||||
struct cfi_shadow *prev;
|
||||
struct cfi_shadow *next;
|
||||
unsigned long min_addr, max_addr;
|
||||
|
||||
next = vmalloc(SHADOW_SIZE);
|
||||
|
||||
mutex_lock(&shadow_update_lock);
|
||||
prev = rcu_dereference_protected(cfi_shadow,
|
||||
mutex_is_locked(&shadow_update_lock));
|
||||
|
||||
if (next) {
|
||||
next->base = base_addr >> PAGE_SHIFT;
|
||||
prepare_next_shadow(prev, next);
|
||||
|
||||
min_addr = (unsigned long)mod->core_layout.base;
|
||||
max_addr = min_addr + mod->core_layout.text_size;
|
||||
fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK);
|
||||
|
||||
set_memory_ro((unsigned long)next, SHADOW_PAGES);
|
||||
}
|
||||
|
||||
rcu_assign_pointer(cfi_shadow, next);
|
||||
mutex_unlock(&shadow_update_lock);
|
||||
synchronize_rcu();
|
||||
|
||||
if (prev) {
|
||||
set_memory_rw((unsigned long)prev, SHADOW_PAGES);
|
||||
vfree(prev);
|
||||
}
|
||||
}
|
||||
|
||||
void cfi_module_add(struct module *mod, unsigned long base_addr)
|
||||
{
|
||||
update_shadow(mod, base_addr, add_module_to_shadow);
|
||||
}
|
||||
|
||||
void cfi_module_remove(struct module *mod, unsigned long base_addr)
|
||||
{
|
||||
update_shadow(mod, base_addr, remove_module_from_shadow);
|
||||
}
|
||||
|
||||
static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
|
||||
unsigned long ptr)
|
||||
{
|
||||
int index;
|
||||
|
||||
if (unlikely(!s))
|
||||
return NULL; /* No shadow available */
|
||||
|
||||
index = ptr_to_shadow(s, ptr);
|
||||
if (index < 0)
|
||||
return NULL; /* Cannot be addressed with shadow */
|
||||
|
||||
return (cfi_check_fn)shadow_to_check_fn(s, index);
|
||||
}
|
||||
|
||||
static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
|
||||
{
|
||||
cfi_check_fn fn;
|
||||
|
||||
rcu_read_lock_sched_notrace();
|
||||
fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr);
|
||||
rcu_read_unlock_sched_notrace();
|
||||
|
||||
return fn;
|
||||
}
|
||||
|
||||
#else /* !CONFIG_CFI_CLANG_SHADOW */
|
||||
|
||||
static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#endif /* CONFIG_CFI_CLANG_SHADOW */
|
||||
|
||||
static inline cfi_check_fn find_module_check_fn(unsigned long ptr)
|
||||
{
|
||||
cfi_check_fn fn = NULL;
|
||||
struct module *mod;
|
||||
bool found = false;
|
||||
|
||||
rcu_read_lock_sched_notrace();
|
||||
mod = __module_address(ptr);
|
||||
|
||||
mod = __module_address(addr);
|
||||
if (mod)
|
||||
fn = mod->cfi_check;
|
||||
found = is_trap(addr, mod->kcfi_traps, mod->kcfi_traps_end);
|
||||
|
||||
rcu_read_unlock_sched_notrace();
|
||||
|
||||
return fn;
|
||||
return found;
|
||||
}
|
||||
|
||||
static inline cfi_check_fn find_check_fn(unsigned long ptr)
|
||||
#else /* CONFIG_MODULES */
|
||||
static inline bool is_module_cfi_trap(unsigned long addr)
|
||||
{
|
||||
cfi_check_fn fn = NULL;
|
||||
unsigned long flags;
|
||||
bool rcu_idle;
|
||||
|
||||
if (is_kernel_text(ptr))
|
||||
return __cfi_check;
|
||||
|
||||
/*
|
||||
* Indirect call checks can happen when RCU is not watching. Both
|
||||
* the shadow and __module_address use RCU, so we need to wake it
|
||||
* up if necessary.
|
||||
*/
|
||||
rcu_idle = !rcu_is_watching();
|
||||
if (rcu_idle) {
|
||||
local_irq_save(flags);
|
||||
ct_irq_enter();
|
||||
}
|
||||
|
||||
if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW))
|
||||
fn = find_shadow_check_fn(ptr);
|
||||
if (!fn)
|
||||
fn = find_module_check_fn(ptr);
|
||||
|
||||
if (rcu_idle) {
|
||||
ct_irq_exit();
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
||||
return fn;
|
||||
return false;
|
||||
}
|
||||
|
||||
void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
|
||||
{
|
||||
cfi_check_fn fn = find_check_fn((unsigned long)ptr);
|
||||
|
||||
if (likely(fn))
|
||||
fn(id, ptr, diag);
|
||||
else /* Don't allow unchecked modules */
|
||||
handle_cfi_failure(ptr);
|
||||
}
|
||||
EXPORT_SYMBOL(__cfi_slowpath_diag);
|
||||
|
||||
#else /* !CONFIG_MODULES */
|
||||
|
||||
void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
|
||||
{
|
||||
handle_cfi_failure(ptr); /* No modules */
|
||||
}
|
||||
EXPORT_SYMBOL(__cfi_slowpath_diag);
|
||||
|
||||
#endif /* CONFIG_MODULES */
|
||||
|
||||
void cfi_failure_handler(void *data, void *ptr, void *vtable)
|
||||
extern s32 __start___kcfi_traps[];
|
||||
extern s32 __stop___kcfi_traps[];
|
||||
|
||||
bool is_cfi_trap(unsigned long addr)
|
||||
{
|
||||
handle_cfi_failure(ptr);
|
||||
if (is_trap(addr, __start___kcfi_traps, __stop___kcfi_traps))
|
||||
return true;
|
||||
|
||||
return is_module_cfi_trap(addr);
|
||||
}
|
||||
EXPORT_SYMBOL(cfi_failure_handler);
|
||||
#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
|
||||
|
|
|
|||
|
|
@ -177,7 +177,6 @@ static bool cleanup_symbol_name(char *s)
|
|||
* character in an identifier in C. Suffixes observed:
|
||||
* - foo.llvm.[0-9a-f]+
|
||||
* - foo.[0-9a-f]+
|
||||
* - foo.[0-9a-f]+.cfi_jt
|
||||
*/
|
||||
res = strchr(s, '.');
|
||||
if (res) {
|
||||
|
|
@ -185,22 +184,6 @@ static bool cleanup_symbol_name(char *s)
|
|||
return true;
|
||||
}
|
||||
|
||||
if (!IS_ENABLED(CONFIG_CFI_CLANG) ||
|
||||
!IS_ENABLED(CONFIG_LTO_CLANG_THIN) ||
|
||||
CONFIG_CLANG_VERSION >= 130000)
|
||||
return false;
|
||||
|
||||
/*
|
||||
* Prior to LLVM 13, the following suffixes were observed when thinLTO
|
||||
* and CFI are both enabled:
|
||||
* - foo$[0-9]+
|
||||
*/
|
||||
res = strrchr(s, '$');
|
||||
if (res) {
|
||||
*res = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1050,8 +1050,7 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker,
|
|||
struct timer_list *timer = &dwork->timer;
|
||||
struct kthread_work *work = &dwork->work;
|
||||
|
||||
WARN_ON_FUNCTION_MISMATCH(timer->function,
|
||||
kthread_delayed_work_timer_fn);
|
||||
WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
|
||||
|
||||
/*
|
||||
* If @delay is 0, queue @dwork->work immediately. This is for
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@
|
|||
#include <linux/bsearch.h>
|
||||
#include <linux/dynamic_debug.h>
|
||||
#include <linux/audit.h>
|
||||
#include <linux/cfi.h>
|
||||
#include <uapi/linux/module.h>
|
||||
#include "internal.h"
|
||||
|
||||
|
|
@ -1144,8 +1145,6 @@ void __weak module_arch_freeing_init(struct module *mod)
|
|||
{
|
||||
}
|
||||
|
||||
static void cfi_cleanup(struct module *mod);
|
||||
|
||||
/* Free a module, remove from lists, etc. */
|
||||
static void free_module(struct module *mod)
|
||||
{
|
||||
|
|
@ -1190,9 +1189,6 @@ static void free_module(struct module *mod)
|
|||
mod->name);
|
||||
mutex_unlock(&module_mutex);
|
||||
|
||||
/* Clean up CFI for the module. */
|
||||
cfi_cleanup(mod);
|
||||
|
||||
/* This may be empty, but that's OK */
|
||||
module_arch_freeing_init(mod);
|
||||
module_memfree(mod->init_layout.base);
|
||||
|
|
@ -2602,8 +2598,9 @@ static int complete_formation(struct module *mod, struct load_info *info)
|
|||
if (err < 0)
|
||||
goto out;
|
||||
|
||||
/* This relies on module_mutex for list integrity. */
|
||||
/* These rely on module_mutex for list integrity. */
|
||||
module_bug_finalize(info->hdr, info->sechdrs, mod);
|
||||
module_cfi_finalize(info->hdr, info->sechdrs, mod);
|
||||
|
||||
if (module_check_misalignment(mod))
|
||||
goto out_misaligned;
|
||||
|
|
@ -2665,8 +2662,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void cfi_init(struct module *mod);
|
||||
|
||||
/*
|
||||
* Allocate and load the module: note that size of section 0 is always
|
||||
* zero, and we rely on this for optional sections.
|
||||
|
|
@ -2796,9 +2791,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
|
|||
|
||||
flush_module_icache(mod);
|
||||
|
||||
/* Setup CFI for the module. */
|
||||
cfi_init(mod);
|
||||
|
||||
/* Now copy in args */
|
||||
mod->args = strndup_user(uargs, ~0UL >> 1);
|
||||
if (IS_ERR(mod->args)) {
|
||||
|
|
@ -2875,7 +2867,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
|
|||
synchronize_rcu();
|
||||
kfree(mod->args);
|
||||
free_arch_cleanup:
|
||||
cfi_cleanup(mod);
|
||||
module_arch_cleanup(mod);
|
||||
free_modinfo:
|
||||
free_modinfo(mod);
|
||||
|
|
@ -2961,41 +2952,6 @@ static inline int within(unsigned long addr, void *start, unsigned long size)
|
|||
return ((void *)addr >= start && (void *)addr < start + size);
|
||||
}
|
||||
|
||||
static void cfi_init(struct module *mod)
|
||||
{
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
initcall_t *init;
|
||||
#ifdef CONFIG_MODULE_UNLOAD
|
||||
exitcall_t *exit;
|
||||
#endif
|
||||
|
||||
rcu_read_lock_sched();
|
||||
mod->cfi_check = (cfi_check_fn)
|
||||
find_kallsyms_symbol_value(mod, "__cfi_check");
|
||||
init = (initcall_t *)
|
||||
find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
|
||||
/* Fix init/exit functions to point to the CFI jump table */
|
||||
if (init)
|
||||
mod->init = *init;
|
||||
#ifdef CONFIG_MODULE_UNLOAD
|
||||
exit = (exitcall_t *)
|
||||
find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
|
||||
if (exit)
|
||||
mod->exit = *exit;
|
||||
#endif
|
||||
rcu_read_unlock_sched();
|
||||
|
||||
cfi_module_add(mod, mod_tree.addr_min);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void cfi_cleanup(struct module *mod)
|
||||
{
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
cfi_module_remove(mod, mod_tree.addr_min);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
|
||||
char *module_flags(struct module *mod, char *buf, bool show_state)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1651,7 +1651,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
|
|||
struct work_struct *work = &dwork->work;
|
||||
|
||||
WARN_ON_ONCE(!wq);
|
||||
WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn);
|
||||
WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
|
||||
WARN_ON_ONCE(timer_pending(timer));
|
||||
WARN_ON_ONCE(!list_empty(&work->entry));
|
||||
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ static bool is_ignored_symbol(const char *name, char type)
|
|||
"__ThumbV7PILongThunk_",
|
||||
"__LA25Thunk_", /* mips lld */
|
||||
"__microLA25Thunk_",
|
||||
"__kcfi_typeid_", /* CFI type identifiers */
|
||||
NULL
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -3,20 +3,10 @@
|
|||
* Archs are free to supply their own linker scripts. ld will
|
||||
* combine them automatically.
|
||||
*/
|
||||
#ifdef CONFIG_CFI_CLANG
|
||||
# include <asm/page.h>
|
||||
# define ALIGN_CFI ALIGN(PAGE_SIZE)
|
||||
# define SANITIZER_DISCARDS *(.eh_frame)
|
||||
#else
|
||||
# define ALIGN_CFI
|
||||
# define SANITIZER_DISCARDS
|
||||
#endif
|
||||
|
||||
SECTIONS {
|
||||
/DISCARD/ : {
|
||||
*(.discard)
|
||||
*(.discard.*)
|
||||
SANITIZER_DISCARDS
|
||||
}
|
||||
|
||||
__ksymtab 0 : { *(SORT(___ksymtab+*)) }
|
||||
|
|
@ -33,6 +23,10 @@ SECTIONS {
|
|||
|
||||
__patchable_function_entries : { *(__patchable_function_entries) }
|
||||
|
||||
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
|
||||
__kcfi_traps : { KEEP(*(.kcfi_traps)) }
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_LTO_CLANG
|
||||
/*
|
||||
* With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and
|
||||
|
|
@ -53,15 +47,6 @@ SECTIONS {
|
|||
*(.rodata .rodata.[0-9a-zA-Z_]*)
|
||||
*(.rodata..L*)
|
||||
}
|
||||
|
||||
/*
|
||||
* With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning
|
||||
* of the .text section, and is aligned to PAGE_SIZE.
|
||||
*/
|
||||
.text : ALIGN_CFI {
|
||||
*(.text.__cfi_check)
|
||||
*(.text .text.[0-9a-zA-Z_]* .text..L.cfi*)
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3316,6 +3316,10 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
|
|||
next_insn = next_insn_to_validate(file, insn);
|
||||
|
||||
if (func && insn->func && func != insn->func->pfunc) {
|
||||
/* Ignore KCFI type preambles, which always fall through */
|
||||
if (!strncmp(func->name, "__cfi_", 6))
|
||||
return 0;
|
||||
|
||||
WARN("%s() falls through to next function %s()",
|
||||
func->name, insn->func->name);
|
||||
return 1;
|
||||
|
|
@ -4113,7 +4117,8 @@ static int validate_ibt(struct objtool_file *file)
|
|||
!strcmp(sec->name, "__bug_table") ||
|
||||
!strcmp(sec->name, "__ex_table") ||
|
||||
!strcmp(sec->name, "__jump_table") ||
|
||||
!strcmp(sec->name, "__mcount_loc"))
|
||||
!strcmp(sec->name, "__mcount_loc") ||
|
||||
!strcmp(sec->name, ".kcfi_traps"))
|
||||
continue;
|
||||
|
||||
list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
|
||||
|
|
|
|||
|
|
@ -619,6 +619,11 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
|
|||
Elf64_Xword entsize = symtab->sh.sh_entsize;
|
||||
int max_idx, idx = sym->idx;
|
||||
Elf_Scn *s, *t = NULL;
|
||||
bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE &&
|
||||
sym->sym.st_shndx != SHN_XINDEX;
|
||||
|
||||
if (is_special_shndx)
|
||||
shndx = sym->sym.st_shndx;
|
||||
|
||||
s = elf_getscn(elf->elf, symtab->idx);
|
||||
if (!s) {
|
||||
|
|
@ -704,7 +709,7 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
|
|||
}
|
||||
|
||||
/* setup extended section index magic and write the symbol */
|
||||
if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) {
|
||||
if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) {
|
||||
sym->sym.st_shndx = shndx;
|
||||
if (!shndx_data)
|
||||
shndx = 0;
|
||||
|
|
|
|||
Loading…
Reference in New Issue