mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

CFI on arm64 series for v5.13-rc1 

- Clean up list_sort prototypes (Sami Tolvanen)
 
 - Introduce CONFIG_CFI_CLANG for arm64 (Sami Tolvanen)
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEpcP2jyKd1g9yPm4TiXL039xtwCYFAmCHCR8ACgkQiXL039xt
 wCZyFQ//fnUZaXR2K354zDyW6CJljMf+d94RF6rH+J6eMTH2/HXa5v0iJokwABLf
 ussP6qF4k5wtmI22Gm9A5Zc3e4iiry5pC0jOdk0mk4gzWwFN9MdgNxJZIGA3xqhS
 bsBK4AGrVKjtZl48G1/ZxJuNDeJhVp6GNK2n6/Gl4rZF6R7D/Upz0XelyJRdDpcM
 HIGma7jZl6xfGU0mdWCzpOGK1zdMca1WVs7A4YuurSbLn5PZJrcNVWLouDqt/Si2
 AduSri1gyPClicgvqWjMOzhUpuw/nJtBLRl1x1EsWk/KSZ1/uNVjlewfzdN4fZrr
 zbtFr2gLubYLK6JOX7/LqoHlOTgE3tYLL+WIVN75DsOGZBKgHhmebTmWLyqzV0SL
 oqcyM5d3ucC6msdtAK5Fv4MSp8rpjqlK1Ha4SGRT6kC2wut7AhZ3KD7eyRIz8mV9
 Sa9mhignGFJnTEUp+LSbYdrAudgSKxB40WyXPmswAXX4VJFRD4ONrrcAON/SzkUT
 Hw/JdFRCKkJjgwNQjIQoZcUNMTbFz2PlNIEnjJWm38YImQKQlCb2mXaZKCwBkf45
 aheCZk17eKoxTCXFMd+KxlyNEtS2yBfq/PpZgvw7GW/pfFbWUg1+2O41LnihIe5v
 zu0hN1wNCQqgfxiMZqX1OTb9C/2vybzGsXILt+9nppjZ8EBU7iU=
 =wU6U
 -----END PGP SIGNATURE-----

Merge tag 'cfi-v5.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux

Pull CFI on arm64 support from Kees Cook:
 "This builds on last cycle's LTO work, and allows the arm64 kernels to
  be built with Clang's Control Flow Integrity feature. This feature has
  happily lived in Android kernels for almost 3 years[1], so I'm excited
  to have it ready for upstream.

  The wide diffstat is mainly due to the treewide fixing of mismatched
  list_sort prototypes. Other things in core kernel are to address
  various CFI corner cases. The largest code portion is the CFI runtime
  implementation itself (which will be shared by all architectures
  implementing support for CFI). The arm64 pieces are Acked by arm64
  maintainers rather than coming through the arm64 tree since carrying
  this tree over there was going to be awkward.

  CFI support for x86 is still under development, but is pretty close.
  There are a handful of corner cases on x86 that need some improvements
  to Clang and objtool, but otherwise works well.

  Summary:

   - Clean up list_sort prototypes (Sami Tolvanen)

   - Introduce CONFIG_CFI_CLANG for arm64 (Sami Tolvanen)"

* tag 'cfi-v5.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux:
  arm64: allow CONFIG_CFI_CLANG to be selected
  KVM: arm64: Disable CFI for nVHE
  arm64: ftrace: use function_nocfi for ftrace_call
  arm64: add __nocfi to __apply_alternatives
  arm64: add __nocfi to functions that jump to a physical address
  arm64: use function_nocfi with __pa_symbol
  arm64: implement function_nocfi
  psci: use function_nocfi for cpu_resume
  lkdtm: use function_nocfi
  treewide: Change list_sort to use const pointers
  bpf: disable CFI in dispatcher functions
  kallsyms: strip ThinLTO hashes from static functions
  kthread: use WARN_ON_FUNCTION_MISMATCH
  workqueue: use WARN_ON_FUNCTION_MISMATCH
  module: ensure __cfi_check alignment
  mm: add generic function_nocfi macro
  cfi: add __cficanonical
  add support for Clang CFI
This commit is contained in:
Linus Torvalds 2021-04-27 10:16:46 -07:00
parent 2fbc66c783 9186ad8e66
commit 57fa2369ab
75 changed files with 760 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Makefile
View File

@ -924,6 +924,23 @@ KBUILD_AFLAGS += -fno-lto
export CC_FLAGS_LTO
endif
ifdef CONFIG_CFI_CLANG
CC_FLAGS_CFI := -fsanitize=cfi \
-fsanitize-cfi-cross-dso \
-fno-sanitize-cfi-canonical-jump-tables \
-fno-sanitize-trap=cfi \
-fno-sanitize-blacklist
ifdef CONFIG_CFI_PERMISSIVE
CC_FLAGS_CFI += -fsanitize-recover=cfi
endif
# If LTO flags are filtered out, we must also filter out CFI.
CC_FLAGS_LTO += $(CC_FLAGS_CFI)
KBUILD_CFLAGS += $(CC_FLAGS_CFI)
export CC_FLAGS_CFI
endif
ifdef CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_32B
KBUILD_CFLAGS += -falign-functions=32
endif

45
arch/Kconfig
View File

@ -692,6 +692,51 @@ config LTO_CLANG_THIN
If unsure, say Y.
endchoice
config ARCH_SUPPORTS_CFI_CLANG
bool
help
An architecture should select this option if it can support Clang's
Control-Flow Integrity (CFI) checking.
config CFI_CLANG
bool "Use Clang's Control Flow Integrity (CFI)"
depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG
# Clang >= 12:
# - https://bugs.llvm.org/show_bug.cgi?id=46258
# - https://bugs.llvm.org/show_bug.cgi?id=47479
depends on CLANG_VERSION >= 120000
select KALLSYMS
help
This option enables Clang’s forward-edge Control Flow Integrity
(CFI) checking, where the compiler injects a runtime check to each
indirect function call to ensure the target is a valid function with
the correct static type. This restricts possible call targets and
makes it more difficult for an attacker to exploit bugs that allow
the modification of stored function pointers. More information can be
found from Clang's documentation:
https://clang.llvm.org/docs/ControlFlowIntegrity.html
config CFI_CLANG_SHADOW
bool "Use CFI shadow to speed up cross-module checks"
default y
depends on CFI_CLANG && MODULES
help
If you select this option, the kernel builds a fast look-up table of
CFI check functions in loaded modules to reduce performance overhead.
If unsure, say Y.
config CFI_PERMISSIVE
bool "Use CFI in permissive mode"
depends on CFI_CLANG
help
When selected, Control Flow Integrity (CFI) violations result in a
warning instead of a kernel panic. This option should only be used
for finding indirect call type mismatches during development.
If unsure, say N.
config HAVE_ARCH_WITHIN_STACK_FRAMES
bool
help

1
arch/arm64/Kconfig
View File

@ -75,6 +75,7 @@ config ARM64
select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
select ARCH_SUPPORTS_LTO_CLANG if CPU_LITTLE_ENDIAN
select ARCH_SUPPORTS_LTO_CLANG_THIN
select ARCH_SUPPORTS_CFI_CLANG
select ARCH_SUPPORTS_ATOMIC_RMW
select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
select ARCH_SUPPORTS_NUMA_BALANCING

16
arch/arm64/include/asm/memory.h
View File

@ -323,6 +323,22 @@ static inline void *phys_to_virt(phys_addr_t x)
#define virt_to_pfn(x) __phys_to_pfn(__virt_to_phys((unsigned long)(x)))
#define sym_to_pfn(x) __phys_to_pfn(__pa_symbol(x))
#ifdef CONFIG_CFI_CLANG
/*
* With CONFIG_CFI_CLANG, the compiler replaces function address
* references with the address of the function's CFI jump table
* entry. The function_nocfi macro always returns the address of the
* actual function instead.
*/
#define function_nocfi(x) ({ \
void *addr; \
asm("adrp %0, " __stringify(x) "\n\t" \
"add %0, %0, :lo12:" __stringify(x) \
: "=r" (addr)); \
addr; \
})
#endif
/*
* virt_to_page(x) convert a _valid_ virtual address to struct page *
* virt_addr_valid(x) indicates whether a virtual address is valid

4
arch/arm64/include/asm/mmu_context.h
View File

@ -119,7 +119,7 @@ static inline void cpu_install_idmap(void)
* Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
* avoiding the possibility of conflicting TLB entries being allocated.
*/
static inline void cpu_replace_ttbr1(pgd_t *pgdp)
static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp)
{
typedef void (ttbr_replace_func)(phys_addr_t);
extern ttbr_replace_func idmap_cpu_replace_ttbr1;
@ -140,7 +140,7 @@ static inline void cpu_replace_ttbr1(pgd_t *pgdp)
ttbr1 |= TTBR_CNP_BIT;
}
replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1));
cpu_install_idmap();
replace_phys(ttbr1);

3
arch/arm64/kernel/acpi_parking_protocol.c
View File

@ -99,7 +99,8 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu)
* that read this address need to convert this address to the
* Boot-Loader's endianness before jumping.
*/
writeq_relaxed(__pa_symbol(secondary_entry), &mailbox->entry_point);
writeq_relaxed(__pa_symbol(function_nocfi(secondary_entry)),
&mailbox->entry_point);
writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id);
arch_send_wakeup_ipi_mask(cpumask_of(cpu));

4
arch/arm64/kernel/alternative.c
View File

@ -133,8 +133,8 @@ static void clean_dcache_range_nopatch(u64 start, u64 end)
} while (cur += d_size, cur < end);
}
static void __apply_alternatives(void *alt_region, bool is_module,
unsigned long *feature_mask)
static void __nocfi __apply_alternatives(void *alt_region, bool is_module,
unsigned long *feature_mask)
{
struct alt_instr *alt;
struct alt_region *region = alt_region;

10
arch/arm64/kernel/cpu-reset.h
View File

@ -13,16 +13,16 @@
void __cpu_soft_restart(unsigned long el2_switch, unsigned long entry,
unsigned long arg0, unsigned long arg1, unsigned long arg2);
static inline void __noreturn cpu_soft_restart(unsigned long entry,
unsigned long arg0,
unsigned long arg1,
unsigned long arg2)
static inline void __noreturn __nocfi cpu_soft_restart(unsigned long entry,
unsigned long arg0,
unsigned long arg1,
unsigned long arg2)
{
typeof(__cpu_soft_restart) *restart;
unsigned long el2_switch = !is_kernel_in_hyp_mode() &&
is_hyp_mode_available();
restart = (void *)__pa_symbol(__cpu_soft_restart);
restart = (void *)__pa_symbol(function_nocfi(__cpu_soft_restart));
cpu_install_idmap();
restart(el2_switch, entry, arg0, arg1, arg2);

4
arch/arm64/kernel/cpufeature.c
View File

@ -1451,7 +1451,7 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
}
#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
static void
static void __nocfi
kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
{
typedef void (kpti_remap_fn)(int, int, phys_addr_t);
@ -1468,7 +1468,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
if (arm64_use_ng_mappings)
return;
remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings));
cpu_install_idmap();
remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));

2
arch/arm64/kernel/ftrace.c
View File

@ -55,7 +55,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned long pc;
u32 new;
pc = (unsigned long)&ftrace_call;
pc = (unsigned long)function_nocfi(ftrace_call);
new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
AARCH64_INSN_BRANCH_LINK);

3
arch/arm64/kernel/psci.c
View File

@ -38,7 +38,8 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu)
static int cpu_psci_cpu_boot(unsigned int cpu)
{
int err = psci_ops.cpu_on(cpu_logical_map(cpu), __pa_symbol(secondary_entry));
phys_addr_t pa_secondary_entry = __pa_symbol(function_nocfi(secondary_entry));
int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry);
if (err)
pr_err("failed to boot CPU%d (%d)\n", cpu, err);

3
arch/arm64/kernel/smp_spin_table.c
View File

@ -66,6 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu)
static int smp_spin_table_cpu_prepare(unsigned int cpu)
{
__le64 __iomem *release_addr;
phys_addr_t pa_holding_pen = __pa_symbol(function_nocfi(secondary_holding_pen));
if (!cpu_release_addr[cpu])
return -ENODEV;
@ -88,7 +89,7 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu)
* boot-loader's endianness before jumping. This is mandated by
* the boot protocol.
*/
writeq_relaxed(__pa_symbol(secondary_holding_pen), release_addr);
writeq_relaxed(pa_holding_pen, release_addr);
__flush_dcache_area((__force void *)release_addr,
sizeof(*release_addr));

6
arch/arm64/kvm/hyp/nvhe/Makefile
View File

@ -75,9 +75,9 @@ quiet_cmd_hyprel = HYPREL $@
quiet_cmd_hypcopy = HYPCOPY $@
cmd_hypcopy = $(OBJCOPY) --prefix-symbols=__kvm_nvhe_ $< $@
# Remove ftrace and Shadow Call Stack CFLAGS.
# This is equivalent to the 'notrace' and '__noscs' annotations.
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
# Remove ftrace, Shadow Call Stack, and CFI CFLAGS.
# This is equivalent to the 'notrace', '__noscs', and '__nocfi' annotations.
KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS) $(CC_FLAGS_CFI), $(KBUILD_CFLAGS))
# KVM nVHE code is run at a different exception code with a different map, so
# compiler instrumentation that inserts callbacks or checks into the code may

8
arch/arm64/kvm/vgic/vgic-its.c
View File

@ -2190,8 +2190,8 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
return offset;
}
static int vgic_its_ite_cmp(void *priv, struct list_head *a,
struct list_head *b)
static int vgic_its_ite_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct its_ite *itea = container_of(a, struct its_ite, ite_list);
struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
@ -2329,8 +2329,8 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
return offset;
}
static int vgic_its_device_cmp(void *priv, struct list_head *a,
struct list_head *b)
static int vgic_its_device_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct its_device *deva = container_of(a, struct its_device, dev_list);
struct its_device *devb = container_of(b, struct its_device, dev_list);

3
arch/arm64/kvm/vgic/vgic.c
View File

@ -255,7 +255,8 @@ static struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq)
* Return negative if "a" sorts before "b", 0 to preserve order, and positive
* to sort "b" before "a".
*/
static int vgic_irq_cmp(void *priv, struct list_head *a, struct list_head *b)
static int vgic_irq_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct vgic_irq *irqa = container_of(a, struct vgic_irq, ap_list);
struct vgic_irq *irqb = container_of(b, struct vgic_irq, ap_list);

3
block/blk-mq-sched.c
View File

@ -75,7 +75,8 @@ void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
blk_mq_run_hw_queue(hctx, true);
}
static int sched_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
static int sched_rq_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct request *rqa = container_of(a, struct request, queuelist);
struct request *rqb = container_of(b, struct request, queuelist);

3
block/blk-mq.c
View File

@ -1895,7 +1895,8 @@ void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
spin_unlock(&ctx->lock);
}
static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
static int plug_rq_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct request *rqa = container_of(a, struct request, queuelist);
struct request *rqb = container_of(b, struct request, queuelist);

3
drivers/acpi/nfit/core.c
View File

@ -1195,7 +1195,8 @@ static int __nfit_mem_init(struct acpi_nfit_desc *acpi_desc,
return 0;
}
static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
static int nfit_mem_cmp(void *priv, const struct list_head *_a,
const struct list_head *_b)
{
struct nfit_mem *a = container_of(_a, typeof(*a), list);
struct nfit_mem *b = container_of(_b, typeof(*b), list);

3
drivers/acpi/numa/hmat.c
View File

@ -558,7 +558,8 @@ static bool hmat_update_best(u8 type, u32 value, u32 *best)
return updated;
}
static int initiator_cmp(void *priv, struct list_head *a, struct list_head *b)
static int initiator_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct memory_initiator *ia;
struct memory_initiator *ib;

4
drivers/clk/keystone/sci-clk.c
View File

@ -503,8 +503,8 @@ static int ti_sci_scan_clocks_from_fw(struct sci_clk_provider *provider)
#else
static int _cmp_sci_clk_list(void *priv, struct list_head *a,
struct list_head *b)
static int _cmp_sci_clk_list(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct sci_clk *ca = container_of(a, struct sci_clk, node);
struct sci_clk *cb = container_of(b, struct sci_clk, node);

7
drivers/firmware/psci/psci.c
View File

@ -325,8 +325,9 @@ static int __init psci_features(u32 psci_func_id)
static int psci_suspend_finisher(unsigned long state)
{
u32 power_state = state;
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
return psci_ops.cpu_suspend(power_state, __pa_symbol(cpu_resume));
return psci_ops.cpu_suspend(power_state, pa_cpu_resume);
}
int psci_cpu_suspend_enter(u32 state)
@ -344,8 +345,10 @@ int psci_cpu_suspend_enter(u32 state)
static int psci_system_suspend(unsigned long unused)
{
phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND),
__pa_symbol(cpu_resume), 0, 0);
pa_cpu_resume, 0, 0);
}
static int psci_system_suspend_enter(suspend_state_t state)

3
drivers/gpu/drm/drm_modes.c
View File

@ -1290,7 +1290,8 @@ EXPORT_SYMBOL(drm_mode_prune_invalid);
* Negative if @lh_a is better than @lh_b, zero if they're equivalent, or
* positive if @lh_b is better than @lh_a.
*/
static int drm_mode_compare(void *priv, struct list_head *lh_a, struct list_head *lh_b)
static int drm_mode_compare(void *priv, const struct list_head *lh_a,
const struct list_head *lh_b)
{
struct drm_display_mode *a = list_entry(lh_a, struct drm_display_mode, head);
struct drm_display_mode *b = list_entry(lh_b, struct drm_display_mode, head);

3
drivers/gpu/drm/i915/gt/intel_engine_user.c
View File

@ -49,7 +49,8 @@ static const u8 uabi_classes[] = {
[VIDEO_ENHANCEMENT_CLASS] = I915_ENGINE_CLASS_VIDEO_ENHANCE,
};
static int engine_cmp(void *priv, struct list_head *A, struct list_head *B)
static int engine_cmp(void *priv, const struct list_head *A,
const struct list_head *B)
{
const struct intel_engine_cs *a =
container_of((struct rb_node *)A, typeof(*a), uabi_node);

2
drivers/gpu/drm/i915/gvt/debugfs.c
View File

@ -41,7 +41,7 @@ struct diff_mmio {
/* Compare two diff_mmio items. */
static int mmio_offset_compare(void *priv,
struct list_head *a, struct list_head *b)
const struct list_head *a, const struct list_head *b)
{
struct diff_mmio *ma;
struct diff_mmio *mb;

3
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
View File

@ -1076,7 +1076,8 @@ static int igt_ppgtt_shrink_boom(void *arg)
return exercise_ppgtt(arg, shrink_boom);
}
static int sort_holes(void *priv, struct list_head *A, struct list_head *B)
static int sort_holes(void *priv, const struct list_head *A,
const struct list_head *B)
{
struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack);
struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack);

4
drivers/gpu/drm/radeon/radeon_cs.c
View File

@ -393,8 +393,8 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
return 0;
}
static int cmp_size_smaller_first(void *priv, struct list_head *a,
struct list_head *b)
static int cmp_size_smaller_first(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct radeon_bo_list *la = list_entry(a, struct radeon_bo_list, tv.head);
struct radeon_bo_list *lb = list_entry(b, struct radeon_bo_list, tv.head);

3
drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.c
View File

@ -83,7 +83,8 @@ usnic_uiom_interval_node_alloc(long int start, long int last, int ref_cnt,
return interval;
}
static int interval_cmp(void *priv, struct list_head *a, struct list_head *b)
static int interval_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct usnic_uiom_interval_node *node_a, *node_b;

2
drivers/interconnect/qcom/bcm-voter.c
View File

@ -39,7 +39,7 @@ struct bcm_voter {
u32 tcs_wait;
};
static int cmp_vcd(void *priv, struct list_head *a, struct list_head *b)
static int cmp_vcd(void *priv, const struct list_head *a, const struct list_head *b)
{
const struct qcom_icc_bcm *bcm_a = list_entry(a, struct qcom_icc_bcm, list);
const struct qcom_icc_bcm *bcm_b = list_entry(b, struct qcom_icc_bcm, list);

3
drivers/md/raid5.c
View File

@ -953,7 +953,8 @@ static void dispatch_bio_list(struct bio_list *tmp)
submit_bio_noacct(bio);
}
static int cmp_stripe(void *priv, struct list_head *a, struct list_head *b)
static int cmp_stripe(void *priv, const struct list_head *a,
const struct list_head *b)
{
const struct r5pending_data *da = list_entry(a,
struct r5pending_data, sibling);

2
drivers/misc/lkdtm/usercopy.c
View File

@ -314,7 +314,7 @@ void lkdtm_USERCOPY_KERNEL(void)
pr_info("attempting bad copy_to_user from kernel text: %px\n",
vm_mmap);
if (copy_to_user((void __user *)user_addr, vm_mmap,
if (copy_to_user((void __user *)user_addr, function_nocfi(vm_mmap),
unconst + PAGE_SIZE)) {
pr_warn("copy_to_user failed, but lacked Oops\n");
goto free_user;

4
drivers/misc/sram.c
View File

@ -144,8 +144,8 @@ static void sram_free_partitions(struct sram_dev *sram)
}
}
static int sram_reserve_cmp(void *priv, struct list_head *a,
struct list_head *b)
static int sram_reserve_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct sram_reserve *ra = list_entry(a, struct sram_reserve, list);
struct sram_reserve *rb = list_entry(b, struct sram_reserve, list);

3
drivers/nvme/host/core.c
View File

@ -3855,7 +3855,8 @@ out_unlock:
return ret;
}
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
static int ns_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

3
drivers/pci/controller/cadence/pcie-cadence-host.c
View File

@ -345,7 +345,8 @@ static int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc,
return 0;
}
static int cdns_pcie_host_dma_ranges_cmp(void *priv, struct list_head *a, struct list_head *b)
static int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct resource_entry *entry1, *entry2;

3
drivers/spi/spi-loopback-test.c
View File

@ -454,7 +454,8 @@ struct rx_ranges {
u8 *end;
};
static int rx_ranges_cmp(void *priv, struct list_head *a, struct list_head *b)
static int rx_ranges_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct rx_ranges *rx_a = list_entry(a, struct rx_ranges, list);
struct rx_ranges *rx_b = list_entry(b, struct rx_ranges, list);

3
fs/btrfs/raid56.c
View File

@ -1633,7 +1633,8 @@ struct btrfs_plug_cb {
/*
* rbios on the plug list are sorted for easier merging.
*/
static int plug_cmp(void *priv, struct list_head *a, struct list_head *b)
static int plug_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct btrfs_raid_bio *ra = container_of(a, struct btrfs_raid_bio,
plug_list);

3
fs/btrfs/tree-log.c
View File

@ -4138,7 +4138,8 @@ static noinline int copy_items(struct btrfs_trans_handle *trans,
return ret;
}
static int extent_cmp(void *priv, struct list_head *a, struct list_head *b)
static int extent_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct extent_map *em1, *em2;

3
fs/btrfs/volumes.c
View File

@ -1224,7 +1224,8 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
return 0;
}
static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
static int devid_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct btrfs_device *dev1, *dev2;

4
fs/ext4/fsmap.c
View File

@ -354,8 +354,8 @@ static unsigned int ext4_getfsmap_find_sb(struct super_block *sb,
/* Compare two fsmap items. */
static int ext4_getfsmap_compare(void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct ext4_fsmap *fa;
struct ext4_fsmap *fb;

3
fs/gfs2/glock.c
View File

@ -1732,7 +1732,8 @@ void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
spin_unlock(&gl->gl_lockref.lock);
}
static int glock_cmp(void *priv, struct list_head *a, struct list_head *b)
static int glock_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct gfs2_glock *gla, *glb;

2
fs/gfs2/log.c
View File

@ -695,7 +695,7 @@ void log_flush_wait(struct gfs2_sbd *sdp)
}
}
static int ip_cmp(void *priv, struct list_head *a, struct list_head *b)
static int ip_cmp(void *priv, const struct list_head *a, const struct list_head *b)
{
struct gfs2_inode *ipa, *ipb;

3
fs/gfs2/lops.c
View File

@ -634,7 +634,8 @@ static void gfs2_check_magic(struct buffer_head *bh)
kunmap_atomic(kaddr);
}
static int blocknr_cmp(void *priv, struct list_head *a, struct list_head *b)
static int blocknr_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct gfs2_bufdata *bda, *bdb;

3
fs/iomap/buffered-io.c
View File

@ -1155,7 +1155,8 @@ iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends,
EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
static int
iomap_ioend_compare(void *priv, struct list_head *a, struct list_head *b)
iomap_ioend_compare(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);

7
fs/ubifs/gc.c
View File

@ -102,7 +102,8 @@ static int switch_gc_head(struct ubifs_info *c)
* This function compares data nodes @a and @b. Returns %1 if @a has greater
* inode or block number, and %-1 otherwise.
*/
static int data_nodes_cmp(void *priv, struct list_head *a, struct list_head *b)
static int data_nodes_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
ino_t inuma, inumb;
struct ubifs_info *c = priv;
@ -145,8 +146,8 @@ static int data_nodes_cmp(void *priv, struct list_head *a, struct list_head *b)
* first and sorted by length in descending order. Directory entry nodes go
* after inode nodes and are sorted in ascending hash valuer order.
*/
static int nondata_nodes_cmp(void *priv, struct list_head *a,
struct list_head *b)
static int nondata_nodes_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
ino_t inuma, inumb;
struct ubifs_info *c = priv;

4
fs/ubifs/replay.c
View File

@ -298,8 +298,8 @@ static int apply_replay_entry(struct ubifs_info *c, struct replay_entry *r)
* entries @a and @b by comparing their sequence numer. Returns %1 if @a has
* greater sequence number and %-1 otherwise.
*/
static int replay_entries_cmp(void *priv, struct list_head *a,
struct list_head *b)
static int replay_entries_cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct ubifs_info *c = priv;
struct replay_entry *ra, *rb;

4
fs/xfs/scrub/bitmap.c
View File

@ -63,8 +63,8 @@ xbitmap_init(
static int
xbitmap_range_cmp(
void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct xbitmap_range *ap;
struct xbitmap_range *bp;

4
fs/xfs/xfs_bmap_item.c
View File

@ -265,8 +265,8 @@ xfs_trans_log_finish_bmap_update(
static int
xfs_bmap_update_diff_items(
void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct xfs_bmap_intent *ba;
struct xfs_bmap_intent *bb;

6
fs/xfs/xfs_buf.c
View File

@ -2124,9 +2124,9 @@ xfs_buf_delwri_queue(
*/
static int
xfs_buf_cmp(
void *priv,
struct list_head *a,
struct list_head *b)
void *priv,
const struct list_head *a,
const struct list_head *b)
{
struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);

4
fs/xfs/xfs_extent_busy.c
View File

@ -629,8 +629,8 @@ xfs_extent_busy_wait_all(
int
xfs_extent_busy_ag_cmp(
void *priv,
struct list_head *l1,
struct list_head *l2)
const struct list_head *l1,
const struct list_head *l2)
{
struct xfs_extent_busy *b1 =
container_of(l1, struct xfs_extent_busy, list);

3
fs/xfs/xfs_extent_busy.h
View File

@ -58,7 +58,8 @@ void
xfs_extent_busy_wait_all(struct xfs_mount *mp);
int
xfs_extent_busy_ag_cmp(void *priv, struct list_head *a, struct list_head *b);
xfs_extent_busy_ag_cmp(void *priv, const struct list_head *a,
const struct list_head *b);
static inline void xfs_extent_busy_sort(struct list_head *list)
{

4
fs/xfs/xfs_extfree_item.c
View File

@ -397,8 +397,8 @@ xfs_trans_free_extent(
static int
xfs_extent_free_diff_items(
void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct xfs_mount *mp = priv;
struct xfs_extent_free_item *ra;

4
fs/xfs/xfs_refcount_item.c
View File

@ -269,8 +269,8 @@ xfs_trans_log_finish_refcount_update(
static int
xfs_refcount_update_diff_items(
void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct xfs_mount *mp = priv;
struct xfs_refcount_intent *ra;

4
fs/xfs/xfs_rmap_item.c
View File

@ -337,8 +337,8 @@ xfs_trans_log_finish_rmap_update(
static int
xfs_rmap_update_diff_items(
void *priv,
struct list_head *a,
struct list_head *b)
const struct list_head *a,
const struct list_head *b)
{
struct xfs_mount *mp = priv;
struct xfs_rmap_intent *ra;

16
include/asm-generic/bug.h
View File

@ -241,6 +241,22 @@ void __warn(const char *file, int line, void *caller, unsigned taint,
# define WARN_ON_SMP(x) ({0;})
#endif
/*
* WARN_ON_FUNCTION_MISMATCH() warns if a value doesn't match a
* function address, and can be useful for catching issues with
* callback functions, for example.
*
* With CONFIG_CFI_CLANG, the warning is disabled because the
* compiler replaces function addresses taken in C code with
* local jump table addresses, which breaks cross-module function
* address equality.
*/
#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_MODULES)
# define WARN_ON_FUNCTION_MISMATCH(x, fn) ({ 0; })
#else
# define WARN_ON_FUNCTION_MISMATCH(x, fn) WARN_ON_ONCE((x) != (fn))
#endif
#endif /* __ASSEMBLY__ */
#endif

20
include/asm-generic/vmlinux.lds.h
View File

@ -544,6 +544,22 @@
. = ALIGN((align)); \
__end_rodata = .;
/*
* .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI)
* jump table entries.
*/
#ifdef CONFIG_CFI_CLANG
#define TEXT_CFI_JT \
. = ALIGN(PMD_SIZE); \
__cfi_jt_start = .; \
*(.text..L.cfi.jumptable .text..L.cfi.jumptable.*) \
. = ALIGN(PMD_SIZE); \
__cfi_jt_end = .;
#else
#define TEXT_CFI_JT
#endif
/*
* Non-instrumentable text section
*/
@ -570,6 +586,7 @@
NOINSTR_TEXT \
*(.text..refcount) \
*(.ref.text) \
TEXT_CFI_JT \
MEM_KEEP(init.text*) \
MEM_KEEP(exit.text*) \
@ -974,7 +991,8 @@
* keep any .init_array.* sections.
* https://bugs.llvm.org/show_bug.cgi?id=46478
*/
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \
defined(CONFIG_CFI_CLANG)
# ifdef CONFIG_CONSTRUCTORS
# define SANITIZER_DISCARDS \
*(.eh_frame)

4
include/linux/bpf.h
View File

@ -652,7 +652,7 @@ struct bpf_dispatcher {
struct bpf_ksym ksym;
};
static __always_inline unsigned int bpf_dispatcher_nop_func(
static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func(
const void *ctx,
const struct bpf_insn *insnsi,
unsigned int (*bpf_func)(const void *,
@ -680,7 +680,7 @@ void bpf_trampoline_put(struct bpf_trampoline *tr);
}
#define DEFINE_BPF_DISPATCHER(name) \
noinline unsigned int bpf_dispatcher_##name##_func( \
noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \
const void *ctx, \
const struct bpf_insn *insnsi, \
unsigned int (*bpf_func)(const void *, \

41
include/linux/cfi.h Normal file
View File

@ -0,0 +1,41 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Clang Control Flow Integrity (CFI) support.
*
* Copyright (C) 2021 Google LLC
*/
#ifndef _LINUX_CFI_H
#define _LINUX_CFI_H
#ifdef CONFIG_CFI_CLANG
typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag);
/* Compiler-generated function in each module, and the kernel */
extern void __cfi_check(uint64_t id, void *ptr, void *diag);
/*
* Force the compiler to generate a CFI jump table entry for a function
* and store the jump table address to __cfi_jt_<function>.
*/
#define __CFI_ADDRESSABLE(fn, __attr) \
const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
#ifdef CONFIG_CFI_CLANG_SHADOW
extern void cfi_module_add(struct module *mod, unsigned long base_addr);
extern void cfi_module_remove(struct module *mod, unsigned long base_addr);
#else
static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {}
static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {}
#endif /* CONFIG_CFI_CLANG_SHADOW */
#else /* !CONFIG_CFI_CLANG */
#define __CFI_ADDRESSABLE(fn, __attr)
#endif /* CONFIG_CFI_CLANG */
#endif /* _LINUX_CFI_H */

3
include/linux/compiler-clang.h
View File

@ -61,3 +61,6 @@
#if __has_feature(shadow_call_stack)
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
#endif
#define __nocfi __attribute__((__no_sanitize__("cfi")))
#define __cficanonical __attribute__((__cfi_canonical_jump_table__))

8
include/linux/compiler_types.h
View File

@ -242,6 +242,14 @@ struct ftrace_likely_data {
# define __noscs
#endif
#ifndef __nocfi
# define __nocfi
#endif
#ifndef __cficanonical
# define __cficanonical
#endif
#ifndef asm_volatile_goto
#define asm_volatile_goto(x...) asm goto(x)
#endif

6
include/linux/init.h
View File

@ -47,7 +47,7 @@
/* These are for everybody (although not all archs will actually
discard it in modules) */
#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline
#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline __nocfi
#define __initdata __section(".init.data")
#define __initconst __section(".init.rodata")
#define __exitdata __section(".exit.data")
@ -220,8 +220,8 @@ extern bool initcall_debug;
__initcall_name(initstub, __iid, id)
#define __define_initcall_stub(__stub, fn) \
int __init __stub(void); \
int __init __stub(void) \
int __init __cficanonical __stub(void); \
int __init __cficanonical __stub(void) \
{ \
return fn(); \
} \

7
include/linux/list_sort.h
View File

@ -6,8 +6,9 @@
struct list_head;
typedef int __attribute__((nonnull(2,3))) (*list_cmp_func_t)(void *,
const struct list_head *, const struct list_head *);
__attribute__((nonnull(2,3)))
void list_sort(void *priv, struct list_head *head,
int (*cmp)(void *priv, struct list_head *a,
struct list_head *b));
void list_sort(void *priv, struct list_head *head, list_cmp_func_t cmp);
#endif

10
include/linux/mm.h
View File

@ -124,6 +124,16 @@ extern int mmap_rnd_compat_bits __read_mostly;
#define lm_alias(x) __va(__pa_symbol(x))
#endif
/*
* With CONFIG_CFI_CLANG, the compiler replaces function addresses in
* instrumented C code with jump table addresses. Architectures that
* support CFI can define this macro to return the actual function address
* when needed.
*/
#ifndef function_nocfi
#define function_nocfi(x) (x)
#endif
/*
* To prevent common memory management code establishing
* a zero page mapping on a read fault.

13
include/linux/module.h
View File

@ -26,6 +26,7 @@
#include <linux/tracepoint-defs.h>
#include <linux/srcu.h>
#include <linux/static_call_types.h>
#include <linux/cfi.h>
#include <linux/percpu.h>
#include <asm/module.h>
@ -128,13 +129,17 @@ extern void cleanup_module(void);
#define module_init(initfn) \
static inline initcall_t __maybe_unused __inittest(void) \
{ return initfn; } \
int init_module(void) __copy(initfn) __attribute__((alias(#initfn)));
int init_module(void) __copy(initfn) \
__attribute__((alias(#initfn))); \
__CFI_ADDRESSABLE(init_module, __initdata);
/* This is only required if you want to be unloadable. */
#define module_exit(exitfn) \
static inline exitcall_t __maybe_unused __exittest(void) \
{ return exitfn; } \
void cleanup_module(void) __copy(exitfn) __attribute__((alias(#exitfn)));
void cleanup_module(void) __copy(exitfn) \
__attribute__((alias(#exitfn))); \
__CFI_ADDRESSABLE(cleanup_module, __exitdata);
#endif
@ -376,6 +381,10 @@ struct module {
const s32 *crcs;
unsigned int num_syms;
#ifdef CONFIG_CFI_CLANG
cfi_check_fn cfi_check;
#endif
/* Kernel parameters. */
#ifdef CONFIG_SYSFS
struct mutex param_lock;

4
include/linux/pci.h
View File

@ -1944,8 +1944,8 @@ enum pci_fixup_pass {
#ifdef CONFIG_LTO_CLANG
#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \
class_shift, hook, stub) \
void stub(struct pci_dev *dev); \
void stub(struct pci_dev *dev) \
void __cficanonical stub(struct pci_dev *dev); \
void __cficanonical stub(struct pci_dev *dev) \
{ \
hook(dev); \
} \

2
init/Kconfig
View File

@ -2296,7 +2296,7 @@ endif # MODULES
config MODULES_TREE_LOOKUP
def_bool y
depends on PERF_EVENTS || TRACING
depends on PERF_EVENTS || TRACING || CFI_CLANG
config INIT_ALL_POSSIBLE
bool

4
kernel/Makefile
View File

@ -41,6 +41,9 @@ KCSAN_SANITIZE_kcov.o := n
UBSAN_SANITIZE_kcov.o := n
CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector
# Don't instrument error handlers
CFLAGS_REMOVE_cfi.o := $(CC_FLAGS_CFI)
obj-y += sched/
obj-y += locking/
obj-y += power/
@ -111,6 +114,7 @@ obj-$(CONFIG_BPF) += bpf/
obj-$(CONFIG_KCSAN) += kcsan/
obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o
obj-$(CONFIG_CFI_CLANG) += cfi.o
obj-$(CONFIG_PERF_EVENTS) += events/

329
kernel/cfi.c Normal file
View File

@ -0,0 +1,329 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Clang Control Flow Integrity (CFI) error and slowpath handling.
*
* Copyright (C) 2021 Google LLC
*/
#include <linux/hardirq.h>
#include <linux/kallsyms.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/printk.h>
#include <linux/ratelimit.h>
#include <linux/rcupdate.h>
#include <linux/vmalloc.h>
#include <asm/cacheflush.h>
#include <asm/set_memory.h>
/* Compiler-defined handler names */
#ifdef CONFIG_CFI_PERMISSIVE
#define cfi_failure_handler __ubsan_handle_cfi_check_fail
#else
#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort
#endif
static inline void handle_cfi_failure(void *ptr)
{
if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
else
panic("CFI failure (target: %pS)\n", ptr);
}
#ifdef CONFIG_MODULES
#ifdef CONFIG_CFI_CLANG_SHADOW
/*
* Index type. A 16-bit index can address at most (2^16)-2 pages (taking
* into account SHADOW_INVALID), i.e. ~256M with 4k pages.
*/
typedef u16 shadow_t;
#define SHADOW_INVALID ((shadow_t)~0UL)
struct cfi_shadow {
/* Page index for the beginning of the shadow */
unsigned long base;
/* An array of __cfi_check locations (as indices to the shadow) */
shadow_t shadow[1];
} __packed;
/*
* The shadow covers ~128M from the beginning of the module region. If
* the region is larger, we fall back to __module_address for the rest.
*/
#define __SHADOW_RANGE (_UL(SZ_128M) >> PAGE_SHIFT)
/* The in-memory size of struct cfi_shadow, always at least one page */
#define __SHADOW_PAGES ((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT)
#define SHADOW_PAGES max(1UL, __SHADOW_PAGES)
#define SHADOW_SIZE (SHADOW_PAGES << PAGE_SHIFT)
/* The actual size of the shadow array, minus metadata */
#define SHADOW_ARR_SIZE (SHADOW_SIZE - offsetof(struct cfi_shadow, shadow))
#define SHADOW_ARR_SLOTS (SHADOW_ARR_SIZE / sizeof(shadow_t))
static DEFINE_MUTEX(shadow_update_lock);
static struct cfi_shadow __rcu *cfi_shadow __read_mostly;
/* Returns the index in the shadow for the given address */
static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
{
unsigned long index;
unsigned long page = ptr >> PAGE_SHIFT;
if (unlikely(page < s->base))
return -1; /* Outside of module area */
index = page - s->base;
if (index >= SHADOW_ARR_SLOTS)
return -1; /* Cannot be addressed with shadow */
return (int)index;
}
/* Returns the page address for an index in the shadow */
static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
int index)
{
if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
return 0;
return (s->base + index) << PAGE_SHIFT;
}
/* Returns the __cfi_check function address for the given shadow location */
static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s,
int index)
{
if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
return 0;
if (unlikely(s->shadow[index] == SHADOW_INVALID))
return 0;
/* __cfi_check is always page aligned */
return (s->base + s->shadow[index]) << PAGE_SHIFT;
}
static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
struct cfi_shadow *next)
{
int i, index, check;
/* Mark everything invalid */
memset(next->shadow, 0xFF, SHADOW_ARR_SIZE);
if (!prev)
return; /* No previous shadow */
/* If the base address didn't change, an update is not needed */
if (prev->base == next->base) {
memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE);
return;
}
/* Convert the previous shadow to the new address range */
for (i = 0; i < SHADOW_ARR_SLOTS; ++i) {
if (prev->shadow[i] == SHADOW_INVALID)
continue;
index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
if (index < 0)
continue;
check = ptr_to_shadow(next,
shadow_to_check_fn(prev, prev->shadow[i]));
if (check < 0)
continue;
next->shadow[index] = (shadow_t)check;
}
}
static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod,
unsigned long min_addr, unsigned long max_addr)
{
int check_index;
unsigned long check = (unsigned long)mod->cfi_check;
unsigned long ptr;
if (unlikely(!PAGE_ALIGNED(check))) {
pr_warn("cfi: not using shadow for module %s\n", mod->name);
return;
}
check_index = ptr_to_shadow(s, check);
if (check_index < 0)
return; /* Module not addressable with shadow */
/* For each page, store the check function index in the shadow */
for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
int index = ptr_to_shadow(s, ptr);
if (index >= 0) {
/* Each page must only contain one module */
WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID);
s->shadow[index] = (shadow_t)check_index;
}
}
}
static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod,
unsigned long min_addr, unsigned long max_addr)
{
unsigned long ptr;
for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
int index = ptr_to_shadow(s, ptr);
if (index >= 0)
s->shadow[index] = SHADOW_INVALID;
}
}
typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *,
unsigned long min_addr, unsigned long max_addr);
static void update_shadow(struct module *mod, unsigned long base_addr,
update_shadow_fn fn)
{
struct cfi_shadow *prev;
struct cfi_shadow *next;
unsigned long min_addr, max_addr;
next = vmalloc(SHADOW_SIZE);
mutex_lock(&shadow_update_lock);
prev = rcu_dereference_protected(cfi_shadow,
mutex_is_locked(&shadow_update_lock));
if (next) {
next->base = base_addr >> PAGE_SHIFT;
prepare_next_shadow(prev, next);
min_addr = (unsigned long)mod->core_layout.base;
max_addr = min_addr + mod->core_layout.text_size;
fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK);
set_memory_ro((unsigned long)next, SHADOW_PAGES);
}
rcu_assign_pointer(cfi_shadow, next);
mutex_unlock(&shadow_update_lock);
synchronize_rcu();
if (prev) {
set_memory_rw((unsigned long)prev, SHADOW_PAGES);
vfree(prev);
}
}
void cfi_module_add(struct module *mod, unsigned long base_addr)
{
update_shadow(mod, base_addr, add_module_to_shadow);
}
void cfi_module_remove(struct module *mod, unsigned long base_addr)
{
update_shadow(mod, base_addr, remove_module_from_shadow);
}
static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
unsigned long ptr)
{
int index;
if (unlikely(!s))
return NULL; /* No shadow available */
index = ptr_to_shadow(s, ptr);
if (index < 0)
return NULL; /* Cannot be addressed with shadow */
return (cfi_check_fn)shadow_to_check_fn(s, index);
}
static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
{
cfi_check_fn fn;
rcu_read_lock_sched();
fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr);
rcu_read_unlock_sched();
return fn;
}
#else /* !CONFIG_CFI_CLANG_SHADOW */
static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
{
return NULL;
}
#endif /* CONFIG_CFI_CLANG_SHADOW */
static inline cfi_check_fn find_module_check_fn(unsigned long ptr)
{
cfi_check_fn fn = NULL;
struct module *mod;
rcu_read_lock_sched();
mod = __module_address(ptr);
if (mod)
fn = mod->cfi_check;
rcu_read_unlock_sched();
return fn;
}
static inline cfi_check_fn find_check_fn(unsigned long ptr)
{
cfi_check_fn fn = NULL;
if (is_kernel_text(ptr))
return __cfi_check;
/*
* Indirect call checks can happen when RCU is not watching. Both
* the shadow and __module_address use RCU, so we need to wake it
* up if necessary.
*/
RCU_NONIDLE({
if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW))
fn = find_shadow_check_fn(ptr);
if (!fn)
fn = find_module_check_fn(ptr);
});
return fn;
}
void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
{
cfi_check_fn fn = find_check_fn((unsigned long)ptr);
if (likely(fn))
fn(id, ptr, diag);
else /* Don't allow unchecked modules */
handle_cfi_failure(ptr);
}
EXPORT_SYMBOL(__cfi_slowpath_diag);
#else /* !CONFIG_MODULES */
void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
{
handle_cfi_failure(ptr); /* No modules */
}
EXPORT_SYMBOL(__cfi_slowpath_diag);
#endif /* CONFIG_MODULES */
void cfi_failure_handler(void *data, void *ptr, void *vtable)
{
handle_cfi_failure(ptr);
}
EXPORT_SYMBOL(cfi_failure_handler);

55
kernel/kallsyms.c
View File

@ -161,6 +161,27 @@ static unsigned long kallsyms_sym_address(int idx)
return kallsyms_relative_base - 1 - kallsyms_offsets[idx];
}
#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_LTO_CLANG_THIN)
/*
* LLVM appends a hash to static function names when ThinLTO and CFI are
* both enabled, i.e. foo() becomes foo$707af9a22804d33c81801f27dcfe489b.
* This causes confusion and potentially breaks user space tools, so we
* strip the suffix from expanded symbol names.
*/
static inline bool cleanup_symbol_name(char *s)
{
char *res;
res = strrchr(s, '$');
if (res)
*res = '\0';
return res != NULL;
}
#else
static inline bool cleanup_symbol_name(char *s) { return false; }
#endif
/* Lookup the address for this symbol. Returns 0 if not found. */
unsigned long kallsyms_lookup_name(const char *name)
{
@ -173,6 +194,9 @@ unsigned long kallsyms_lookup_name(const char *name)
if (strcmp(namebuf, name) == 0)
return kallsyms_sym_address(i);
if (cleanup_symbol_name(namebuf) && strcmp(namebuf, name) == 0)
return kallsyms_sym_address(i);
}
return module_kallsyms_lookup_name(name);
}
@ -303,7 +327,9 @@ const char *kallsyms_lookup(unsigned long addr,
namebuf, KSYM_NAME_LEN);
if (modname)
*modname = NULL;
return namebuf;
ret = namebuf;
goto found;
}
/* See if it's in a module or a BPF JITed image. */
@ -316,11 +342,16 @@ const char *kallsyms_lookup(unsigned long addr,
if (!ret)
ret = ftrace_mod_address_lookup(addr, symbolsize,
offset, modname, namebuf);
found:
cleanup_symbol_name(namebuf);
return ret;
}
int lookup_symbol_name(unsigned long addr, char *symname)
{
int res;
symname[0] = '\0';
symname[KSYM_NAME_LEN - 1] = '\0';
@ -331,15 +362,23 @@ int lookup_symbol_name(unsigned long addr, char *symname)
/* Grab name */
kallsyms_expand_symbol(get_symbol_offset(pos),
symname, KSYM_NAME_LEN);
return 0;
goto found;
}
/* See if it's in a module. */
return lookup_module_symbol_name(addr, symname);
res = lookup_module_symbol_name(addr, symname);
if (res)
return res;
found:
cleanup_symbol_name(symname);
return 0;
}
int lookup_symbol_attrs(unsigned long addr, unsigned long *size,
unsigned long *offset, char *modname, char *name)
{
int res;
name[0] = '\0';
name[KSYM_NAME_LEN - 1] = '\0';
@ -351,10 +390,16 @@ int lookup_symbol_attrs(unsigned long addr, unsigned long *size,
kallsyms_expand_symbol(get_symbol_offset(pos),
name, KSYM_NAME_LEN);
modname[0] = '\0';
return 0;
goto found;
}
/* See if it's in a module. */
return lookup_module_symbol_attrs(addr, size, offset, modname, name);
res = lookup_module_symbol_attrs(addr, size, offset, modname, name);
if (res)
return res;
found:
cleanup_symbol_name(name);
return 0;
}
/* Look up a kernel symbol and return it in a text buffer. */

3
kernel/kthread.c
View File

@ -963,7 +963,8 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker,
struct timer_list *timer = &dwork->timer;
struct kthread_work *work = &dwork->work;
WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
WARN_ON_FUNCTION_MISMATCH(timer->function,
kthread_delayed_work_timer_fn);
/*
* If @delay is 0, queue @dwork->work immediately. This is for

43
kernel/module.c
View File

@ -2146,6 +2146,8 @@ void __weak module_arch_freeing_init(struct module *mod)
{
}
static void cfi_cleanup(struct module *mod);
/* Free a module, remove from lists, etc. */
static void free_module(struct module *mod)
{
@ -2187,6 +2189,9 @@ static void free_module(struct module *mod)
synchronize_rcu();
mutex_unlock(&module_mutex);
/* Clean up CFI for the module. */
cfi_cleanup(mod);
/* This may be empty, but that's OK */
module_arch_freeing_init(mod);
module_memfree(mod->init_layout.base);
@ -3866,6 +3871,8 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname,
return 0;
}
static void cfi_init(struct module *mod);
/*
* Allocate and load the module: note that size of section 0 is always
* zero, and we rely on this for optional sections.
@ -3997,6 +4004,9 @@ static int load_module(struct load_info *info, const char __user *uargs,
flush_module_icache(mod);
/* Setup CFI for the module. */
cfi_init(mod);
/* Now copy in args */
mod->args = strndup_user(uargs, ~0UL >> 1);
if (IS_ERR(mod->args)) {
@ -4070,6 +4080,7 @@ static int load_module(struct load_info *info, const char __user *uargs,
synchronize_rcu();
kfree(mod->args);
free_arch_cleanup:
cfi_cleanup(mod);
module_arch_cleanup(mod);
free_modinfo:
free_modinfo(mod);
@ -4415,6 +4426,38 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
#endif /* CONFIG_LIVEPATCH */
#endif /* CONFIG_KALLSYMS */
static void cfi_init(struct module *mod)
{
#ifdef CONFIG_CFI_CLANG
initcall_t *init;
exitcall_t *exit;
rcu_read_lock_sched();
mod->cfi_check = (cfi_check_fn)
find_kallsyms_symbol_value(mod, "__cfi_check");
init = (initcall_t *)
find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
exit = (exitcall_t *)
find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
rcu_read_unlock_sched();
/* Fix init/exit functions to point to the CFI jump table */
if (init)
mod->init = *init;
if (exit)
mod->exit = *exit;
cfi_module_add(mod, module_addr_min);
#endif
}
static void cfi_cleanup(struct module *mod)
{
#ifdef CONFIG_CFI_CLANG
cfi_module_remove(mod, module_addr_min);
#endif
}
/* Maximum number of characters written by module_flags() */
#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)

2
kernel/workqueue.c
View File

@ -1630,7 +1630,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
struct work_struct *work = &dwork->work;
WARN_ON_ONCE(!wq);
WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn);
WARN_ON_ONCE(timer_pending(timer));
WARN_ON_ONCE(!list_empty(&work->entry));

17
lib/list_sort.c
View File

@ -7,16 +7,13 @@
#include <linux/list_sort.h>
#include <linux/list.h>
typedef int __attribute__((nonnull(2,3))) (*cmp_func)(void *,
struct list_head const *, struct list_head const *);
/*
* Returns a list organized in an intermediate format suited
* to chaining of merge() calls: null-terminated, no reserved or
* sentinel head node, "prev" links not maintained.
*/
__attribute__((nonnull(2,3,4)))
static struct list_head *merge(void *priv, cmp_func cmp,
static struct list_head *merge(void *priv, list_cmp_func_t cmp,
struct list_head *a, struct list_head *b)
{
struct list_head *head, **tail = &head;
@ -52,7 +49,7 @@ static struct list_head *merge(void *priv, cmp_func cmp,
* throughout.
*/
__attribute__((nonnull(2,3,4,5)))
static void merge_final(void *priv, cmp_func cmp, struct list_head *head,
static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head,
struct list_head *a, struct list_head *b)
{
struct list_head *tail = head;
@ -185,9 +182,7 @@ static void merge_final(void *priv, cmp_func cmp, struct list_head *head,
* 2^(k+1) - 1 (second merge of case 5 when x == 2^(k-1) - 1).
*/
__attribute__((nonnull(2,3)))
void list_sort(void *priv, struct list_head *head,
int (*cmp)(void *priv, struct list_head *a,
struct list_head *b))
void list_sort(void *priv, struct list_head *head, list_cmp_func_t cmp)
{
struct list_head *list = head->next, *pending = NULL;
size_t count = 0; /* Count of pending */
@ -227,7 +222,7 @@ void list_sort(void *priv, struct list_head *head,
if (likely(bits)) {
struct list_head *a = *tail, *b = a->prev;
a = merge(priv, (cmp_func)cmp, b, a);
a = merge(priv, cmp, b, a);
/* Install the merged result in place of the inputs */
a->prev = b->prev;
*tail = a;
@ -249,10 +244,10 @@ void list_sort(void *priv, struct list_head *head,
if (!next)
break;
list = merge(priv, (cmp_func)cmp, pending, list);
list = merge(priv, cmp, pending, list);
pending = next;
}
/* The final merge, rebuilding prev links */
merge_final(priv, (cmp_func)cmp, head, pending, list);
merge_final(priv, cmp, head, pending, list);
}
EXPORT_SYMBOL(list_sort);

3
lib/test_list_sort.c
View File

@ -56,7 +56,8 @@ static int __init check(struct debug_el *ela, struct debug_el *elb)
return 0;
}
static int __init cmp(void *priv, struct list_head *a, struct list_head *b)
static int __init cmp(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct debug_el *ela, *elb;

4
net/tipc/name_table.c
View File

@ -397,8 +397,8 @@ static struct publication *tipc_service_remove_publ(struct service_range *sr,
* Code reused: time_after32() for the same purpose
*/
#define publication_after(pa, pb) time_after32((pa)->id, (pb)->id)
static int tipc_publ_sort(void *priv, struct list_head *a,
struct list_head *b)
static int tipc_publ_sort(void *priv, const struct list_head *a,
const struct list_head *b)
{
struct publication *pa, *pb;

2
scripts/Makefile.modfinal
View File

@ -23,7 +23,7 @@ modname = $(notdir $(@:.mod.o=))
part-of-module = y
quiet_cmd_cc_o_c = CC [M] $@
cmd_cc_o_c = $(CC) $(c_flags) -c -o $@ $<
cmd_cc_o_c = $(CC) $(filter-out $(CC_FLAGS_CFI), $(c_flags)) -c -o $@ $<
%.mod.o: %.mod.c FORCE
$(call if_changed_dep,cc_o_c)

19
scripts/module.lds.S
View File

@ -3,10 +3,20 @@
* Archs are free to supply their own linker scripts. ld will
* combine them automatically.
*/
#ifdef CONFIG_CFI_CLANG
# include <asm/page.h>
# define ALIGN_CFI ALIGN(PAGE_SIZE)
# define SANITIZER_DISCARDS *(.eh_frame)
#else
# define ALIGN_CFI
# define SANITIZER_DISCARDS
#endif
SECTIONS {
/DISCARD/ : {
*(.discard)
*(.discard.*)
SANITIZER_DISCARDS
}
__ksymtab 0 : { *(SORT(___ksymtab+*)) }
@ -41,7 +51,14 @@ SECTIONS {
*(.rodata..L*)
}
.text : { *(.text .text.[0-9a-zA-Z_]*) }
/*
* With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning
* of the .text section, and is aligned to PAGE_SIZE.
*/
.text : ALIGN_CFI {
*(.text.__cfi_check)
*(.text .text.[0-9a-zA-Z_]* .text..L.cfi*)
}
#endif
}