mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-27 12:57:53 +00:00
29083fd84d
When booting with 'kasan.vmalloc=off', a kernel configured with support
for KASAN_HW_TAGS will explode at boot time due to bogus use of
virt_to_page() on a vmalloc adddress. With CONFIG_DEBUG_VIRTUAL selected
this will be reported explicitly, and with or without CONFIG_DEBUG_VIRTUAL
the kernel will dereference a bogus address:
| ------------[ cut here ]------------
| virt_to_phys used for non-linear address: (____ptrval____) (0xffff800008000000)
| WARNING: CPU: 0 PID: 0 at arch/arm64/mm/physaddr.c:15 __virt_to_phys+0x78/0x80
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 600000c5 (nZCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __virt_to_phys+0x78/0x80
| lr : __virt_to_phys+0x78/0x80
| sp : ffffcd076afd3c80
| x29: ffffcd076afd3c80 x28: 0068000000000f07 x27: ffff800008000000
| x26: fffffbfff0000000 x25: fffffbffff000000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000003
| x11: 00000000ffffefff x10: c0000000ffffefff x9 : 0000000000000000
| x8 : 0000000000000000 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : 000000000000004f
| Call trace:
| __virt_to_phys+0x78/0x80
| __kasan_unpoison_vmalloc+0xd4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| ---[ end trace 0000000000000000 ]---
| Unable to handle kernel paging request at virtual address 03fffacbe27b8000
| Mem abort info:
| ESR = 0x0000000096000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| FSC = 0x04: level 0 translation fault
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000000041bc5000
| [03fffacbe27b8000] pgd=0000000000000000, p4d=0000000000000000
| Internal error: Oops: 0000000096000004 [#1] PREEMPT SMP
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 200000c5 (nzCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __kasan_unpoison_vmalloc+0xe4/0x478
| lr : __kasan_unpoison_vmalloc+0xd4/0x478
| sp : ffffcd076afd3ca0
| x29: ffffcd076afd3ca0 x28: 0068000000000f07 x27: ffff800008000000
| x26: 0000000000000000 x25: 03fffacbe27b8000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000001
| x11: 0000800008000000 x10: ffff800008000000 x9 : ffffb2f8dee00000
| x8 : 000ffffb2f8dee00 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : ffffb2f8dee00000
| Call trace:
| __kasan_unpoison_vmalloc+0xe4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| Code: d34cfc08 aa1f03fa 8b081b39 d503201f (f9400328)
| ---[ end trace 0000000000000000 ]---
| Kernel panic - not syncing: Attempted to kill the idle task!
This is because init_vmalloc_pages() erroneously calls virt_to_page() on
a vmalloc address, while virt_to_page() is only valid for addresses in
the linear/direct map. Since init_vmalloc_pages() expects virtual
addresses in the vmalloc range, it must use vmalloc_to_page() rather
than virt_to_page().
We call init_vmalloc_pages() from __kasan_unpoison_vmalloc(), where we
check !is_vmalloc_or_module_addr(), suggesting that we might encounter a
non-vmalloc address. Luckily, this never happens. By design, we only
call __kasan_unpoison_vmalloc() on pointers in the vmalloc area, and I
have verified that we don't violate that expectation. Given that,
is_vmalloc_or_module_addr() must always be true for any legitimate
argument to __kasan_unpoison_vmalloc().
Correct init_vmalloc_pages() to use vmalloc_to_page(), and remove the
redundant and misleading use of is_vmalloc_or_module_addr() in
__kasan_unpoison_vmalloc().
Link: https://lkml.kernel.org/r/20230418164212.1775741-1-mark.rutland@arm.com
Fixes: 6c2f761dad
("kasan: fix zeroing vmalloc memory with HW_TAGS")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
396 lines
9.5 KiB
C
396 lines
9.5 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* This file contains core hardware tag-based KASAN code.
|
|
*
|
|
* Copyright (c) 2020 Google, Inc.
|
|
* Author: Andrey Konovalov <andreyknvl@google.com>
|
|
*/
|
|
|
|
#define pr_fmt(fmt) "kasan: " fmt
|
|
|
|
#include <linux/init.h>
|
|
#include <linux/kasan.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/memory.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/static_key.h>
|
|
#include <linux/string.h>
|
|
#include <linux/types.h>
|
|
|
|
#include "kasan.h"
|
|
|
|
enum kasan_arg {
|
|
KASAN_ARG_DEFAULT,
|
|
KASAN_ARG_OFF,
|
|
KASAN_ARG_ON,
|
|
};
|
|
|
|
enum kasan_arg_mode {
|
|
KASAN_ARG_MODE_DEFAULT,
|
|
KASAN_ARG_MODE_SYNC,
|
|
KASAN_ARG_MODE_ASYNC,
|
|
KASAN_ARG_MODE_ASYMM,
|
|
};
|
|
|
|
enum kasan_arg_vmalloc {
|
|
KASAN_ARG_VMALLOC_DEFAULT,
|
|
KASAN_ARG_VMALLOC_OFF,
|
|
KASAN_ARG_VMALLOC_ON,
|
|
};
|
|
|
|
static enum kasan_arg kasan_arg __ro_after_init;
|
|
static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
|
|
static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
|
|
|
|
/*
|
|
* Whether KASAN is enabled at all.
|
|
* The value remains false until KASAN is initialized by kasan_init_hw_tags().
|
|
*/
|
|
DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
|
|
EXPORT_SYMBOL(kasan_flag_enabled);
|
|
|
|
/*
|
|
* Whether the selected mode is synchronous, asynchronous, or asymmetric.
|
|
* Defaults to KASAN_MODE_SYNC.
|
|
*/
|
|
enum kasan_mode kasan_mode __ro_after_init;
|
|
EXPORT_SYMBOL_GPL(kasan_mode);
|
|
|
|
/* Whether to enable vmalloc tagging. */
|
|
DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
|
|
|
|
#define PAGE_ALLOC_SAMPLE_DEFAULT 1
|
|
#define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
|
|
|
|
/*
|
|
* Sampling interval of page_alloc allocation (un)poisoning.
|
|
* Defaults to no sampling.
|
|
*/
|
|
unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
|
|
|
|
/*
|
|
* Minimum order of page_alloc allocations to be affected by sampling.
|
|
* The default value is chosen to match both
|
|
* PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
|
|
*/
|
|
unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
|
|
|
|
DEFINE_PER_CPU(long, kasan_page_alloc_skip);
|
|
|
|
/* kasan=off/on */
|
|
static int __init early_kasan_flag(char *arg)
|
|
{
|
|
if (!arg)
|
|
return -EINVAL;
|
|
|
|
if (!strcmp(arg, "off"))
|
|
kasan_arg = KASAN_ARG_OFF;
|
|
else if (!strcmp(arg, "on"))
|
|
kasan_arg = KASAN_ARG_ON;
|
|
else
|
|
return -EINVAL;
|
|
|
|
return 0;
|
|
}
|
|
early_param("kasan", early_kasan_flag);
|
|
|
|
/* kasan.mode=sync/async/asymm */
|
|
static int __init early_kasan_mode(char *arg)
|
|
{
|
|
if (!arg)
|
|
return -EINVAL;
|
|
|
|
if (!strcmp(arg, "sync"))
|
|
kasan_arg_mode = KASAN_ARG_MODE_SYNC;
|
|
else if (!strcmp(arg, "async"))
|
|
kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
|
|
else if (!strcmp(arg, "asymm"))
|
|
kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
|
|
else
|
|
return -EINVAL;
|
|
|
|
return 0;
|
|
}
|
|
early_param("kasan.mode", early_kasan_mode);
|
|
|
|
/* kasan.vmalloc=off/on */
|
|
static int __init early_kasan_flag_vmalloc(char *arg)
|
|
{
|
|
if (!arg)
|
|
return -EINVAL;
|
|
|
|
if (!strcmp(arg, "off"))
|
|
kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
|
|
else if (!strcmp(arg, "on"))
|
|
kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
|
|
else
|
|
return -EINVAL;
|
|
|
|
return 0;
|
|
}
|
|
early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
|
|
|
|
static inline const char *kasan_mode_info(void)
|
|
{
|
|
if (kasan_mode == KASAN_MODE_ASYNC)
|
|
return "async";
|
|
else if (kasan_mode == KASAN_MODE_ASYMM)
|
|
return "asymm";
|
|
else
|
|
return "sync";
|
|
}
|
|
|
|
/* kasan.page_alloc.sample=<sampling interval> */
|
|
static int __init early_kasan_flag_page_alloc_sample(char *arg)
|
|
{
|
|
int rv;
|
|
|
|
if (!arg)
|
|
return -EINVAL;
|
|
|
|
rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
|
|
if (rv)
|
|
return rv;
|
|
|
|
if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
|
|
kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
|
|
|
|
/* kasan.page_alloc.sample.order=<minimum page order> */
|
|
static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
|
|
{
|
|
int rv;
|
|
|
|
if (!arg)
|
|
return -EINVAL;
|
|
|
|
rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
|
|
if (rv)
|
|
return rv;
|
|
|
|
if (kasan_page_alloc_sample_order > INT_MAX) {
|
|
kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
|
|
|
|
/*
|
|
* kasan_init_hw_tags_cpu() is called for each CPU.
|
|
* Not marked as __init as a CPU can be hot-plugged after boot.
|
|
*/
|
|
void kasan_init_hw_tags_cpu(void)
|
|
{
|
|
/*
|
|
* There's no need to check that the hardware is MTE-capable here,
|
|
* as this function is only called for MTE-capable hardware.
|
|
*/
|
|
|
|
/*
|
|
* If KASAN is disabled via command line, don't initialize it.
|
|
* When this function is called, kasan_flag_enabled is not yet
|
|
* set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
|
|
*/
|
|
if (kasan_arg == KASAN_ARG_OFF)
|
|
return;
|
|
|
|
/*
|
|
* Enable async or asymm modes only when explicitly requested
|
|
* through the command line.
|
|
*/
|
|
kasan_enable_hw_tags();
|
|
}
|
|
|
|
/* kasan_init_hw_tags() is called once on boot CPU. */
|
|
void __init kasan_init_hw_tags(void)
|
|
{
|
|
/* If hardware doesn't support MTE, don't initialize KASAN. */
|
|
if (!system_supports_mte())
|
|
return;
|
|
|
|
/* If KASAN is disabled via command line, don't initialize it. */
|
|
if (kasan_arg == KASAN_ARG_OFF)
|
|
return;
|
|
|
|
switch (kasan_arg_mode) {
|
|
case KASAN_ARG_MODE_DEFAULT:
|
|
/* Default is specified by kasan_mode definition. */
|
|
break;
|
|
case KASAN_ARG_MODE_SYNC:
|
|
kasan_mode = KASAN_MODE_SYNC;
|
|
break;
|
|
case KASAN_ARG_MODE_ASYNC:
|
|
kasan_mode = KASAN_MODE_ASYNC;
|
|
break;
|
|
case KASAN_ARG_MODE_ASYMM:
|
|
kasan_mode = KASAN_MODE_ASYMM;
|
|
break;
|
|
}
|
|
|
|
switch (kasan_arg_vmalloc) {
|
|
case KASAN_ARG_VMALLOC_DEFAULT:
|
|
/* Default is specified by kasan_flag_vmalloc definition. */
|
|
break;
|
|
case KASAN_ARG_VMALLOC_OFF:
|
|
static_branch_disable(&kasan_flag_vmalloc);
|
|
break;
|
|
case KASAN_ARG_VMALLOC_ON:
|
|
static_branch_enable(&kasan_flag_vmalloc);
|
|
break;
|
|
}
|
|
|
|
kasan_init_tags();
|
|
|
|
/* KASAN is now initialized, enable it. */
|
|
static_branch_enable(&kasan_flag_enabled);
|
|
|
|
pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
|
|
kasan_mode_info(),
|
|
kasan_vmalloc_enabled() ? "on" : "off",
|
|
kasan_stack_collection_enabled() ? "on" : "off");
|
|
}
|
|
|
|
#ifdef CONFIG_KASAN_VMALLOC
|
|
|
|
static void unpoison_vmalloc_pages(const void *addr, u8 tag)
|
|
{
|
|
struct vm_struct *area;
|
|
int i;
|
|
|
|
/*
|
|
* As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
|
|
* (see the comment in __kasan_unpoison_vmalloc), all of the pages
|
|
* should belong to a single area.
|
|
*/
|
|
area = find_vm_area((void *)addr);
|
|
if (WARN_ON(!area))
|
|
return;
|
|
|
|
for (i = 0; i < area->nr_pages; i++) {
|
|
struct page *page = area->pages[i];
|
|
|
|
page_kasan_tag_set(page, tag);
|
|
}
|
|
}
|
|
|
|
static void init_vmalloc_pages(const void *start, unsigned long size)
|
|
{
|
|
const void *addr;
|
|
|
|
for (addr = start; addr < start + size; addr += PAGE_SIZE) {
|
|
struct page *page = vmalloc_to_page(addr);
|
|
|
|
clear_highpage_kasan_tagged(page);
|
|
}
|
|
}
|
|
|
|
void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
|
|
kasan_vmalloc_flags_t flags)
|
|
{
|
|
u8 tag;
|
|
unsigned long redzone_start, redzone_size;
|
|
|
|
if (!kasan_vmalloc_enabled()) {
|
|
if (flags & KASAN_VMALLOC_INIT)
|
|
init_vmalloc_pages(start, size);
|
|
return (void *)start;
|
|
}
|
|
|
|
/*
|
|
* Don't tag non-VM_ALLOC mappings, as:
|
|
*
|
|
* 1. Unlike the software KASAN modes, hardware tag-based KASAN only
|
|
* supports tagging physical memory. Therefore, it can only tag a
|
|
* single mapping of normal physical pages.
|
|
* 2. Hardware tag-based KASAN can only tag memory mapped with special
|
|
* mapping protection bits, see arch_vmap_pgprot_tagged().
|
|
* As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
|
|
* providing these bits would require tracking all non-VM_ALLOC
|
|
* mappers.
|
|
*
|
|
* Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
|
|
* the first virtual mapping, which is created by vmalloc().
|
|
* Tagging the page_alloc memory backing that vmalloc() allocation is
|
|
* skipped, see ___GFP_SKIP_KASAN.
|
|
*
|
|
* For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
|
|
*/
|
|
if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
|
|
WARN_ON(flags & KASAN_VMALLOC_INIT);
|
|
return (void *)start;
|
|
}
|
|
|
|
/*
|
|
* Don't tag executable memory.
|
|
* The kernel doesn't tolerate having the PC register tagged.
|
|
*/
|
|
if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
|
|
WARN_ON(flags & KASAN_VMALLOC_INIT);
|
|
return (void *)start;
|
|
}
|
|
|
|
tag = kasan_random_tag();
|
|
start = set_tag(start, tag);
|
|
|
|
/* Unpoison and initialize memory up to size. */
|
|
kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
|
|
|
|
/*
|
|
* Explicitly poison and initialize the in-page vmalloc() redzone.
|
|
* Unlike software KASAN modes, hardware tag-based KASAN doesn't
|
|
* unpoison memory when populating shadow for vmalloc() space.
|
|
*/
|
|
redzone_start = round_up((unsigned long)start + size,
|
|
KASAN_GRANULE_SIZE);
|
|
redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
|
|
kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
|
|
flags & KASAN_VMALLOC_INIT);
|
|
|
|
/*
|
|
* Set per-page tag flags to allow accessing physical memory for the
|
|
* vmalloc() mapping through page_address(vmalloc_to_page()).
|
|
*/
|
|
unpoison_vmalloc_pages(start, tag);
|
|
|
|
return (void *)start;
|
|
}
|
|
|
|
void __kasan_poison_vmalloc(const void *start, unsigned long size)
|
|
{
|
|
/*
|
|
* No tagging here.
|
|
* The physical pages backing the vmalloc() allocation are poisoned
|
|
* through the usual page_alloc paths.
|
|
*/
|
|
}
|
|
|
|
#endif
|
|
|
|
void kasan_enable_hw_tags(void)
|
|
{
|
|
if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
|
|
hw_enable_tag_checks_async();
|
|
else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
|
|
hw_enable_tag_checks_asymm();
|
|
else
|
|
hw_enable_tag_checks_sync();
|
|
}
|
|
|
|
#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
|
|
|
|
EXPORT_SYMBOL_GPL(kasan_enable_hw_tags);
|
|
|
|
void kasan_force_async_fault(void)
|
|
{
|
|
hw_force_async_tag_fault();
|
|
}
|
|
EXPORT_SYMBOL_GPL(kasan_force_async_fault);
|
|
|
|
#endif
|