2020-12-22 20:02:10 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* This file contains core hardware tag-based KASAN code.
|
|
|
|
*
|
|
|
|
* Copyright (c) 2020 Google, Inc.
|
|
|
|
* Author: Andrey Konovalov <andreyknvl@google.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) "kasan: " fmt
|
|
|
|
|
2020-12-22 20:03:06 +00:00
|
|
|
#include <linux/init.h>
|
2020-12-22 20:02:10 +00:00
|
|
|
#include <linux/kasan.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/memory.h>
|
|
|
|
#include <linux/mm.h>
|
2020-12-22 20:03:06 +00:00
|
|
|
#include <linux/static_key.h>
|
2020-12-22 20:02:10 +00:00
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
|
|
|
|
#include "kasan.h"
|
|
|
|
|
2021-01-24 05:01:34 +00:00
|
|
|
enum kasan_arg {
|
|
|
|
KASAN_ARG_DEFAULT,
|
|
|
|
KASAN_ARG_OFF,
|
|
|
|
KASAN_ARG_ON,
|
2020-12-22 20:03:06 +00:00
|
|
|
};
|
|
|
|
|
2021-03-15 13:20:12 +00:00
|
|
|
enum kasan_arg_mode {
|
|
|
|
KASAN_ARG_MODE_DEFAULT,
|
|
|
|
KASAN_ARG_MODE_SYNC,
|
|
|
|
KASAN_ARG_MODE_ASYNC,
|
2021-10-06 15:47:51 +00:00
|
|
|
KASAN_ARG_MODE_ASYMM,
|
2021-03-15 13:20:12 +00:00
|
|
|
};
|
|
|
|
|
2022-03-25 01:11:47 +00:00
|
|
|
enum kasan_arg_vmalloc {
|
|
|
|
KASAN_ARG_VMALLOC_DEFAULT,
|
|
|
|
KASAN_ARG_VMALLOC_OFF,
|
|
|
|
KASAN_ARG_VMALLOC_ON,
|
|
|
|
};
|
|
|
|
|
2021-01-24 05:01:34 +00:00
|
|
|
static enum kasan_arg kasan_arg __ro_after_init;
|
2021-03-15 13:20:12 +00:00
|
|
|
static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
|
2022-03-25 01:11:47 +00:00
|
|
|
static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
|
2020-12-22 20:03:06 +00:00
|
|
|
|
2022-03-25 01:11:44 +00:00
|
|
|
/*
|
|
|
|
* Whether KASAN is enabled at all.
|
|
|
|
* The value remains false until KASAN is initialized by kasan_init_hw_tags().
|
|
|
|
*/
|
2020-12-22 20:03:06 +00:00
|
|
|
DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
|
|
|
|
EXPORT_SYMBOL(kasan_flag_enabled);
|
|
|
|
|
2022-03-25 01:11:44 +00:00
|
|
|
/*
|
|
|
|
* Whether the selected mode is synchronous, asynchronous, or asymmetric.
|
|
|
|
* Defaults to KASAN_MODE_SYNC.
|
|
|
|
*/
|
2021-10-06 15:47:51 +00:00
|
|
|
enum kasan_mode kasan_mode __ro_after_init;
|
|
|
|
EXPORT_SYMBOL_GPL(kasan_mode);
|
2021-03-15 13:20:12 +00:00
|
|
|
|
2022-03-25 01:11:47 +00:00
|
|
|
/* Whether to enable vmalloc tagging. */
|
|
|
|
DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
|
|
|
|
|
kasan: allow sampling page_alloc allocations for HW_TAGS
As Hardware Tag-Based KASAN is intended to be used in production, its
performance impact is crucial. As page_alloc allocations tend to be big,
tagging and checking all such allocations can introduce a significant
slowdown.
Add two new boot parameters that allow to alleviate that slowdown:
- kasan.page_alloc.sample, which makes Hardware Tag-Based KASAN tag only
every Nth page_alloc allocation with the order configured by the second
added parameter (default: tag every such allocation).
- kasan.page_alloc.sample.order, which makes sampling enabled by the first
parameter only affect page_alloc allocations with the order equal or
greater than the specified value (default: 3, see below).
The exact performance improvement caused by using the new parameters
depends on their values and the applied workload.
The chosen default value for kasan.page_alloc.sample.order is 3, which
matches both PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER. This is
done for two reasons:
1. PAGE_ALLOC_COSTLY_ORDER is "the order at which allocations are deemed
costly to service", which corresponds to the idea that only large and
thus costly allocations are supposed to sampled.
2. One of the workloads targeted by this patch is a benchmark that sends
a large amount of data over a local loopback connection. Most multi-page
data allocations in the networking subsystem have the order of
SKB_FRAG_PAGE_ORDER (or PAGE_ALLOC_COSTLY_ORDER).
When running a local loopback test on a testing MTE-enabled device in sync
mode, enabling Hardware Tag-Based KASAN introduces a ~50% slowdown.
Applying this patch and setting kasan.page_alloc.sampling to a value
higher than 1 allows to lower the slowdown. The performance improvement
saturates around the sampling interval value of 10 with the default
sampling page order of 3. This lowers the slowdown to ~20%. The slowdown
in real scenarios involving the network will likely be better.
Enabling page_alloc sampling has a downside: KASAN misses bad accesses to
a page_alloc allocation that has not been tagged. This lowers the value
of KASAN as a security mitigation.
However, based on measuring the number of page_alloc allocations of
different orders during boot in a test build, sampling with the default
kasan.page_alloc.sample.order value affects only ~7% of allocations. The
rest ~93% of allocations are still checked deterministically.
Link: https://lkml.kernel.org/r/129da0614123bb85ed4dd61ae30842b2dd7c903f.1671471846.git.andreyknvl@google.com
Signed-off-by: Andrey Konovalov <andreyknvl@google.com>
Reviewed-by: Marco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Mark Brand <markbrand@google.com>
Cc: Peter Collingbourne <pcc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2022-12-19 18:09:18 +00:00
|
|
|
#define PAGE_ALLOC_SAMPLE_DEFAULT 1
|
|
|
|
#define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Sampling interval of page_alloc allocation (un)poisoning.
|
|
|
|
* Defaults to no sampling.
|
|
|
|
*/
|
|
|
|
unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Minimum order of page_alloc allocations to be affected by sampling.
|
|
|
|
* The default value is chosen to match both
|
|
|
|
* PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
|
|
|
|
*/
|
|
|
|
unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
|
|
|
|
|
|
|
|
DEFINE_PER_CPU(long, kasan_page_alloc_skip);
|
|
|
|
|
2021-01-24 05:01:34 +00:00
|
|
|
/* kasan=off/on */
|
|
|
|
static int __init early_kasan_flag(char *arg)
|
2020-12-22 20:03:06 +00:00
|
|
|
{
|
|
|
|
if (!arg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!strcmp(arg, "off"))
|
2021-01-24 05:01:34 +00:00
|
|
|
kasan_arg = KASAN_ARG_OFF;
|
|
|
|
else if (!strcmp(arg, "on"))
|
|
|
|
kasan_arg = KASAN_ARG_ON;
|
2020-12-22 20:03:06 +00:00
|
|
|
else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2021-01-24 05:01:34 +00:00
|
|
|
early_param("kasan", early_kasan_flag);
|
2020-12-22 20:03:06 +00:00
|
|
|
|
2021-10-06 15:47:51 +00:00
|
|
|
/* kasan.mode=sync/async/asymm */
|
2021-03-15 13:20:12 +00:00
|
|
|
static int __init early_kasan_mode(char *arg)
|
|
|
|
{
|
|
|
|
if (!arg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!strcmp(arg, "sync"))
|
|
|
|
kasan_arg_mode = KASAN_ARG_MODE_SYNC;
|
|
|
|
else if (!strcmp(arg, "async"))
|
|
|
|
kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
|
2021-10-06 15:47:51 +00:00
|
|
|
else if (!strcmp(arg, "asymm"))
|
|
|
|
kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
|
2021-03-15 13:20:12 +00:00
|
|
|
else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("kasan.mode", early_kasan_mode);
|
|
|
|
|
2022-03-25 01:11:47 +00:00
|
|
|
/* kasan.vmalloc=off/on */
|
|
|
|
static int __init early_kasan_flag_vmalloc(char *arg)
|
|
|
|
{
|
|
|
|
if (!arg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!strcmp(arg, "off"))
|
|
|
|
kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
|
|
|
|
else if (!strcmp(arg, "on"))
|
|
|
|
kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
|
|
|
|
else
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
|
|
|
|
|
2021-11-11 04:32:49 +00:00
|
|
|
static inline const char *kasan_mode_info(void)
|
|
|
|
{
|
|
|
|
if (kasan_mode == KASAN_MODE_ASYNC)
|
|
|
|
return "async";
|
|
|
|
else if (kasan_mode == KASAN_MODE_ASYMM)
|
|
|
|
return "asymm";
|
|
|
|
else
|
|
|
|
return "sync";
|
|
|
|
}
|
|
|
|
|
kasan: allow sampling page_alloc allocations for HW_TAGS
As Hardware Tag-Based KASAN is intended to be used in production, its
performance impact is crucial. As page_alloc allocations tend to be big,
tagging and checking all such allocations can introduce a significant
slowdown.
Add two new boot parameters that allow to alleviate that slowdown:
- kasan.page_alloc.sample, which makes Hardware Tag-Based KASAN tag only
every Nth page_alloc allocation with the order configured by the second
added parameter (default: tag every such allocation).
- kasan.page_alloc.sample.order, which makes sampling enabled by the first
parameter only affect page_alloc allocations with the order equal or
greater than the specified value (default: 3, see below).
The exact performance improvement caused by using the new parameters
depends on their values and the applied workload.
The chosen default value for kasan.page_alloc.sample.order is 3, which
matches both PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER. This is
done for two reasons:
1. PAGE_ALLOC_COSTLY_ORDER is "the order at which allocations are deemed
costly to service", which corresponds to the idea that only large and
thus costly allocations are supposed to sampled.
2. One of the workloads targeted by this patch is a benchmark that sends
a large amount of data over a local loopback connection. Most multi-page
data allocations in the networking subsystem have the order of
SKB_FRAG_PAGE_ORDER (or PAGE_ALLOC_COSTLY_ORDER).
When running a local loopback test on a testing MTE-enabled device in sync
mode, enabling Hardware Tag-Based KASAN introduces a ~50% slowdown.
Applying this patch and setting kasan.page_alloc.sampling to a value
higher than 1 allows to lower the slowdown. The performance improvement
saturates around the sampling interval value of 10 with the default
sampling page order of 3. This lowers the slowdown to ~20%. The slowdown
in real scenarios involving the network will likely be better.
Enabling page_alloc sampling has a downside: KASAN misses bad accesses to
a page_alloc allocation that has not been tagged. This lowers the value
of KASAN as a security mitigation.
However, based on measuring the number of page_alloc allocations of
different orders during boot in a test build, sampling with the default
kasan.page_alloc.sample.order value affects only ~7% of allocations. The
rest ~93% of allocations are still checked deterministically.
Link: https://lkml.kernel.org/r/129da0614123bb85ed4dd61ae30842b2dd7c903f.1671471846.git.andreyknvl@google.com
Signed-off-by: Andrey Konovalov <andreyknvl@google.com>
Reviewed-by: Marco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Jann Horn <jannh@google.com>
Cc: Mark Brand <markbrand@google.com>
Cc: Peter Collingbourne <pcc@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2022-12-19 18:09:18 +00:00
|
|
|
/* kasan.page_alloc.sample=<sampling interval> */
|
|
|
|
static int __init early_kasan_flag_page_alloc_sample(char *arg)
|
|
|
|
{
|
|
|
|
int rv;
|
|
|
|
|
|
|
|
if (!arg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
|
|
|
|
if (rv)
|
|
|
|
return rv;
|
|
|
|
|
|
|
|
if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
|
|
|
|
kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
|
|
|
|
|
|
|
|
/* kasan.page_alloc.sample.order=<minimum page order> */
|
|
|
|
static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
|
|
|
|
{
|
|
|
|
int rv;
|
|
|
|
|
|
|
|
if (!arg)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
|
|
|
|
if (rv)
|
|
|
|
return rv;
|
|
|
|
|
|
|
|
if (kasan_page_alloc_sample_order > INT_MAX) {
|
|
|
|
kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
|
|
|
|
|
2022-03-25 01:11:41 +00:00
|
|
|
/*
|
|
|
|
* kasan_init_hw_tags_cpu() is called for each CPU.
|
|
|
|
* Not marked as __init as a CPU can be hot-plugged after boot.
|
|
|
|
*/
|
2020-12-22 20:02:10 +00:00
|
|
|
void kasan_init_hw_tags_cpu(void)
|
|
|
|
{
|
2020-12-22 20:03:06 +00:00
|
|
|
/*
|
|
|
|
* There's no need to check that the hardware is MTE-capable here,
|
|
|
|
* as this function is only called for MTE-capable hardware.
|
|
|
|
*/
|
|
|
|
|
2022-03-25 01:11:44 +00:00
|
|
|
/*
|
|
|
|
* If KASAN is disabled via command line, don't initialize it.
|
|
|
|
* When this function is called, kasan_flag_enabled is not yet
|
|
|
|
* set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
|
|
|
|
*/
|
2021-01-24 05:01:34 +00:00
|
|
|
if (kasan_arg == KASAN_ARG_OFF)
|
2020-12-22 20:03:06 +00:00
|
|
|
return;
|
|
|
|
|
2021-03-15 13:20:12 +00:00
|
|
|
/*
|
2021-10-06 15:47:51 +00:00
|
|
|
* Enable async or asymm modes only when explicitly requested
|
|
|
|
* through the command line.
|
2021-03-15 13:20:12 +00:00
|
|
|
*/
|
2023-03-10 23:43:30 +00:00
|
|
|
kasan_enable_hw_tags();
|
2020-12-22 20:02:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* kasan_init_hw_tags() is called once on boot CPU. */
|
|
|
|
void __init kasan_init_hw_tags(void)
|
|
|
|
{
|
2021-01-24 05:01:34 +00:00
|
|
|
/* If hardware doesn't support MTE, don't initialize KASAN. */
|
2020-12-22 20:03:06 +00:00
|
|
|
if (!system_supports_mte())
|
|
|
|
return;
|
|
|
|
|
2021-01-24 05:01:34 +00:00
|
|
|
/* If KASAN is disabled via command line, don't initialize it. */
|
|
|
|
if (kasan_arg == KASAN_ARG_OFF)
|
2020-12-22 20:03:06 +00:00
|
|
|
return;
|
|
|
|
|
2021-03-15 13:20:12 +00:00
|
|
|
switch (kasan_arg_mode) {
|
|
|
|
case KASAN_ARG_MODE_DEFAULT:
|
2022-03-25 01:11:44 +00:00
|
|
|
/* Default is specified by kasan_mode definition. */
|
|
|
|
break;
|
2021-03-15 13:20:12 +00:00
|
|
|
case KASAN_ARG_MODE_SYNC:
|
2021-10-06 15:47:51 +00:00
|
|
|
kasan_mode = KASAN_MODE_SYNC;
|
2021-03-15 13:20:12 +00:00
|
|
|
break;
|
|
|
|
case KASAN_ARG_MODE_ASYNC:
|
2021-10-06 15:47:51 +00:00
|
|
|
kasan_mode = KASAN_MODE_ASYNC;
|
|
|
|
break;
|
|
|
|
case KASAN_ARG_MODE_ASYMM:
|
|
|
|
kasan_mode = KASAN_MODE_ASYMM;
|
2021-03-15 13:20:12 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2022-03-25 01:11:47 +00:00
|
|
|
switch (kasan_arg_vmalloc) {
|
|
|
|
case KASAN_ARG_VMALLOC_DEFAULT:
|
|
|
|
/* Default is specified by kasan_flag_vmalloc definition. */
|
|
|
|
break;
|
|
|
|
case KASAN_ARG_VMALLOC_OFF:
|
|
|
|
static_branch_disable(&kasan_flag_vmalloc);
|
|
|
|
break;
|
|
|
|
case KASAN_ARG_VMALLOC_ON:
|
|
|
|
static_branch_enable(&kasan_flag_vmalloc);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2022-09-05 21:05:46 +00:00
|
|
|
kasan_init_tags();
|
2020-12-22 20:03:06 +00:00
|
|
|
|
2022-03-25 01:11:44 +00:00
|
|
|
/* KASAN is now initialized, enable it. */
|
|
|
|
static_branch_enable(&kasan_flag_enabled);
|
|
|
|
|
2022-03-25 01:11:47 +00:00
|
|
|
pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
|
2021-11-11 04:32:49 +00:00
|
|
|
kasan_mode_info(),
|
2022-03-25 01:11:47 +00:00
|
|
|
kasan_vmalloc_enabled() ? "on" : "off",
|
2021-11-11 04:32:49 +00:00
|
|
|
kasan_stack_collection_enabled() ? "on" : "off");
|
2020-12-22 20:02:10 +00:00
|
|
|
}
|
|
|
|
|
2022-03-25 01:11:32 +00:00
|
|
|
#ifdef CONFIG_KASAN_VMALLOC
|
|
|
|
|
|
|
|
static void unpoison_vmalloc_pages(const void *addr, u8 tag)
|
|
|
|
{
|
|
|
|
struct vm_struct *area;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
|
|
|
|
* (see the comment in __kasan_unpoison_vmalloc), all of the pages
|
|
|
|
* should belong to a single area.
|
|
|
|
*/
|
|
|
|
area = find_vm_area((void *)addr);
|
|
|
|
if (WARN_ON(!area))
|
|
|
|
return;
|
|
|
|
|
|
|
|
for (i = 0; i < area->nr_pages; i++) {
|
|
|
|
struct page *page = area->pages[i];
|
|
|
|
|
|
|
|
page_kasan_tag_set(page, tag);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-06-09 18:18:47 +00:00
|
|
|
static void init_vmalloc_pages(const void *start, unsigned long size)
|
|
|
|
{
|
|
|
|
const void *addr;
|
|
|
|
|
|
|
|
for (addr = start; addr < start + size; addr += PAGE_SIZE) {
|
kasan: hw_tags: avoid invalid virt_to_page()
When booting with 'kasan.vmalloc=off', a kernel configured with support
for KASAN_HW_TAGS will explode at boot time due to bogus use of
virt_to_page() on a vmalloc adddress. With CONFIG_DEBUG_VIRTUAL selected
this will be reported explicitly, and with or without CONFIG_DEBUG_VIRTUAL
the kernel will dereference a bogus address:
| ------------[ cut here ]------------
| virt_to_phys used for non-linear address: (____ptrval____) (0xffff800008000000)
| WARNING: CPU: 0 PID: 0 at arch/arm64/mm/physaddr.c:15 __virt_to_phys+0x78/0x80
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 600000c5 (nZCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __virt_to_phys+0x78/0x80
| lr : __virt_to_phys+0x78/0x80
| sp : ffffcd076afd3c80
| x29: ffffcd076afd3c80 x28: 0068000000000f07 x27: ffff800008000000
| x26: fffffbfff0000000 x25: fffffbffff000000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000003
| x11: 00000000ffffefff x10: c0000000ffffefff x9 : 0000000000000000
| x8 : 0000000000000000 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : 000000000000004f
| Call trace:
| __virt_to_phys+0x78/0x80
| __kasan_unpoison_vmalloc+0xd4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| ---[ end trace 0000000000000000 ]---
| Unable to handle kernel paging request at virtual address 03fffacbe27b8000
| Mem abort info:
| ESR = 0x0000000096000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| FSC = 0x04: level 0 translation fault
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000000041bc5000
| [03fffacbe27b8000] pgd=0000000000000000, p4d=0000000000000000
| Internal error: Oops: 0000000096000004 [#1] PREEMPT SMP
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 200000c5 (nzCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __kasan_unpoison_vmalloc+0xe4/0x478
| lr : __kasan_unpoison_vmalloc+0xd4/0x478
| sp : ffffcd076afd3ca0
| x29: ffffcd076afd3ca0 x28: 0068000000000f07 x27: ffff800008000000
| x26: 0000000000000000 x25: 03fffacbe27b8000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000001
| x11: 0000800008000000 x10: ffff800008000000 x9 : ffffb2f8dee00000
| x8 : 000ffffb2f8dee00 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : ffffb2f8dee00000
| Call trace:
| __kasan_unpoison_vmalloc+0xe4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| Code: d34cfc08 aa1f03fa 8b081b39 d503201f (f9400328)
| ---[ end trace 0000000000000000 ]---
| Kernel panic - not syncing: Attempted to kill the idle task!
This is because init_vmalloc_pages() erroneously calls virt_to_page() on
a vmalloc address, while virt_to_page() is only valid for addresses in
the linear/direct map. Since init_vmalloc_pages() expects virtual
addresses in the vmalloc range, it must use vmalloc_to_page() rather
than virt_to_page().
We call init_vmalloc_pages() from __kasan_unpoison_vmalloc(), where we
check !is_vmalloc_or_module_addr(), suggesting that we might encounter a
non-vmalloc address. Luckily, this never happens. By design, we only
call __kasan_unpoison_vmalloc() on pointers in the vmalloc area, and I
have verified that we don't violate that expectation. Given that,
is_vmalloc_or_module_addr() must always be true for any legitimate
argument to __kasan_unpoison_vmalloc().
Correct init_vmalloc_pages() to use vmalloc_to_page(), and remove the
redundant and misleading use of is_vmalloc_or_module_addr() in
__kasan_unpoison_vmalloc().
Link: https://lkml.kernel.org/r/20230418164212.1775741-1-mark.rutland@arm.com
Fixes: 6c2f761dad7851d8 ("kasan: fix zeroing vmalloc memory with HW_TAGS")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-18 16:42:12 +00:00
|
|
|
struct page *page = vmalloc_to_page(addr);
|
2022-06-09 18:18:47 +00:00
|
|
|
|
|
|
|
clear_highpage_kasan_tagged(page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-25 01:11:32 +00:00
|
|
|
void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
|
|
|
|
kasan_vmalloc_flags_t flags)
|
|
|
|
{
|
|
|
|
u8 tag;
|
|
|
|
unsigned long redzone_start, redzone_size;
|
|
|
|
|
kasan: hw_tags: avoid invalid virt_to_page()
When booting with 'kasan.vmalloc=off', a kernel configured with support
for KASAN_HW_TAGS will explode at boot time due to bogus use of
virt_to_page() on a vmalloc adddress. With CONFIG_DEBUG_VIRTUAL selected
this will be reported explicitly, and with or without CONFIG_DEBUG_VIRTUAL
the kernel will dereference a bogus address:
| ------------[ cut here ]------------
| virt_to_phys used for non-linear address: (____ptrval____) (0xffff800008000000)
| WARNING: CPU: 0 PID: 0 at arch/arm64/mm/physaddr.c:15 __virt_to_phys+0x78/0x80
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Not tainted 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 600000c5 (nZCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __virt_to_phys+0x78/0x80
| lr : __virt_to_phys+0x78/0x80
| sp : ffffcd076afd3c80
| x29: ffffcd076afd3c80 x28: 0068000000000f07 x27: ffff800008000000
| x26: fffffbfff0000000 x25: fffffbffff000000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000003
| x11: 00000000ffffefff x10: c0000000ffffefff x9 : 0000000000000000
| x8 : 0000000000000000 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : 000000000000004f
| Call trace:
| __virt_to_phys+0x78/0x80
| __kasan_unpoison_vmalloc+0xd4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| ---[ end trace 0000000000000000 ]---
| Unable to handle kernel paging request at virtual address 03fffacbe27b8000
| Mem abort info:
| ESR = 0x0000000096000004
| EC = 0x25: DABT (current EL), IL = 32 bits
| SET = 0, FnV = 0
| EA = 0, S1PTW = 0
| FSC = 0x04: level 0 translation fault
| Data abort info:
| ISV = 0, ISS = 0x00000004
| CM = 0, WnR = 0
| swapper pgtable: 4k pages, 48-bit VAs, pgdp=0000000041bc5000
| [03fffacbe27b8000] pgd=0000000000000000, p4d=0000000000000000
| Internal error: Oops: 0000000096000004 [#1] PREEMPT SMP
| Modules linked in:
| CPU: 0 PID: 0 Comm: swapper/0 Tainted: G W 6.3.0-rc3-00073-g83865133300d-dirty #4
| Hardware name: linux,dummy-virt (DT)
| pstate: 200000c5 (nzCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
| pc : __kasan_unpoison_vmalloc+0xe4/0x478
| lr : __kasan_unpoison_vmalloc+0xd4/0x478
| sp : ffffcd076afd3ca0
| x29: ffffcd076afd3ca0 x28: 0068000000000f07 x27: ffff800008000000
| x26: 0000000000000000 x25: 03fffacbe27b8000 x24: ff00000000000000
| x23: ffffcd076ad3c000 x22: fffffc0000000000 x21: ffff800008000000
| x20: ffff800008004000 x19: ffff800008000000 x18: ffff800008004000
| x17: 666678302820295f x16: ffffffffffffffff x15: 0000000000000004
| x14: ffffcd076b009e88 x13: 0000000000000fff x12: 0000000000000001
| x11: 0000800008000000 x10: ffff800008000000 x9 : ffffb2f8dee00000
| x8 : 000ffffb2f8dee00 x7 : 205d303030303030 x6 : 302e30202020205b
| x5 : ffffcd076b41d63f x4 : ffffcd076afd3827 x3 : 0000000000000000
| x2 : 0000000000000000 x1 : ffffcd076afd3a30 x0 : ffffb2f8dee00000
| Call trace:
| __kasan_unpoison_vmalloc+0xe4/0x478
| __vmalloc_node_range+0x77c/0x7b8
| __vmalloc_node+0x54/0x64
| init_IRQ+0x94/0xc8
| start_kernel+0x194/0x420
| __primary_switched+0xbc/0xc4
| Code: d34cfc08 aa1f03fa 8b081b39 d503201f (f9400328)
| ---[ end trace 0000000000000000 ]---
| Kernel panic - not syncing: Attempted to kill the idle task!
This is because init_vmalloc_pages() erroneously calls virt_to_page() on
a vmalloc address, while virt_to_page() is only valid for addresses in
the linear/direct map. Since init_vmalloc_pages() expects virtual
addresses in the vmalloc range, it must use vmalloc_to_page() rather
than virt_to_page().
We call init_vmalloc_pages() from __kasan_unpoison_vmalloc(), where we
check !is_vmalloc_or_module_addr(), suggesting that we might encounter a
non-vmalloc address. Luckily, this never happens. By design, we only
call __kasan_unpoison_vmalloc() on pointers in the vmalloc area, and I
have verified that we don't violate that expectation. Given that,
is_vmalloc_or_module_addr() must always be true for any legitimate
argument to __kasan_unpoison_vmalloc().
Correct init_vmalloc_pages() to use vmalloc_to_page(), and remove the
redundant and misleading use of is_vmalloc_or_module_addr() in
__kasan_unpoison_vmalloc().
Link: https://lkml.kernel.org/r/20230418164212.1775741-1-mark.rutland@arm.com
Fixes: 6c2f761dad7851d8 ("kasan: fix zeroing vmalloc memory with HW_TAGS")
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Marco Elver <elver@google.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-04-18 16:42:12 +00:00
|
|
|
if (!kasan_vmalloc_enabled()) {
|
2022-06-09 18:18:47 +00:00
|
|
|
if (flags & KASAN_VMALLOC_INIT)
|
|
|
|
init_vmalloc_pages(start, size);
|
2022-03-25 01:11:32 +00:00
|
|
|
return (void *)start;
|
2022-06-09 18:18:47 +00:00
|
|
|
}
|
2022-03-25 01:11:32 +00:00
|
|
|
|
|
|
|
/*
|
2022-06-09 18:18:47 +00:00
|
|
|
* Don't tag non-VM_ALLOC mappings, as:
|
2022-03-25 01:11:32 +00:00
|
|
|
*
|
|
|
|
* 1. Unlike the software KASAN modes, hardware tag-based KASAN only
|
|
|
|
* supports tagging physical memory. Therefore, it can only tag a
|
|
|
|
* single mapping of normal physical pages.
|
|
|
|
* 2. Hardware tag-based KASAN can only tag memory mapped with special
|
2022-06-09 18:18:47 +00:00
|
|
|
* mapping protection bits, see arch_vmap_pgprot_tagged().
|
2022-03-25 01:11:32 +00:00
|
|
|
* As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
|
|
|
|
* providing these bits would require tracking all non-VM_ALLOC
|
|
|
|
* mappers.
|
|
|
|
*
|
|
|
|
* Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
|
|
|
|
* the first virtual mapping, which is created by vmalloc().
|
|
|
|
* Tagging the page_alloc memory backing that vmalloc() allocation is
|
2023-03-10 04:29:14 +00:00
|
|
|
* skipped, see ___GFP_SKIP_KASAN.
|
2022-03-25 01:11:32 +00:00
|
|
|
*
|
|
|
|
* For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
|
|
|
|
*/
|
2022-06-09 18:18:47 +00:00
|
|
|
if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
|
|
|
|
WARN_ON(flags & KASAN_VMALLOC_INIT);
|
2022-03-25 01:11:32 +00:00
|
|
|
return (void *)start;
|
2022-06-09 18:18:47 +00:00
|
|
|
}
|
2022-03-25 01:11:32 +00:00
|
|
|
|
2022-03-25 01:11:35 +00:00
|
|
|
/*
|
|
|
|
* Don't tag executable memory.
|
|
|
|
* The kernel doesn't tolerate having the PC register tagged.
|
|
|
|
*/
|
2022-06-09 18:18:47 +00:00
|
|
|
if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
|
|
|
|
WARN_ON(flags & KASAN_VMALLOC_INIT);
|
2022-03-25 01:11:35 +00:00
|
|
|
return (void *)start;
|
2022-06-09 18:18:47 +00:00
|
|
|
}
|
2022-03-25 01:11:35 +00:00
|
|
|
|
2022-03-25 01:11:32 +00:00
|
|
|
tag = kasan_random_tag();
|
|
|
|
start = set_tag(start, tag);
|
|
|
|
|
|
|
|
/* Unpoison and initialize memory up to size. */
|
|
|
|
kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Explicitly poison and initialize the in-page vmalloc() redzone.
|
|
|
|
* Unlike software KASAN modes, hardware tag-based KASAN doesn't
|
|
|
|
* unpoison memory when populating shadow for vmalloc() space.
|
|
|
|
*/
|
|
|
|
redzone_start = round_up((unsigned long)start + size,
|
|
|
|
KASAN_GRANULE_SIZE);
|
|
|
|
redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
|
|
|
|
kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
|
|
|
|
flags & KASAN_VMALLOC_INIT);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set per-page tag flags to allow accessing physical memory for the
|
|
|
|
* vmalloc() mapping through page_address(vmalloc_to_page()).
|
|
|
|
*/
|
|
|
|
unpoison_vmalloc_pages(start, tag);
|
|
|
|
|
|
|
|
return (void *)start;
|
|
|
|
}
|
|
|
|
|
|
|
|
void __kasan_poison_vmalloc(const void *start, unsigned long size)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* No tagging here.
|
|
|
|
* The physical pages backing the vmalloc() allocation are poisoned
|
|
|
|
* through the usual page_alloc paths.
|
|
|
|
*/
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
2023-03-10 23:43:30 +00:00
|
|
|
void kasan_enable_hw_tags(void)
|
2021-02-24 20:05:26 +00:00
|
|
|
{
|
2022-03-25 01:12:02 +00:00
|
|
|
if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
|
2023-03-10 23:43:30 +00:00
|
|
|
hw_enable_tag_checks_async();
|
2022-03-25 01:12:02 +00:00
|
|
|
else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
|
2023-03-10 23:43:30 +00:00
|
|
|
hw_enable_tag_checks_asymm();
|
2022-03-25 01:12:02 +00:00
|
|
|
else
|
2023-03-10 23:43:30 +00:00
|
|
|
hw_enable_tag_checks_sync();
|
2021-02-24 20:05:26 +00:00
|
|
|
}
|
2022-04-15 02:13:37 +00:00
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
|
|
|
|
|
2023-03-10 23:43:30 +00:00
|
|
|
EXPORT_SYMBOL_GPL(kasan_enable_hw_tags);
|
2021-02-24 20:05:26 +00:00
|
|
|
|
2021-03-15 13:20:19 +00:00
|
|
|
void kasan_force_async_fault(void)
|
|
|
|
{
|
|
|
|
hw_force_async_tag_fault();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(kasan_force_async_fault);
|
|
|
|
|
2021-02-24 20:05:26 +00:00
|
|
|
#endif
|