mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 17:30:02 +00:00
![Peter Zijlstra](/assets/img/avatar_default.png)
Provided the target address of a R_X86_64_PC32 relocation is aligned,
the low two bits should be invariant between the relative and absolute
value.
Turns out the address is not aligned and things go sideways, ensure we
transfer the bits in the absolute form when fixing up the key address.
Fixes: 73f44fe19d
("static_call: Allow module use without exposing static_call_key")
Reported-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Tested-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Link: https://lkml.kernel.org/r/20210225220351.GE4746@worktop.programming.kicks-ass.net
539 lines
12 KiB
C
539 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/init.h>
|
|
#include <linux/static_call.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/sort.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/module.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/processor.h>
|
|
#include <asm/sections.h>
|
|
|
|
extern struct static_call_site __start_static_call_sites[],
|
|
__stop_static_call_sites[];
|
|
extern struct static_call_tramp_key __start_static_call_tramp_key[],
|
|
__stop_static_call_tramp_key[];
|
|
|
|
static bool static_call_initialized;
|
|
|
|
/* mutex to protect key modules/sites */
|
|
static DEFINE_MUTEX(static_call_mutex);
|
|
|
|
static void static_call_lock(void)
|
|
{
|
|
mutex_lock(&static_call_mutex);
|
|
}
|
|
|
|
static void static_call_unlock(void)
|
|
{
|
|
mutex_unlock(&static_call_mutex);
|
|
}
|
|
|
|
static inline void *static_call_addr(struct static_call_site *site)
|
|
{
|
|
return (void *)((long)site->addr + (long)&site->addr);
|
|
}
|
|
|
|
|
|
static inline struct static_call_key *static_call_key(const struct static_call_site *site)
|
|
{
|
|
return (struct static_call_key *)
|
|
(((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);
|
|
}
|
|
|
|
/* These assume the key is word-aligned. */
|
|
static inline bool static_call_is_init(struct static_call_site *site)
|
|
{
|
|
return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;
|
|
}
|
|
|
|
static inline bool static_call_is_tail(struct static_call_site *site)
|
|
{
|
|
return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;
|
|
}
|
|
|
|
static inline void static_call_set_init(struct static_call_site *site)
|
|
{
|
|
site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -
|
|
(long)&site->key;
|
|
}
|
|
|
|
static int static_call_site_cmp(const void *_a, const void *_b)
|
|
{
|
|
const struct static_call_site *a = _a;
|
|
const struct static_call_site *b = _b;
|
|
const struct static_call_key *key_a = static_call_key(a);
|
|
const struct static_call_key *key_b = static_call_key(b);
|
|
|
|
if (key_a < key_b)
|
|
return -1;
|
|
|
|
if (key_a > key_b)
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void static_call_site_swap(void *_a, void *_b, int size)
|
|
{
|
|
long delta = (unsigned long)_a - (unsigned long)_b;
|
|
struct static_call_site *a = _a;
|
|
struct static_call_site *b = _b;
|
|
struct static_call_site tmp = *a;
|
|
|
|
a->addr = b->addr - delta;
|
|
a->key = b->key - delta;
|
|
|
|
b->addr = tmp.addr + delta;
|
|
b->key = tmp.key + delta;
|
|
}
|
|
|
|
static inline void static_call_sort_entries(struct static_call_site *start,
|
|
struct static_call_site *stop)
|
|
{
|
|
sort(start, stop - start, sizeof(struct static_call_site),
|
|
static_call_site_cmp, static_call_site_swap);
|
|
}
|
|
|
|
static inline bool static_call_key_has_mods(struct static_call_key *key)
|
|
{
|
|
return !(key->type & 1);
|
|
}
|
|
|
|
static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
|
|
{
|
|
if (!static_call_key_has_mods(key))
|
|
return NULL;
|
|
|
|
return key->mods;
|
|
}
|
|
|
|
static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
|
|
{
|
|
if (static_call_key_has_mods(key))
|
|
return NULL;
|
|
|
|
return (struct static_call_site *)(key->type & ~1);
|
|
}
|
|
|
|
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
|
|
{
|
|
struct static_call_site *site, *stop;
|
|
struct static_call_mod *site_mod, first;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
|
|
if (key->func == func)
|
|
goto done;
|
|
|
|
key->func = func;
|
|
|
|
arch_static_call_transform(NULL, tramp, func, false);
|
|
|
|
/*
|
|
* If uninitialized, we'll not update the callsites, but they still
|
|
* point to the trampoline and we just patched that.
|
|
*/
|
|
if (WARN_ON_ONCE(!static_call_initialized))
|
|
goto done;
|
|
|
|
first = (struct static_call_mod){
|
|
.next = static_call_key_next(key),
|
|
.mod = NULL,
|
|
.sites = static_call_key_sites(key),
|
|
};
|
|
|
|
for (site_mod = &first; site_mod; site_mod = site_mod->next) {
|
|
struct module *mod = site_mod->mod;
|
|
|
|
if (!site_mod->sites) {
|
|
/*
|
|
* This can happen if the static call key is defined in
|
|
* a module which doesn't use it.
|
|
*
|
|
* It also happens in the has_mods case, where the
|
|
* 'first' entry has no sites associated with it.
|
|
*/
|
|
continue;
|
|
}
|
|
|
|
stop = __stop_static_call_sites;
|
|
|
|
#ifdef CONFIG_MODULES
|
|
if (mod) {
|
|
stop = mod->static_call_sites +
|
|
mod->num_static_call_sites;
|
|
}
|
|
#endif
|
|
|
|
for (site = site_mod->sites;
|
|
site < stop && static_call_key(site) == key; site++) {
|
|
void *site_addr = static_call_addr(site);
|
|
|
|
if (static_call_is_init(site)) {
|
|
/*
|
|
* Don't write to call sites which were in
|
|
* initmem and have since been freed.
|
|
*/
|
|
if (!mod && system_state >= SYSTEM_RUNNING)
|
|
continue;
|
|
if (mod && !within_module_init((unsigned long)site_addr, mod))
|
|
continue;
|
|
}
|
|
|
|
if (!kernel_text_address((unsigned long)site_addr)) {
|
|
WARN_ONCE(1, "can't patch static call site at %pS",
|
|
site_addr);
|
|
continue;
|
|
}
|
|
|
|
arch_static_call_transform(site_addr, NULL, func,
|
|
static_call_is_tail(site));
|
|
}
|
|
}
|
|
|
|
done:
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
}
|
|
EXPORT_SYMBOL_GPL(__static_call_update);
|
|
|
|
static int __static_call_init(struct module *mod,
|
|
struct static_call_site *start,
|
|
struct static_call_site *stop)
|
|
{
|
|
struct static_call_site *site;
|
|
struct static_call_key *key, *prev_key = NULL;
|
|
struct static_call_mod *site_mod;
|
|
|
|
if (start == stop)
|
|
return 0;
|
|
|
|
static_call_sort_entries(start, stop);
|
|
|
|
for (site = start; site < stop; site++) {
|
|
void *site_addr = static_call_addr(site);
|
|
|
|
if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
|
|
(!mod && init_section_contains(site_addr, 1)))
|
|
static_call_set_init(site);
|
|
|
|
key = static_call_key(site);
|
|
if (key != prev_key) {
|
|
prev_key = key;
|
|
|
|
/*
|
|
* For vmlinux (!mod) avoid the allocation by storing
|
|
* the sites pointer in the key itself. Also see
|
|
* __static_call_update()'s @first.
|
|
*
|
|
* This allows architectures (eg. x86) to call
|
|
* static_call_init() before memory allocation works.
|
|
*/
|
|
if (!mod) {
|
|
key->sites = site;
|
|
key->type |= 1;
|
|
goto do_transform;
|
|
}
|
|
|
|
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
|
if (!site_mod)
|
|
return -ENOMEM;
|
|
|
|
/*
|
|
* When the key has a direct sites pointer, extract
|
|
* that into an explicit struct static_call_mod, so we
|
|
* can have a list of modules.
|
|
*/
|
|
if (static_call_key_sites(key)) {
|
|
site_mod->mod = NULL;
|
|
site_mod->next = NULL;
|
|
site_mod->sites = static_call_key_sites(key);
|
|
|
|
key->mods = site_mod;
|
|
|
|
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
|
|
if (!site_mod)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
site_mod->mod = mod;
|
|
site_mod->sites = site;
|
|
site_mod->next = static_call_key_next(key);
|
|
key->mods = site_mod;
|
|
}
|
|
|
|
do_transform:
|
|
arch_static_call_transform(site_addr, NULL, key->func,
|
|
static_call_is_tail(site));
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int addr_conflict(struct static_call_site *site, void *start, void *end)
|
|
{
|
|
unsigned long addr = (unsigned long)static_call_addr(site);
|
|
|
|
if (addr <= (unsigned long)end &&
|
|
addr + CALL_INSN_SIZE > (unsigned long)start)
|
|
return 1;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int __static_call_text_reserved(struct static_call_site *iter_start,
|
|
struct static_call_site *iter_stop,
|
|
void *start, void *end)
|
|
{
|
|
struct static_call_site *iter = iter_start;
|
|
|
|
while (iter < iter_stop) {
|
|
if (addr_conflict(iter, start, end))
|
|
return 1;
|
|
iter++;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_MODULES
|
|
|
|
static int __static_call_mod_text_reserved(void *start, void *end)
|
|
{
|
|
struct module *mod;
|
|
int ret;
|
|
|
|
preempt_disable();
|
|
mod = __module_text_address((unsigned long)start);
|
|
WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
|
|
if (!try_module_get(mod))
|
|
mod = NULL;
|
|
preempt_enable();
|
|
|
|
if (!mod)
|
|
return 0;
|
|
|
|
ret = __static_call_text_reserved(mod->static_call_sites,
|
|
mod->static_call_sites + mod->num_static_call_sites,
|
|
start, end);
|
|
|
|
module_put(mod);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static unsigned long tramp_key_lookup(unsigned long addr)
|
|
{
|
|
struct static_call_tramp_key *start = __start_static_call_tramp_key;
|
|
struct static_call_tramp_key *stop = __stop_static_call_tramp_key;
|
|
struct static_call_tramp_key *tramp_key;
|
|
|
|
for (tramp_key = start; tramp_key != stop; tramp_key++) {
|
|
unsigned long tramp;
|
|
|
|
tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp;
|
|
if (tramp == addr)
|
|
return (long)tramp_key->key + (long)&tramp_key->key;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int static_call_add_module(struct module *mod)
|
|
{
|
|
struct static_call_site *start = mod->static_call_sites;
|
|
struct static_call_site *stop = start + mod->num_static_call_sites;
|
|
struct static_call_site *site;
|
|
|
|
for (site = start; site != stop; site++) {
|
|
unsigned long s_key = (long)site->key + (long)&site->key;
|
|
unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS;
|
|
unsigned long key;
|
|
|
|
/*
|
|
* Is the key is exported, 'addr' points to the key, which
|
|
* means modules are allowed to call static_call_update() on
|
|
* it.
|
|
*
|
|
* Otherwise, the key isn't exported, and 'addr' points to the
|
|
* trampoline so we need to lookup the key.
|
|
*
|
|
* We go through this dance to prevent crazy modules from
|
|
* abusing sensitive static calls.
|
|
*/
|
|
if (!kernel_text_address(addr))
|
|
continue;
|
|
|
|
key = tramp_key_lookup(addr);
|
|
if (!key) {
|
|
pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n",
|
|
static_call_addr(site));
|
|
return -EINVAL;
|
|
}
|
|
|
|
key |= s_key & STATIC_CALL_SITE_FLAGS;
|
|
site->key = key - (long)&site->key;
|
|
}
|
|
|
|
return __static_call_init(mod, start, stop);
|
|
}
|
|
|
|
static void static_call_del_module(struct module *mod)
|
|
{
|
|
struct static_call_site *start = mod->static_call_sites;
|
|
struct static_call_site *stop = mod->static_call_sites +
|
|
mod->num_static_call_sites;
|
|
struct static_call_key *key, *prev_key = NULL;
|
|
struct static_call_mod *site_mod, **prev;
|
|
struct static_call_site *site;
|
|
|
|
for (site = start; site < stop; site++) {
|
|
key = static_call_key(site);
|
|
if (key == prev_key)
|
|
continue;
|
|
|
|
prev_key = key;
|
|
|
|
for (prev = &key->mods, site_mod = key->mods;
|
|
site_mod && site_mod->mod != mod;
|
|
prev = &site_mod->next, site_mod = site_mod->next)
|
|
;
|
|
|
|
if (!site_mod)
|
|
continue;
|
|
|
|
*prev = site_mod->next;
|
|
kfree(site_mod);
|
|
}
|
|
}
|
|
|
|
static int static_call_module_notify(struct notifier_block *nb,
|
|
unsigned long val, void *data)
|
|
{
|
|
struct module *mod = data;
|
|
int ret = 0;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
|
|
switch (val) {
|
|
case MODULE_STATE_COMING:
|
|
ret = static_call_add_module(mod);
|
|
if (ret) {
|
|
WARN(1, "Failed to allocate memory for static calls");
|
|
static_call_del_module(mod);
|
|
}
|
|
break;
|
|
case MODULE_STATE_GOING:
|
|
static_call_del_module(mod);
|
|
break;
|
|
}
|
|
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
|
|
return notifier_from_errno(ret);
|
|
}
|
|
|
|
static struct notifier_block static_call_module_nb = {
|
|
.notifier_call = static_call_module_notify,
|
|
};
|
|
|
|
#else
|
|
|
|
static inline int __static_call_mod_text_reserved(void *start, void *end)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#endif /* CONFIG_MODULES */
|
|
|
|
int static_call_text_reserved(void *start, void *end)
|
|
{
|
|
int ret = __static_call_text_reserved(__start_static_call_sites,
|
|
__stop_static_call_sites, start, end);
|
|
|
|
if (ret)
|
|
return ret;
|
|
|
|
return __static_call_mod_text_reserved(start, end);
|
|
}
|
|
|
|
int __init static_call_init(void)
|
|
{
|
|
int ret;
|
|
|
|
if (static_call_initialized)
|
|
return 0;
|
|
|
|
cpus_read_lock();
|
|
static_call_lock();
|
|
ret = __static_call_init(NULL, __start_static_call_sites,
|
|
__stop_static_call_sites);
|
|
static_call_unlock();
|
|
cpus_read_unlock();
|
|
|
|
if (ret) {
|
|
pr_err("Failed to allocate memory for static_call!\n");
|
|
BUG();
|
|
}
|
|
|
|
static_call_initialized = true;
|
|
|
|
#ifdef CONFIG_MODULES
|
|
register_module_notifier(&static_call_module_nb);
|
|
#endif
|
|
return 0;
|
|
}
|
|
early_initcall(static_call_init);
|
|
|
|
long __static_call_return0(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_STATIC_CALL_SELFTEST
|
|
|
|
static int func_a(int x)
|
|
{
|
|
return x+1;
|
|
}
|
|
|
|
static int func_b(int x)
|
|
{
|
|
return x+2;
|
|
}
|
|
|
|
DEFINE_STATIC_CALL(sc_selftest, func_a);
|
|
|
|
static struct static_call_data {
|
|
int (*func)(int);
|
|
int val;
|
|
int expect;
|
|
} static_call_data [] __initdata = {
|
|
{ NULL, 2, 3 },
|
|
{ func_b, 2, 4 },
|
|
{ func_a, 2, 3 }
|
|
};
|
|
|
|
static int __init test_static_call_init(void)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) {
|
|
struct static_call_data *scd = &static_call_data[i];
|
|
|
|
if (scd->func)
|
|
static_call_update(sc_selftest, scd->func);
|
|
|
|
WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
early_initcall(test_static_call_init);
|
|
|
|
#endif /* CONFIG_STATIC_CALL_SELFTEST */
|