2019-01-17 18:30:40 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0+ */
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2009-09-18 17:28:19 +00:00
|
|
|
* Read-Copy Update mechanism for mutual exclusion
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2008-01-25 20:08:24 +00:00
|
|
|
* Copyright IBM Corporation, 2001
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* Author: Dipankar Sarma <dipankar@in.ibm.com>
|
2009-09-18 17:28:19 +00:00
|
|
|
*
|
2019-01-17 18:30:40 +00:00
|
|
|
* Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
|
2005-04-16 22:20:36 +00:00
|
|
|
* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
|
|
|
|
* Papers:
|
|
|
|
* http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
|
|
|
|
* http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
|
|
|
|
*
|
|
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
2009-09-18 17:28:19 +00:00
|
|
|
* http://lse.sourceforge.net/locking/rcupdate.html
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __LINUX_RCUPDATE_H
|
|
|
|
#define __LINUX_RCUPDATE_H
|
|
|
|
|
2011-06-01 04:03:55 +00:00
|
|
|
#include <linux/types.h>
|
2010-04-28 21:39:09 +00:00
|
|
|
#include <linux/compiler.h>
|
2017-05-03 22:24:25 +00:00
|
|
|
#include <linux/atomic.h>
|
2016-05-02 18:58:56 +00:00
|
|
|
#include <linux/irqflags.h>
|
2017-05-03 22:24:25 +00:00
|
|
|
#include <linux/preempt.h>
|
|
|
|
#include <linux/bottom_half.h>
|
|
|
|
#include <linux/lockdep.h>
|
|
|
|
#include <asm/processor.h>
|
|
|
|
#include <linux/cpumask.h>
|
2015-04-14 21:08:58 +00:00
|
|
|
|
2010-08-13 23:16:25 +00:00
|
|
|
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
|
|
|
|
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
|
2012-12-28 19:30:36 +00:00
|
|
|
#define ulong2long(a) (*(long *)(&(a)))
|
2020-11-13 20:54:48 +00:00
|
|
|
#define USHORT_CMP_GE(a, b) (USHRT_MAX / 2 >= (unsigned short)((a) - (b)))
|
|
|
|
#define USHORT_CMP_LT(a, b) (USHRT_MAX / 2 < (unsigned short)((a) - (b)))
|
2010-08-13 23:16:25 +00:00
|
|
|
|
2009-06-25 16:08:16 +00:00
|
|
|
/* Exported common interfaces */
|
2018-07-03 17:44:44 +00:00
|
|
|
void call_rcu(struct rcu_head *head, rcu_callback_t func);
|
2014-07-01 19:22:23 +00:00
|
|
|
void rcu_barrier_tasks(void);
|
2020-03-03 05:06:43 +00:00
|
|
|
void rcu_barrier_tasks_rude(void);
|
2018-07-03 17:44:44 +00:00
|
|
|
void synchronize_rcu(void);
|
2014-06-27 20:42:20 +00:00
|
|
|
|
2010-08-13 23:16:25 +00:00
|
|
|
#ifdef CONFIG_PREEMPT_RCU
|
|
|
|
|
2013-11-11 15:11:23 +00:00
|
|
|
void __rcu_read_lock(void);
|
|
|
|
void __rcu_read_unlock(void);
|
2010-08-17 21:18:46 +00:00
|
|
|
|
2010-08-13 23:16:25 +00:00
|
|
|
/*
|
|
|
|
* Defined as a macro as it is a very low level header included from
|
|
|
|
* areas that don't even know about current. This gives the rcu_read_lock()
|
|
|
|
* nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
|
|
|
|
* types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
|
|
|
|
*/
|
2021-05-20 20:35:50 +00:00
|
|
|
#define rcu_preempt_depth() READ_ONCE(current->rcu_read_lock_nesting)
|
2010-08-13 23:16:25 +00:00
|
|
|
|
2010-08-17 21:18:46 +00:00
|
|
|
#else /* #ifdef CONFIG_PREEMPT_RCU */
|
|
|
|
|
2020-08-10 16:58:03 +00:00
|
|
|
#ifdef CONFIG_TINY_RCU
|
|
|
|
#define rcu_read_unlock_strict() do { } while (0)
|
|
|
|
#else
|
|
|
|
void rcu_read_unlock_strict(void);
|
|
|
|
#endif
|
|
|
|
|
2010-08-17 21:18:46 +00:00
|
|
|
static inline void __rcu_read_lock(void)
|
|
|
|
{
|
2019-06-03 20:26:20 +00:00
|
|
|
preempt_disable();
|
2010-08-17 21:18:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __rcu_read_unlock(void)
|
|
|
|
{
|
2019-06-03 20:26:20 +00:00
|
|
|
preempt_enable();
|
2021-08-27 02:21:22 +00:00
|
|
|
if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
|
|
|
|
rcu_read_unlock_strict();
|
2010-08-17 21:18:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int rcu_preempt_depth(void)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
|
|
|
|
|
|
|
|
/* Internal to kernel */
|
2013-11-11 15:11:23 +00:00
|
|
|
void rcu_init(void);
|
2017-05-26 23:16:40 +00:00
|
|
|
extern int rcu_scheduler_active __read_mostly;
|
2018-11-21 19:35:03 +00:00
|
|
|
void rcu_sched_clock_irq(int user);
|
2016-02-26 18:43:44 +00:00
|
|
|
void rcu_report_dead(unsigned int cpu);
|
rcu: Migrate callbacks earlier in the CPU-offline timeline
RCU callbacks must be migrated away from an outgoing CPU, and this is
done near the end of the CPU-hotplug operation, after the outgoing CPU is
long gone. Unfortunately, this means that other CPU-hotplug callbacks
can execute while the outgoing CPU's callbacks are still immobilized
on the long-gone CPU's callback lists. If any of these CPU-hotplug
callbacks must wait, either directly or indirectly, for the invocation
of any of the immobilized RCU callbacks, the system will hang.
This commit avoids such hangs by migrating the callbacks away from the
outgoing CPU immediately upon its departure, shortly after the return
from __cpu_die() in takedown_cpu(). Thus, RCU is able to advance these
callbacks and invoke them, which allows all the after-the-fact CPU-hotplug
callbacks to wait on these RCU callbacks without risk of a hang.
While in the neighborhood, this commit also moves rcu_send_cbs_to_orphanage()
and rcu_adopt_orphan_cbs() under a pre-existing #ifdef to avoid including
dead code on the one hand and to avoid define-without-use warnings on the
other hand.
Reported-by: Jeffrey Hugo <jhugo@codeaurora.org>
Link: http://lkml.kernel.org/r/db9c91f6-1b17-6136-84f0-03c3c2581ab4@codeaurora.org
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Anna-Maria Gleixner <anna-maria@linutronix.de>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Richard Weinberger <richard@nod.at>
2017-06-20 19:11:34 +00:00
|
|
|
void rcutree_migrate_callbacks(int cpu);
|
2012-07-11 18:26:30 +00:00
|
|
|
|
2020-12-09 20:27:31 +00:00
|
|
|
#ifdef CONFIG_TASKS_RCU_GENERIC
|
|
|
|
void rcu_init_tasks_generic(void);
|
|
|
|
#else
|
|
|
|
static inline void rcu_init_tasks_generic(void) { }
|
|
|
|
#endif
|
|
|
|
|
2014-04-27 06:15:35 +00:00
|
|
|
#ifdef CONFIG_RCU_STALL_COMMON
|
|
|
|
void rcu_sysrq_start(void);
|
|
|
|
void rcu_sysrq_end(void);
|
|
|
|
#else /* #ifdef CONFIG_RCU_STALL_COMMON */
|
2017-05-03 17:52:10 +00:00
|
|
|
static inline void rcu_sysrq_start(void) { }
|
|
|
|
static inline void rcu_sysrq_end(void) { }
|
2014-04-27 06:15:35 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */
|
|
|
|
|
2015-05-13 17:41:58 +00:00
|
|
|
#ifdef CONFIG_NO_HZ_FULL
|
2013-11-11 15:11:23 +00:00
|
|
|
void rcu_user_enter(void);
|
|
|
|
void rcu_user_exit(void);
|
2012-07-11 18:26:30 +00:00
|
|
|
#else
|
|
|
|
static inline void rcu_user_enter(void) { }
|
|
|
|
static inline void rcu_user_exit(void) { }
|
2015-05-13 17:41:58 +00:00
|
|
|
#endif /* CONFIG_NO_HZ_FULL */
|
2012-07-11 18:26:30 +00:00
|
|
|
|
2014-07-25 18:21:47 +00:00
|
|
|
#ifdef CONFIG_RCU_NOCB_CPU
|
|
|
|
void rcu_init_nohz(void);
|
2020-11-13 12:13:22 +00:00
|
|
|
int rcu_nocb_cpu_offload(int cpu);
|
2020-11-13 12:13:19 +00:00
|
|
|
int rcu_nocb_cpu_deoffload(int cpu);
|
2021-01-31 23:05:45 +00:00
|
|
|
void rcu_nocb_flush_deferred_wakeup(void);
|
2014-07-25 18:21:47 +00:00
|
|
|
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
|
2017-05-03 17:52:10 +00:00
|
|
|
static inline void rcu_init_nohz(void) { }
|
2020-11-13 12:13:22 +00:00
|
|
|
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
|
2020-11-13 12:13:19 +00:00
|
|
|
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
|
2021-01-31 23:05:45 +00:00
|
|
|
static inline void rcu_nocb_flush_deferred_wakeup(void) { }
|
2014-07-25 18:21:47 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
|
|
|
|
|
2012-02-02 23:42:04 +00:00
|
|
|
/**
|
|
|
|
* RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
|
|
|
|
* @a: Code that RCU needs to pay attention to.
|
|
|
|
*
|
2018-07-08 01:12:26 +00:00
|
|
|
* RCU read-side critical sections are forbidden in the inner idle loop,
|
|
|
|
* that is, between the rcu_idle_enter() and the rcu_idle_exit() -- RCU
|
|
|
|
* will happily ignore any such read-side critical sections. However,
|
|
|
|
* things like powertop need tracepoints in the inner idle loop.
|
2012-02-02 23:42:04 +00:00
|
|
|
*
|
|
|
|
* This macro provides the way out: RCU_NONIDLE(do_something_with_RCU())
|
2016-04-20 16:22:15 +00:00
|
|
|
* will tell RCU that it needs to pay attention, invoke its argument
|
|
|
|
* (in this example, calling the do_something_with_RCU() function),
|
2012-02-02 23:42:04 +00:00
|
|
|
* and then tell RCU to go back to ignoring this CPU. It is permissible
|
2016-04-20 16:22:15 +00:00
|
|
|
* to nest RCU_NONIDLE() wrappers, but not indefinitely (but the limit is
|
|
|
|
* on the order of a million or so, even on 32-bit systems). It is
|
|
|
|
* not legal to block within RCU_NONIDLE(), nor is it permissible to
|
|
|
|
* transfer control either into or out of RCU_NONIDLE()'s statement.
|
2012-02-02 23:42:04 +00:00
|
|
|
*/
|
|
|
|
#define RCU_NONIDLE(a) \
|
|
|
|
do { \
|
2015-10-31 07:59:01 +00:00
|
|
|
rcu_irq_enter_irqson(); \
|
2012-02-02 23:42:04 +00:00
|
|
|
do { a; } while (0); \
|
2015-10-31 07:59:01 +00:00
|
|
|
rcu_irq_exit_irqson(); \
|
2012-02-02 23:42:04 +00:00
|
|
|
} while (0)
|
|
|
|
|
2014-06-27 20:42:20 +00:00
|
|
|
/*
|
2018-05-14 20:52:27 +00:00
|
|
|
* Note a quasi-voluntary context switch for RCU-tasks's benefit.
|
|
|
|
* This is a macro rather than an inline function to avoid #include hell.
|
2014-06-27 20:42:20 +00:00
|
|
|
*/
|
2020-03-03 19:49:21 +00:00
|
|
|
#ifdef CONFIG_TASKS_RCU_GENERIC
|
2020-03-17 03:38:29 +00:00
|
|
|
|
|
|
|
# ifdef CONFIG_TASKS_RCU
|
|
|
|
# define rcu_tasks_classic_qs(t, preempt) \
|
|
|
|
do { \
|
|
|
|
if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
|
|
|
|
WRITE_ONCE((t)->rcu_tasks_holdout, false); \
|
2014-06-27 20:42:20 +00:00
|
|
|
} while (0)
|
2017-05-25 15:05:00 +00:00
|
|
|
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
|
|
|
|
void synchronize_rcu_tasks(void);
|
2020-03-17 03:38:29 +00:00
|
|
|
# else
|
|
|
|
# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
|
|
|
|
# define call_rcu_tasks call_rcu
|
|
|
|
# define synchronize_rcu_tasks synchronize_rcu
|
|
|
|
# endif
|
|
|
|
|
2021-07-13 00:56:45 +00:00
|
|
|
# ifdef CONFIG_TASKS_TRACE_RCU
|
2020-03-17 03:38:29 +00:00
|
|
|
# define rcu_tasks_trace_qs(t) \
|
|
|
|
do { \
|
|
|
|
if (!likely(READ_ONCE((t)->trc_reader_checked)) && \
|
|
|
|
!unlikely(READ_ONCE((t)->trc_reader_nesting))) { \
|
|
|
|
smp_store_release(&(t)->trc_reader_checked, true); \
|
|
|
|
smp_mb(); /* Readers partitioned by store. */ \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
# else
|
|
|
|
# define rcu_tasks_trace_qs(t) do { } while (0)
|
|
|
|
# endif
|
|
|
|
|
|
|
|
#define rcu_tasks_qs(t, preempt) \
|
|
|
|
do { \
|
|
|
|
rcu_tasks_classic_qs((t), (preempt)); \
|
|
|
|
rcu_tasks_trace_qs((t)); \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
# ifdef CONFIG_TASKS_RUDE_RCU
|
2020-03-03 05:06:43 +00:00
|
|
|
void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
|
|
|
|
void synchronize_rcu_tasks_rude(void);
|
2020-03-17 03:38:29 +00:00
|
|
|
# endif
|
|
|
|
|
|
|
|
#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
|
2017-05-25 15:51:48 +00:00
|
|
|
void exit_tasks_rcu_start(void);
|
|
|
|
void exit_tasks_rcu_finish(void);
|
2020-03-03 19:49:21 +00:00
|
|
|
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
|
2020-03-17 03:38:29 +00:00
|
|
|
#define rcu_tasks_qs(t, preempt) do { } while (0)
|
2018-07-10 19:53:40 +00:00
|
|
|
#define rcu_note_voluntary_context_switch(t) do { } while (0)
|
2018-07-08 01:12:26 +00:00
|
|
|
#define call_rcu_tasks call_rcu
|
2018-07-06 18:46:47 +00:00
|
|
|
#define synchronize_rcu_tasks synchronize_rcu
|
2017-05-25 15:51:48 +00:00
|
|
|
static inline void exit_tasks_rcu_start(void) { }
|
|
|
|
static inline void exit_tasks_rcu_finish(void) { }
|
2020-03-03 19:49:21 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
|
2014-06-27 20:42:20 +00:00
|
|
|
|
2014-07-01 18:26:57 +00:00
|
|
|
/**
|
2018-03-03 00:35:27 +00:00
|
|
|
* cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
|
2014-07-01 18:26:57 +00:00
|
|
|
*
|
|
|
|
* This macro resembles cond_resched(), except that it is defined to
|
|
|
|
* report potential quiescent states to RCU-tasks even if the cond_resched()
|
2019-10-15 19:18:14 +00:00
|
|
|
* machinery were to be shut off, as some advocate for PREEMPTION kernels.
|
2014-07-01 18:26:57 +00:00
|
|
|
*/
|
2018-03-03 00:35:27 +00:00
|
|
|
#define cond_resched_tasks_rcu_qs() \
|
2014-07-01 18:26:57 +00:00
|
|
|
do { \
|
2020-03-17 03:38:29 +00:00
|
|
|
rcu_tasks_qs(current, false); \
|
2018-05-11 08:30:34 +00:00
|
|
|
cond_resched(); \
|
2014-07-01 18:26:57 +00:00
|
|
|
} while (0)
|
|
|
|
|
2011-05-27 05:14:36 +00:00
|
|
|
/*
|
|
|
|
* Infrastructure to implement the synchronize_() primitives in
|
|
|
|
* TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
|
|
|
|
*/
|
|
|
|
|
2019-10-15 02:55:57 +00:00
|
|
|
#if defined(CONFIG_TREE_RCU)
|
"Tree RCU": scalable classic RCU implementation
This patch fixes a long-standing performance bug in classic RCU that
results in massive internal-to-RCU lock contention on systems with
more than a few hundred CPUs. Although this patch creates a separate
flavor of RCU for ease of review and patch maintenance, it is intended
to replace classic RCU.
This patch still handles stress better than does mainline, so I am still
calling it ready for inclusion. This patch is against the -tip tree.
Nevertheless, experience on an actual 1000+ CPU machine would still be
most welcome.
Most of the changes noted below were found while creating an rcutiny
(which should permit ejecting the current rcuclassic) and while doing
detailed line-by-line documentation.
Updates from v9 (http://lkml.org/lkml/2008/12/2/334):
o Fixes from remainder of line-by-line code walkthrough,
including comment spelling, initialization, undesirable
narrowing due to type conversion, removing redundant memory
barriers, removing redundant local-variable initialization,
and removing redundant local variables.
I do not believe that any of these fixes address the CPU-hotplug
issues that Andi Kleen was seeing, but please do give it a whirl
in case the machine is smarter than I am.
A writeup from the walkthrough may be found at the following
URL, in case you are suffering from terminal insomnia or
masochism:
http://www.kernel.org/pub/linux/kernel/people/paulmck/tmp/rcutree-walkthrough.2008.12.16a.pdf
o Made rcutree tracing use seq_file, as suggested some time
ago by Lai Jiangshan.
o Added a .csv variant of the rcudata debugfs trace file, to allow
people having thousands of CPUs to drop the data into
a spreadsheet. Tested with oocalc and gnumeric. Updated
documentation to suit.
Updates from v8 (http://lkml.org/lkml/2008/11/15/139):
o Fix a theoretical race between grace-period initialization and
force_quiescent_state() that could occur if more than three
jiffies were required to carry out the grace-period
initialization. Which it might, if you had enough CPUs.
o Apply Ingo's printk-standardization patch.
o Substitute local variables for repeated accesses to global
variables.
o Fix comment misspellings and redundant (but harmless) increments
of ->n_rcu_pending (this latter after having explicitly added it).
o Apply checkpatch fixes.
Updates from v7 (http://lkml.org/lkml/2008/10/10/291):
o Fixed a number of problems noted by Gautham Shenoy, including
the cpu-stall-detection bug that he was having difficulty
convincing me was real. ;-)
o Changed cpu-stall detection to wait for ten seconds rather than
three in order to reduce false positive, as suggested by Ingo
Molnar.
o Produced a design document (http://lwn.net/Articles/305782/).
The act of writing this document uncovered a number of both
theoretical and "here and now" bugs as noted below.
o Fix dynticks_nesting accounting confusion, simplify WARN_ON()
condition, fix kerneldoc comments, and add memory barriers
in dynticks interface functions.
o Add more data to tracing.
o Remove unused "rcu_barrier" field from rcu_data structure.
o Count calls to rcu_pending() from scheduling-clock interrupt
to use as a surrogate timebase should jiffies stop counting.
o Fix a theoretical race between force_quiescent_state() and
grace-period initialization. Yes, initialization does have to
go on for some jiffies for this race to occur, but given enough
CPUs...
Updates from v6 (http://lkml.org/lkml/2008/9/23/448):
o Fix a number of checkpatch.pl complaints.
o Apply review comments from Ingo Molnar and Lai Jiangshan
on the stall-detection code.
o Fix several bugs in !CONFIG_SMP builds.
o Fix a misspelled config-parameter name so that RCU now announces
at boot time if stall detection is configured.
o Run tests on numerous combinations of configurations parameters,
which after the fixes above, now build and run correctly.
Updates from v5 (http://lkml.org/lkml/2008/9/15/92, bad subject line):
o Fix a compiler error in the !CONFIG_FANOUT_EXACT case (blew a
changeset some time ago, and finally got around to retesting
this option).
o Fix some tracing bugs in rcupreempt that caused incorrect
totals to be printed.
o I now test with a more brutal random-selection online/offline
script (attached). Probably more brutal than it needs to be
on the people reading it as well, but so it goes.
o A number of optimizations and usability improvements:
o Make rcu_pending() ignore the grace-period timeout when
there is no grace period in progress.
o Make force_quiescent_state() avoid going for a global
lock in the case where there is no grace period in
progress.
o Rearrange struct fields to improve struct layout.
o Make call_rcu() initiate a grace period if RCU was
idle, rather than waiting for the next scheduling
clock interrupt.
o Invoke rcu_irq_enter() and rcu_irq_exit() only when
idle, as suggested by Andi Kleen. I still don't
completely trust this change, and might back it out.
o Make CONFIG_RCU_TRACE be the single config variable
manipulated for all forms of RCU, instead of the prior
confusion.
o Document tracing files and formats for both rcupreempt
and rcutree.
Updates from v4 for those missing v5 given its bad subject line:
o Separated dynticks interface so that NMIs and irqs call separate
functions, greatly simplifying it. In particular, this code
no longer requires a proof of correctness. ;-)
o Separated dynticks state out into its own per-CPU structure,
avoiding the duplicated accounting.
o The case where a dynticks-idle CPU runs an irq handler that
invokes call_rcu() is now correctly handled, forcing that CPU
out of dynticks-idle mode.
o Review comments have been applied (thank you all!!!).
For but one example, fixed the dynticks-ordering issue that
Manfred pointed out, saving me much debugging. ;-)
o Adjusted rcuclassic and rcupreempt to handle dynticks changes.
Attached is an updated patch to Classic RCU that applies a hierarchy,
greatly reducing the contention on the top-level lock for large machines.
This passes 10-hour concurrent rcutorture and online-offline testing on
128-CPU ppc64 without dynticks enabled, and exposes some timekeeping
bugs in presence of dynticks (exciting working on a system where
"sleep 1" hangs until interrupted...), which were fixed in the
2.6.27 kernel. It is getting more reliable than mainline by some
measures, so the next version will be against -tip for inclusion.
See also Manfred Spraul's recent patches (or his earlier work from
2004 at http://marc.info/?l=linux-kernel&m=108546384711797&w=2).
We will converge onto a common patch in the fullness of time, but are
currently exploring different regions of the design space. That said,
I have already gratefully stolen quite a few of Manfred's ideas.
This patch provides CONFIG_RCU_FANOUT, which controls the bushiness
of the RCU hierarchy. Defaults to 32 on 32-bit machines and 64 on
64-bit machines. If CONFIG_NR_CPUS is less than CONFIG_RCU_FANOUT,
there is no hierarchy. By default, the RCU initialization code will
adjust CONFIG_RCU_FANOUT to balance the hierarchy, so strongly NUMA
architectures may choose to set CONFIG_RCU_FANOUT_EXACT to disable
this balancing, allowing the hierarchy to be exactly aligned to the
underlying hardware. Up to two levels of hierarchy are permitted
(in addition to the root node), allowing up to 16,384 CPUs on 32-bit
systems and up to 262,144 CPUs on 64-bit systems. I just know that I
am going to regret saying this, but this seems more than sufficient
for the foreseeable future. (Some architectures might wish to set
CONFIG_RCU_FANOUT=4, which would limit such architectures to 64 CPUs.
If this becomes a real problem, additional levels can be added, but I
doubt that it will make a significant difference on real hardware.)
In the common case, a given CPU will manipulate its private rcu_data
structure and the rcu_node structure that it shares with its immediate
neighbors. This can reduce both lock and memory contention by multiple
orders of magnitude, which should eliminate the need for the strange
manipulations that are reported to be required when running Linux on
very large systems.
Some shortcomings:
o More bugs will probably surface as a result of an ongoing
line-by-line code inspection.
Patches will be provided as required.
o There are probably hangs, rcutorture failures, &c. Seems
quite stable on a 128-CPU machine, but that is kind of small
compared to 4096 CPUs. However, seems to do better than
mainline.
Patches will be provided as required.
o The memory footprint of this version is several KB larger
than rcuclassic.
A separate UP-only rcutiny patch will be provided, which will
reduce the memory footprint significantly, even compared
to the old rcuclassic. One such patch passes light testing,
and has a memory footprint smaller even than rcuclassic.
Initial reaction from various embedded guys was "it is not
worth it", so am putting it aside.
Credits:
o Manfred Spraul for ideas, review comments, and bugs spotted,
as well as some good friendly competition. ;-)
o Josh Triplett, Ingo Molnar, Peter Zijlstra, Mathieu Desnoyers,
Lai Jiangshan, Andi Kleen, Andy Whitcroft, and Andrew Morton
for reviews and comments.
o Thomas Gleixner for much-needed help with some timer issues
(see patches below).
o Jon M. Tollefson, Tim Pepper, Andrew Theurer, Jose R. Santos,
Andy Whitcroft, Darrick Wong, Nishanth Aravamudan, Anton
Blanchard, Dave Kleikamp, and Nathan Lynch for keeping machines
alive despite my heavy abuse^Wtesting.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-12-18 20:55:32 +00:00
|
|
|
#include <linux/rcutree.h>
|
2013-03-27 15:44:00 +00:00
|
|
|
#elif defined(CONFIG_TINY_RCU)
|
rcu: "Tiny RCU", The Bloatwatch Edition
This patch is a version of RCU designed for !SMP provided for a
small-footprint RCU implementation. In particular, the
implementation of synchronize_rcu() is extremely lightweight and
high performance. It passes rcutorture testing in each of the
four relevant configurations (combinations of NO_HZ and PREEMPT)
on x86. This saves about 1K bytes compared to old Classic RCU
(which is no longer in mainline), and more than three kilobytes
compared to Hierarchical RCU (updated to 2.6.30):
CONFIG_TREE_RCU:
text data bss dec filename
183 4 0 187 kernel/rcupdate.o
2783 520 36 3339 kernel/rcutree.o
3526 Total (vs 4565 for v7)
CONFIG_TREE_PREEMPT_RCU:
text data bss dec filename
263 4 0 267 kernel/rcupdate.o
4594 776 52 5422 kernel/rcutree.o
5689 Total (6155 for v7)
CONFIG_TINY_RCU:
text data bss dec filename
96 4 0 100 kernel/rcupdate.o
734 24 0 758 kernel/rcutiny.o
858 Total (vs 848 for v7)
The above is for x86. Your mileage may vary on other platforms.
Further compression is possible, but is being procrastinated.
Changes from v7 (http://lkml.org/lkml/2009/10/9/388)
o Apply Lai Jiangshan's review comments (aside from
might_sleep() in synchronize_sched(), which is covered by SMP builds).
o Fix up expedited primitives.
Changes from v6 (http://lkml.org/lkml/2009/9/23/293).
o Forward ported to put it into the 2.6.33 stream.
o Added lockdep support.
o Make lightweight rcu_barrier.
Changes from v5 (http://lkml.org/lkml/2009/6/23/12).
o Ported to latest pre-2.6.32 merge window kernel.
- Renamed rcu_qsctr_inc() to rcu_sched_qs().
- Renamed rcu_bh_qsctr_inc() to rcu_bh_qs().
- Provided trivial rcu_cpu_notify().
- Provided trivial exit_rcu().
- Provided trivial rcu_needs_cpu().
- Fixed up the rcu_*_enter/exit() functions in linux/hardirq.h.
o Removed the dependence on EMBEDDED, with a view to making
TINY_RCU default for !SMP at some time in the future.
o Added (trivial) support for expedited grace periods.
Changes from v4 (http://lkml.org/lkml/2009/5/2/91) include:
o Squeeze the size down a bit further by removing the
->completed field from struct rcu_ctrlblk.
o This permits synchronize_rcu() to become the empty function.
Previous concerns about rcutorture were unfounded, as
rcutorture correctly handles a constant value from
rcu_batches_completed() and rcu_batches_completed_bh().
Changes from v3 (http://lkml.org/lkml/2009/3/29/221) include:
o Changed rcu_batches_completed(), rcu_batches_completed_bh()
rcu_enter_nohz(), rcu_exit_nohz(), rcu_nmi_enter(), and
rcu_nmi_exit(), to be static inlines, as suggested by David
Howells. Doing this saves about 100 bytes from rcutiny.o.
(The numbers between v3 and this v4 of the patch are not directly
comparable, since they are against different versions of Linux.)
Changes from v2 (http://lkml.org/lkml/2009/2/3/333) include:
o Fix whitespace issues.
o Change short-circuit "||" operator to instead be "+" in order
to fix performance bug noted by "kraai" on LWN.
(http://lwn.net/Articles/324348/)
Changes from v1 (http://lkml.org/lkml/2009/1/13/440) include:
o This version depends on EMBEDDED as well as !SMP, as suggested
by Ingo.
o Updated rcu_needs_cpu() to unconditionally return zero,
permitting the CPU to enter dynticks-idle mode at any time.
This works because callbacks can be invoked upon entry to
dynticks-idle mode.
o Paul is now OK with this being included, based on a poll at
the Kernel Miniconf at linux.conf.au, where about ten people said
that they cared about saving 900 bytes on single-CPU systems.
o Applies to both mainline and tip/core/rcu.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Acked-by: David Howells <dhowells@redhat.com>
Acked-by: Josh Triplett <josh@joshtriplett.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: avi@redhat.com
Cc: mtosatti@redhat.com
LKML-Reference: <12565226351355-git-send-email->
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-10-26 02:03:50 +00:00
|
|
|
#include <linux/rcutiny.h>
|
"Tree RCU": scalable classic RCU implementation
This patch fixes a long-standing performance bug in classic RCU that
results in massive internal-to-RCU lock contention on systems with
more than a few hundred CPUs. Although this patch creates a separate
flavor of RCU for ease of review and patch maintenance, it is intended
to replace classic RCU.
This patch still handles stress better than does mainline, so I am still
calling it ready for inclusion. This patch is against the -tip tree.
Nevertheless, experience on an actual 1000+ CPU machine would still be
most welcome.
Most of the changes noted below were found while creating an rcutiny
(which should permit ejecting the current rcuclassic) and while doing
detailed line-by-line documentation.
Updates from v9 (http://lkml.org/lkml/2008/12/2/334):
o Fixes from remainder of line-by-line code walkthrough,
including comment spelling, initialization, undesirable
narrowing due to type conversion, removing redundant memory
barriers, removing redundant local-variable initialization,
and removing redundant local variables.
I do not believe that any of these fixes address the CPU-hotplug
issues that Andi Kleen was seeing, but please do give it a whirl
in case the machine is smarter than I am.
A writeup from the walkthrough may be found at the following
URL, in case you are suffering from terminal insomnia or
masochism:
http://www.kernel.org/pub/linux/kernel/people/paulmck/tmp/rcutree-walkthrough.2008.12.16a.pdf
o Made rcutree tracing use seq_file, as suggested some time
ago by Lai Jiangshan.
o Added a .csv variant of the rcudata debugfs trace file, to allow
people having thousands of CPUs to drop the data into
a spreadsheet. Tested with oocalc and gnumeric. Updated
documentation to suit.
Updates from v8 (http://lkml.org/lkml/2008/11/15/139):
o Fix a theoretical race between grace-period initialization and
force_quiescent_state() that could occur if more than three
jiffies were required to carry out the grace-period
initialization. Which it might, if you had enough CPUs.
o Apply Ingo's printk-standardization patch.
o Substitute local variables for repeated accesses to global
variables.
o Fix comment misspellings and redundant (but harmless) increments
of ->n_rcu_pending (this latter after having explicitly added it).
o Apply checkpatch fixes.
Updates from v7 (http://lkml.org/lkml/2008/10/10/291):
o Fixed a number of problems noted by Gautham Shenoy, including
the cpu-stall-detection bug that he was having difficulty
convincing me was real. ;-)
o Changed cpu-stall detection to wait for ten seconds rather than
three in order to reduce false positive, as suggested by Ingo
Molnar.
o Produced a design document (http://lwn.net/Articles/305782/).
The act of writing this document uncovered a number of both
theoretical and "here and now" bugs as noted below.
o Fix dynticks_nesting accounting confusion, simplify WARN_ON()
condition, fix kerneldoc comments, and add memory barriers
in dynticks interface functions.
o Add more data to tracing.
o Remove unused "rcu_barrier" field from rcu_data structure.
o Count calls to rcu_pending() from scheduling-clock interrupt
to use as a surrogate timebase should jiffies stop counting.
o Fix a theoretical race between force_quiescent_state() and
grace-period initialization. Yes, initialization does have to
go on for some jiffies for this race to occur, but given enough
CPUs...
Updates from v6 (http://lkml.org/lkml/2008/9/23/448):
o Fix a number of checkpatch.pl complaints.
o Apply review comments from Ingo Molnar and Lai Jiangshan
on the stall-detection code.
o Fix several bugs in !CONFIG_SMP builds.
o Fix a misspelled config-parameter name so that RCU now announces
at boot time if stall detection is configured.
o Run tests on numerous combinations of configurations parameters,
which after the fixes above, now build and run correctly.
Updates from v5 (http://lkml.org/lkml/2008/9/15/92, bad subject line):
o Fix a compiler error in the !CONFIG_FANOUT_EXACT case (blew a
changeset some time ago, and finally got around to retesting
this option).
o Fix some tracing bugs in rcupreempt that caused incorrect
totals to be printed.
o I now test with a more brutal random-selection online/offline
script (attached). Probably more brutal than it needs to be
on the people reading it as well, but so it goes.
o A number of optimizations and usability improvements:
o Make rcu_pending() ignore the grace-period timeout when
there is no grace period in progress.
o Make force_quiescent_state() avoid going for a global
lock in the case where there is no grace period in
progress.
o Rearrange struct fields to improve struct layout.
o Make call_rcu() initiate a grace period if RCU was
idle, rather than waiting for the next scheduling
clock interrupt.
o Invoke rcu_irq_enter() and rcu_irq_exit() only when
idle, as suggested by Andi Kleen. I still don't
completely trust this change, and might back it out.
o Make CONFIG_RCU_TRACE be the single config variable
manipulated for all forms of RCU, instead of the prior
confusion.
o Document tracing files and formats for both rcupreempt
and rcutree.
Updates from v4 for those missing v5 given its bad subject line:
o Separated dynticks interface so that NMIs and irqs call separate
functions, greatly simplifying it. In particular, this code
no longer requires a proof of correctness. ;-)
o Separated dynticks state out into its own per-CPU structure,
avoiding the duplicated accounting.
o The case where a dynticks-idle CPU runs an irq handler that
invokes call_rcu() is now correctly handled, forcing that CPU
out of dynticks-idle mode.
o Review comments have been applied (thank you all!!!).
For but one example, fixed the dynticks-ordering issue that
Manfred pointed out, saving me much debugging. ;-)
o Adjusted rcuclassic and rcupreempt to handle dynticks changes.
Attached is an updated patch to Classic RCU that applies a hierarchy,
greatly reducing the contention on the top-level lock for large machines.
This passes 10-hour concurrent rcutorture and online-offline testing on
128-CPU ppc64 without dynticks enabled, and exposes some timekeeping
bugs in presence of dynticks (exciting working on a system where
"sleep 1" hangs until interrupted...), which were fixed in the
2.6.27 kernel. It is getting more reliable than mainline by some
measures, so the next version will be against -tip for inclusion.
See also Manfred Spraul's recent patches (or his earlier work from
2004 at http://marc.info/?l=linux-kernel&m=108546384711797&w=2).
We will converge onto a common patch in the fullness of time, but are
currently exploring different regions of the design space. That said,
I have already gratefully stolen quite a few of Manfred's ideas.
This patch provides CONFIG_RCU_FANOUT, which controls the bushiness
of the RCU hierarchy. Defaults to 32 on 32-bit machines and 64 on
64-bit machines. If CONFIG_NR_CPUS is less than CONFIG_RCU_FANOUT,
there is no hierarchy. By default, the RCU initialization code will
adjust CONFIG_RCU_FANOUT to balance the hierarchy, so strongly NUMA
architectures may choose to set CONFIG_RCU_FANOUT_EXACT to disable
this balancing, allowing the hierarchy to be exactly aligned to the
underlying hardware. Up to two levels of hierarchy are permitted
(in addition to the root node), allowing up to 16,384 CPUs on 32-bit
systems and up to 262,144 CPUs on 64-bit systems. I just know that I
am going to regret saying this, but this seems more than sufficient
for the foreseeable future. (Some architectures might wish to set
CONFIG_RCU_FANOUT=4, which would limit such architectures to 64 CPUs.
If this becomes a real problem, additional levels can be added, but I
doubt that it will make a significant difference on real hardware.)
In the common case, a given CPU will manipulate its private rcu_data
structure and the rcu_node structure that it shares with its immediate
neighbors. This can reduce both lock and memory contention by multiple
orders of magnitude, which should eliminate the need for the strange
manipulations that are reported to be required when running Linux on
very large systems.
Some shortcomings:
o More bugs will probably surface as a result of an ongoing
line-by-line code inspection.
Patches will be provided as required.
o There are probably hangs, rcutorture failures, &c. Seems
quite stable on a 128-CPU machine, but that is kind of small
compared to 4096 CPUs. However, seems to do better than
mainline.
Patches will be provided as required.
o The memory footprint of this version is several KB larger
than rcuclassic.
A separate UP-only rcutiny patch will be provided, which will
reduce the memory footprint significantly, even compared
to the old rcuclassic. One such patch passes light testing,
and has a memory footprint smaller even than rcuclassic.
Initial reaction from various embedded guys was "it is not
worth it", so am putting it aside.
Credits:
o Manfred Spraul for ideas, review comments, and bugs spotted,
as well as some good friendly competition. ;-)
o Josh Triplett, Ingo Molnar, Peter Zijlstra, Mathieu Desnoyers,
Lai Jiangshan, Andi Kleen, Andy Whitcroft, and Andrew Morton
for reviews and comments.
o Thomas Gleixner for much-needed help with some timer issues
(see patches below).
o Jon M. Tollefson, Tim Pepper, Andrew Theurer, Jose R. Santos,
Andy Whitcroft, Darrick Wong, Nishanth Aravamudan, Anton
Blanchard, Dave Kleikamp, and Nathan Lynch for keeping machines
alive despite my heavy abuse^Wtesting.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2008-12-18 20:55:32 +00:00
|
|
|
#else
|
|
|
|
#error "Unknown RCU implementation specified to kernel configuration"
|
2009-08-22 20:56:53 +00:00
|
|
|
#endif
|
2008-01-25 20:08:24 +00:00
|
|
|
|
2010-04-17 12:48:42 +00:00
|
|
|
/*
|
2018-01-23 22:48:33 +00:00
|
|
|
* The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
|
|
|
|
* are needed for dynamic initialization and destruction of rcu_head
|
|
|
|
* on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
|
|
|
|
* dynamic initialization and destruction of statically allocated rcu_head
|
|
|
|
* structures. However, rcu_head structures allocated dynamically in the
|
|
|
|
* heap don't need any initialization.
|
2010-04-17 12:48:42 +00:00
|
|
|
*/
|
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
|
2014-06-19 21:57:10 +00:00
|
|
|
void init_rcu_head(struct rcu_head *head);
|
|
|
|
void destroy_rcu_head(struct rcu_head *head);
|
2013-11-11 15:11:23 +00:00
|
|
|
void init_rcu_head_on_stack(struct rcu_head *head);
|
|
|
|
void destroy_rcu_head_on_stack(struct rcu_head *head);
|
2010-04-17 12:48:42 +00:00
|
|
|
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
2017-05-03 17:52:10 +00:00
|
|
|
static inline void init_rcu_head(struct rcu_head *head) { }
|
|
|
|
static inline void destroy_rcu_head(struct rcu_head *head) { }
|
|
|
|
static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
|
|
|
|
static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
|
2010-04-17 12:48:42 +00:00
|
|
|
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
|
2010-04-17 12:48:39 +00:00
|
|
|
|
2012-01-23 20:41:26 +00:00
|
|
|
#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
|
|
|
|
bool rcu_lockdep_current_cpu_online(void);
|
|
|
|
#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
|
2017-05-03 19:32:55 +00:00
|
|
|
static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
|
2012-01-23 20:41:26 +00:00
|
|
|
#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
|
|
|
|
|
2020-09-16 18:45:23 +00:00
|
|
|
extern struct lockdep_map rcu_lock_map;
|
|
|
|
extern struct lockdep_map rcu_bh_lock_map;
|
|
|
|
extern struct lockdep_map rcu_sched_lock_map;
|
|
|
|
extern struct lockdep_map rcu_callback_map;
|
|
|
|
|
2009-08-22 20:56:47 +00:00
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
|
2011-10-07 16:22:02 +00:00
|
|
|
static inline void rcu_lock_acquire(struct lockdep_map *map)
|
|
|
|
{
|
2014-01-20 18:20:06 +00:00
|
|
|
lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
|
2011-10-07 16:22:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void rcu_lock_release(struct lockdep_map *map)
|
|
|
|
{
|
2019-09-19 16:09:40 +00:00
|
|
|
lock_release(map, _THIS_IP_);
|
2011-10-07 16:22:02 +00:00
|
|
|
}
|
|
|
|
|
2014-03-12 16:37:24 +00:00
|
|
|
int debug_lockdep_rcu_enabled(void);
|
2014-07-08 22:17:59 +00:00
|
|
|
int rcu_read_lock_held(void);
|
2013-11-11 15:11:23 +00:00
|
|
|
int rcu_read_lock_bh_held(void);
|
2015-05-26 15:48:34 +00:00
|
|
|
int rcu_read_lock_sched_held(void);
|
2019-07-16 22:12:22 +00:00
|
|
|
int rcu_read_lock_any_held(void);
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
|
|
|
|
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
|
2011-10-07 16:22:03 +00:00
|
|
|
# define rcu_lock_acquire(a) do { } while (0)
|
|
|
|
# define rcu_lock_release(a) do { } while (0)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
|
|
|
|
static inline int rcu_read_lock_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int rcu_read_lock_bh_held(void)
|
|
|
|
{
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
{
|
2016-03-23 15:11:48 +00:00
|
|
|
return !preemptible();
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
}
|
2019-07-16 22:12:22 +00:00
|
|
|
|
|
|
|
static inline int rcu_read_lock_any_held(void)
|
|
|
|
{
|
|
|
|
return !preemptible();
|
|
|
|
}
|
|
|
|
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
|
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
|
2015-06-18 22:50:02 +00:00
|
|
|
/**
|
|
|
|
* RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
|
|
|
|
* @c: condition to check
|
|
|
|
* @s: informative message
|
|
|
|
*/
|
|
|
|
#define RCU_LOCKDEP_WARN(c, s) \
|
|
|
|
do { \
|
2020-10-22 02:36:07 +00:00
|
|
|
static bool __section(".data.unlikely") __warned; \
|
rcu: Reject RCU_LOCKDEP_WARN() false positives
If another lockdep report runs concurrently with an RCU lockdep report
from RCU_LOCKDEP_WARN(), the following sequence of events can occur:
1. debug_lockdep_rcu_enabled() sees that lockdep is enabled
when called from (say) synchronize_rcu().
2. Lockdep is disabled by a concurrent lockdep report.
3. debug_lockdep_rcu_enabled() evaluates its lockdep-expression
argument, for example, lock_is_held(&rcu_bh_lock_map).
4. Because lockdep is now disabled, lock_is_held() plays it safe and
returns the constant 1.
5. But in this case, the constant 1 is not safe, because invoking
synchronize_rcu() under rcu_read_lock_bh() is disallowed.
6. debug_lockdep_rcu_enabled() wrongly invokes lockdep_rcu_suspicious(),
resulting in a false-positive splat.
This commit therefore changes RCU_LOCKDEP_WARN() to check
debug_lockdep_rcu_enabled() after checking the lockdep expression,
so that any "safe" returns from lock_is_held() are rejected by
debug_lockdep_rcu_enabled(). This requires memory ordering, which is
supplied by READ_ONCE(debug_locks). The resulting volatile accesses
prevent the compiler from reordering and the fact that only one variable
is being accessed prevents the underlying hardware from reordering.
The combination works for IA64, which can reorder reads to the same
location, but this is defeated by the volatile accesses, which compile
to load instructions that provide ordering.
Reported-by: syzbot+dde0cc33951735441301@syzkaller.appspotmail.com
Reported-by: Matthew Wilcox <willy@infradead.org>
Reported-by: syzbot+88e4f02896967fe1ab0d@syzkaller.appspotmail.com
Reported-by: Thomas Gleixner <tglx@linutronix.de>
Suggested-by: Boqun Feng <boqun.feng@gmail.com>
Reviewed-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-04-05 16:51:05 +00:00
|
|
|
if ((c) && debug_lockdep_rcu_enabled() && !__warned) { \
|
2015-06-18 22:50:02 +00:00
|
|
|
__warned = true; \
|
|
|
|
lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
2012-01-12 21:49:19 +00:00
|
|
|
#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
|
|
|
|
static inline void rcu_preempt_sleep_check(void)
|
|
|
|
{
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
|
|
|
|
"Illegal context switch in RCU read-side critical section");
|
2012-01-12 21:49:19 +00:00
|
|
|
}
|
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
2017-05-03 17:52:10 +00:00
|
|
|
static inline void rcu_preempt_sleep_check(void) { }
|
2012-01-12 21:49:19 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
2011-05-24 15:31:09 +00:00
|
|
|
#define rcu_sleep_check() \
|
|
|
|
do { \
|
2012-01-12 21:49:19 +00:00
|
|
|
rcu_preempt_sleep_check(); \
|
2021-03-09 08:55:58 +00:00
|
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) \
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map), \
|
2015-06-18 22:50:02 +00:00
|
|
|
"Illegal context switch in RCU-bh read-side critical section"); \
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map), \
|
|
|
|
"Illegal context switch in RCU-sched read-side critical section"); \
|
2011-05-24 15:31:09 +00:00
|
|
|
} while (0)
|
|
|
|
|
2010-04-28 21:39:09 +00:00
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
2020-09-16 18:45:28 +00:00
|
|
|
#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))
|
2011-05-24 15:31:09 +00:00
|
|
|
#define rcu_sleep_check() do { } while (0)
|
2010-04-28 21:39:09 +00:00
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Helper functions for rcu_dereference_check(), rcu_dereference_protected()
|
|
|
|
* and rcu_assign_pointer(). Some of these could be folded into their
|
|
|
|
* callers, but they are left separate in order to ease introduction of
|
2018-07-08 01:12:26 +00:00
|
|
|
* multiple pointers markings to match different RCU implementations
|
|
|
|
* (e.g., __srcu), should this make sense in the future.
|
2010-04-28 21:39:09 +00:00
|
|
|
*/
|
2010-09-14 00:24:21 +00:00
|
|
|
|
|
|
|
#ifdef __CHECKER__
|
2018-12-12 22:37:10 +00:00
|
|
|
#define rcu_check_sparse(p, space) \
|
2010-09-14 00:24:21 +00:00
|
|
|
((void)(((typeof(*p) space *)p) == p))
|
|
|
|
#else /* #ifdef __CHECKER__ */
|
2018-12-12 22:37:10 +00:00
|
|
|
#define rcu_check_sparse(p, space)
|
2010-09-14 00:24:21 +00:00
|
|
|
#endif /* #else #ifdef __CHECKER__ */
|
|
|
|
|
2021-09-15 09:02:18 +00:00
|
|
|
#define __unrcu_pointer(p, local) \
|
|
|
|
({ \
|
|
|
|
typeof(*p) *local = (typeof(*p) *__force)(p); \
|
|
|
|
rcu_check_sparse(p, __rcu); \
|
|
|
|
((typeof(*p) __force __kernel *)(local)); \
|
|
|
|
})
|
2021-04-21 21:30:54 +00:00
|
|
|
/**
|
|
|
|
* unrcu_pointer - mark a pointer as not being RCU protected
|
|
|
|
* @p: pointer needing to lose its __rcu property
|
|
|
|
*
|
|
|
|
* Converts @p from an __rcu pointer to a __kernel pointer.
|
|
|
|
* This allows an __rcu pointer to be used with xchg() and friends.
|
|
|
|
*/
|
2021-09-15 09:02:18 +00:00
|
|
|
#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu))
|
2021-04-21 21:30:54 +00:00
|
|
|
|
2021-09-15 09:02:18 +00:00
|
|
|
#define __rcu_access_pointer(p, local, space) \
|
2013-12-06 00:19:15 +00:00
|
|
|
({ \
|
2021-09-15 09:02:18 +00:00
|
|
|
typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
|
2018-12-12 22:37:10 +00:00
|
|
|
rcu_check_sparse(p, space); \
|
2021-09-15 09:02:18 +00:00
|
|
|
((typeof(*p) __force __kernel *)(local)); \
|
2013-12-06 00:19:15 +00:00
|
|
|
})
|
2021-09-15 09:02:18 +00:00
|
|
|
#define __rcu_dereference_check(p, local, c, space) \
|
2013-12-06 00:19:15 +00:00
|
|
|
({ \
|
2014-11-13 19:24:14 +00:00
|
|
|
/* Dependency order vs. p above. */ \
|
2021-09-15 09:02:18 +00:00
|
|
|
typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
|
2018-12-12 22:37:10 +00:00
|
|
|
rcu_check_sparse(p, space); \
|
2021-09-15 09:02:18 +00:00
|
|
|
((typeof(*p) __force __kernel *)(local)); \
|
2013-12-06 00:19:15 +00:00
|
|
|
})
|
2021-09-15 09:02:18 +00:00
|
|
|
#define __rcu_dereference_protected(p, local, c, space) \
|
2013-12-06 00:19:15 +00:00
|
|
|
({ \
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
|
2018-12-12 22:37:10 +00:00
|
|
|
rcu_check_sparse(p, space); \
|
2013-12-06 00:19:15 +00:00
|
|
|
((typeof(*p) __force __kernel *)(p)); \
|
|
|
|
})
|
2021-09-15 09:02:18 +00:00
|
|
|
#define __rcu_dereference_raw(p, local) \
|
2016-07-01 20:44:53 +00:00
|
|
|
({ \
|
|
|
|
/* Dependency order vs. p above. */ \
|
2021-09-15 09:02:18 +00:00
|
|
|
typeof(p) local = READ_ONCE(p); \
|
|
|
|
((typeof(*p) __force __kernel *)(local)); \
|
2016-07-01 20:44:53 +00:00
|
|
|
})
|
2021-09-15 09:02:18 +00:00
|
|
|
#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu))
|
2010-04-28 21:39:09 +00:00
|
|
|
|
2013-11-11 17:59:34 +00:00
|
|
|
/**
|
|
|
|
* RCU_INITIALIZER() - statically initialize an RCU-protected global variable
|
|
|
|
* @v: The value to statically initialize with.
|
|
|
|
*/
|
|
|
|
#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_assign_pointer() - assign to RCU-protected pointer
|
|
|
|
* @p: pointer to assign to
|
|
|
|
* @v: value to assign (publish)
|
|
|
|
*
|
|
|
|
* Assigns the specified value to the specified RCU-protected
|
|
|
|
* pointer, ensuring that any concurrent RCU readers will see
|
|
|
|
* any prior initialization.
|
|
|
|
*
|
|
|
|
* Inserts memory barriers on architectures that require them
|
|
|
|
* (which is most of them), and also prevents the compiler from
|
|
|
|
* reordering the code that initializes the structure after the pointer
|
|
|
|
* assignment. More importantly, this call documents which pointers
|
|
|
|
* will be dereferenced by RCU read-side code.
|
|
|
|
*
|
|
|
|
* In some special cases, you may use RCU_INIT_POINTER() instead
|
|
|
|
* of rcu_assign_pointer(). RCU_INIT_POINTER() is a bit faster due
|
|
|
|
* to the fact that it does not constrain either the CPU or the compiler.
|
|
|
|
* That said, using RCU_INIT_POINTER() when you should have used
|
|
|
|
* rcu_assign_pointer() is a very bad thing that results in
|
|
|
|
* impossible-to-diagnose memory corruption. So please be careful.
|
|
|
|
* See the RCU_INIT_POINTER() comment header for details.
|
|
|
|
*
|
|
|
|
* Note that rcu_assign_pointer() evaluates each of its arguments only
|
|
|
|
* once, appearances notwithstanding. One of the "extra" evaluations
|
|
|
|
* is in typeof() and the other visible only to sparse (__CHECKER__),
|
|
|
|
* neither of which actually execute the argument. As with most cpp
|
|
|
|
* macros, this execute-arguments-only-once property is important, so
|
|
|
|
* please be careful when making changes to rcu_assign_pointer() and the
|
|
|
|
* other macros that it invokes.
|
|
|
|
*/
|
2016-05-02 01:46:54 +00:00
|
|
|
#define rcu_assign_pointer(p, v) \
|
2019-05-27 08:49:57 +00:00
|
|
|
do { \
|
2016-05-02 01:46:54 +00:00
|
|
|
uintptr_t _r_a_p__v = (uintptr_t)(v); \
|
2019-05-22 17:07:45 +00:00
|
|
|
rcu_check_sparse(p, __rcu); \
|
2016-05-02 01:46:54 +00:00
|
|
|
\
|
|
|
|
if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL) \
|
|
|
|
WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \
|
|
|
|
else \
|
|
|
|
smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
|
2019-05-27 08:49:57 +00:00
|
|
|
} while (0)
|
2010-04-28 21:39:09 +00:00
|
|
|
|
2019-09-23 22:05:11 +00:00
|
|
|
/**
|
|
|
|
* rcu_replace_pointer() - replace an RCU pointer, returning its old value
|
|
|
|
* @rcu_ptr: RCU pointer, whose old value is returned
|
|
|
|
* @ptr: regular pointer
|
|
|
|
* @c: the lockdep conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* Perform a replacement, where @rcu_ptr is an RCU-annotated
|
|
|
|
* pointer and @c is the lockdep argument that is passed to the
|
|
|
|
* rcu_dereference_protected() call used to read that pointer. The old
|
|
|
|
* value of @rcu_ptr is returned, and @rcu_ptr is set to @ptr.
|
|
|
|
*/
|
|
|
|
#define rcu_replace_pointer(rcu_ptr, ptr, c) \
|
|
|
|
({ \
|
|
|
|
typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c)); \
|
|
|
|
rcu_assign_pointer((rcu_ptr), (ptr)); \
|
|
|
|
__tmp; \
|
|
|
|
})
|
|
|
|
|
2010-04-28 21:39:09 +00:00
|
|
|
/**
|
|
|
|
* rcu_access_pointer() - fetch RCU pointer with no dereferencing
|
|
|
|
* @p: The pointer to read
|
|
|
|
*
|
|
|
|
* Return the value of the specified RCU-protected pointer, but omit the
|
2017-10-09 17:30:58 +00:00
|
|
|
* lockdep checks for being in an RCU read-side critical section. This is
|
|
|
|
* useful when the value of this pointer is accessed, but the pointer is
|
|
|
|
* not dereferenced, for example, when testing an RCU-protected pointer
|
|
|
|
* against NULL. Although rcu_access_pointer() may also be used in cases
|
|
|
|
* where update-side locks prevent the value of the pointer from changing,
|
|
|
|
* you should instead use rcu_dereference_protected() for this use case.
|
2012-01-13 01:21:20 +00:00
|
|
|
*
|
|
|
|
* It is also permissible to use rcu_access_pointer() when read-side
|
|
|
|
* access to the pointer was removed at least one grace period ago, as
|
|
|
|
* is the case in the context of the RCU callback that is freeing up
|
|
|
|
* the data, or after a synchronize_rcu() returns. This can be useful
|
|
|
|
* when tearing down multi-linked structures after a grace period
|
|
|
|
* has elapsed.
|
2010-04-28 21:39:09 +00:00
|
|
|
*/
|
2021-09-15 09:02:18 +00:00
|
|
|
#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
|
2010-04-28 21:39:09 +00:00
|
|
|
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_dereference_check() - rcu_dereference with debug checking
|
2010-04-09 22:39:11 +00:00
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
*
|
2010-04-09 22:39:11 +00:00
|
|
|
* Do an rcu_dereference(), but check that the conditions under which the
|
2010-04-28 21:39:09 +00:00
|
|
|
* dereference will take place are correct. Typically the conditions
|
|
|
|
* indicate the various locking conditions that should be held at that
|
|
|
|
* point. The check should return true if the conditions are satisfied.
|
|
|
|
* An implicit check for being in an RCU read-side critical section
|
|
|
|
* (rcu_read_lock()) is included.
|
2010-04-09 22:39:11 +00:00
|
|
|
*
|
|
|
|
* For example:
|
|
|
|
*
|
2010-04-28 21:39:09 +00:00
|
|
|
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
|
2010-04-09 22:39:11 +00:00
|
|
|
*
|
|
|
|
* could be used to indicate to lockdep that foo->bar may only be dereferenced
|
2010-04-28 21:39:09 +00:00
|
|
|
* if either rcu_read_lock() is held, or that the lock required to replace
|
2010-04-09 22:39:11 +00:00
|
|
|
* the bar struct at foo->bar is held.
|
|
|
|
*
|
|
|
|
* Note that the list of conditions may also include indications of when a lock
|
|
|
|
* need not be held, for example during initialisation or destruction of the
|
|
|
|
* target struct:
|
|
|
|
*
|
2010-04-28 21:39:09 +00:00
|
|
|
* bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
|
2010-04-09 22:39:11 +00:00
|
|
|
* atomic_read(&foo->usage) == 0);
|
2010-04-28 21:39:09 +00:00
|
|
|
*
|
|
|
|
* Inserts memory barriers on architectures that require them
|
|
|
|
* (currently only the Alpha), prevents the compiler from refetching
|
|
|
|
* (and from merging fetches), and, more importantly, documents exactly
|
|
|
|
* which pointers are protected by RCU and checks that the pointer is
|
|
|
|
* annotated as __rcu.
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_check(p, c) \
|
2021-09-15 09:02:18 +00:00
|
|
|
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
|
|
|
|
(c) || rcu_read_lock_held(), __rcu)
|
2010-04-28 21:39:09 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
2021-04-29 18:18:01 +00:00
|
|
|
* This is the RCU-bh counterpart to rcu_dereference_check(). However,
|
|
|
|
* please note that starting in v5.0 kernels, vanilla RCU grace periods
|
|
|
|
* wait for local_bh_disable() regions of code in addition to regions of
|
|
|
|
* code demarked by rcu_read_lock() and rcu_read_unlock(). This means
|
|
|
|
* that synchronize_rcu(), call_rcu, and friends all take not only
|
|
|
|
* rcu_read_lock() but also rcu_read_lock_bh() into account.
|
2010-04-28 21:39:09 +00:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_bh_check(p, c) \
|
2021-09-15 09:02:18 +00:00
|
|
|
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
|
|
|
|
(c) || rcu_read_lock_bh_held(), __rcu)
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-23 01:04:45 +00:00
|
|
|
|
2010-04-09 22:39:10 +00:00
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
|
|
|
*
|
|
|
|
* This is the RCU-sched counterpart to rcu_dereference_check().
|
2021-04-29 18:18:01 +00:00
|
|
|
* However, please note that starting in v5.0 kernels, vanilla RCU grace
|
|
|
|
* periods wait for preempt_disable() regions of code in addition to
|
|
|
|
* regions of code demarked by rcu_read_lock() and rcu_read_unlock().
|
|
|
|
* This means that synchronize_rcu(), call_rcu, and friends all take not
|
|
|
|
* only rcu_read_lock() but also rcu_read_lock_sched() into account.
|
2010-04-28 21:39:09 +00:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_sched_check(p, c) \
|
2021-09-15 09:02:18 +00:00
|
|
|
__rcu_dereference_check((p), __UNIQUE_ID(rcu), \
|
|
|
|
(c) || rcu_read_lock_sched_held(), \
|
2010-04-28 21:39:09 +00:00
|
|
|
__rcu)
|
|
|
|
|
2013-05-28 18:38:42 +00:00
|
|
|
/*
|
|
|
|
* The tracing infrastructure traces RCU (we want that), but unfortunately
|
|
|
|
* some of the RCU checks causes tracing to lock up the system.
|
|
|
|
*
|
2015-11-02 02:21:47 +00:00
|
|
|
* The no-tracing version of rcu_dereference_raw() must not call
|
2013-05-28 18:38:42 +00:00
|
|
|
* rcu_read_lock_held().
|
|
|
|
*/
|
2021-09-15 09:02:18 +00:00
|
|
|
#define rcu_dereference_raw_check(p) \
|
|
|
|
__rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu)
|
2013-05-28 18:38:42 +00:00
|
|
|
|
2010-04-28 21:39:09 +00:00
|
|
|
/**
|
|
|
|
* rcu_dereference_protected() - fetch RCU pointer when updates prevented
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
* @c: The conditions under which the dereference will take place
|
2010-04-09 22:39:10 +00:00
|
|
|
*
|
|
|
|
* Return the value of the specified RCU-protected pointer, but omit
|
2017-10-09 17:30:58 +00:00
|
|
|
* the READ_ONCE(). This is useful in cases where update-side locks
|
|
|
|
* prevent the value of the pointer from changing. Please note that this
|
|
|
|
* primitive does *not* prevent the compiler from repeating this reference
|
|
|
|
* or combining it with other references, so it should not be used without
|
|
|
|
* protection of appropriate locks.
|
2010-04-28 21:39:09 +00:00
|
|
|
*
|
|
|
|
* This function is only for update-side use. Using this function
|
|
|
|
* when protected only by rcu_read_lock() will result in infrequent
|
|
|
|
* but very ugly failures.
|
2010-04-09 22:39:10 +00:00
|
|
|
*/
|
|
|
|
#define rcu_dereference_protected(p, c) \
|
2021-09-15 09:02:18 +00:00
|
|
|
__rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu)
|
2010-04-09 22:39:10 +00:00
|
|
|
|
2009-08-22 20:56:47 +00:00
|
|
|
|
2010-04-09 22:39:10 +00:00
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_dereference() - fetch RCU-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
2010-04-09 22:39:10 +00:00
|
|
|
*
|
2010-04-28 21:39:09 +00:00
|
|
|
* This is a simple wrapper around rcu_dereference_check().
|
2010-04-09 22:39:10 +00:00
|
|
|
*/
|
2010-04-28 21:39:09 +00:00
|
|
|
#define rcu_dereference(p) rcu_dereference_check(p, 0)
|
2010-04-09 22:39:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
*
|
|
|
|
* Makes rcu_dereference_check() do the dirty work.
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)
|
|
|
|
|
|
|
|
/**
|
|
|
|
* rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
|
|
|
|
* @p: The pointer to read, prior to dereferencing
|
|
|
|
*
|
|
|
|
* Makes rcu_dereference_check() do the dirty work.
|
|
|
|
*/
|
|
|
|
#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)
|
|
|
|
|
2015-09-10 23:29:02 +00:00
|
|
|
/**
|
|
|
|
* rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
|
|
|
|
* @p: The pointer to hand off
|
|
|
|
*
|
|
|
|
* This is simply an identity function, but it documents where a pointer
|
|
|
|
* is handed off from RCU to some other synchronization mechanism, for
|
|
|
|
* example, reference counting or locking. In C11, it would map to
|
2018-05-07 09:35:46 +00:00
|
|
|
* kill_dependency(). It could be used as follows::
|
|
|
|
*
|
2015-09-10 23:29:02 +00:00
|
|
|
* rcu_read_lock();
|
|
|
|
* p = rcu_dereference(gp);
|
|
|
|
* long_lived = is_long_lived(p);
|
|
|
|
* if (long_lived) {
|
|
|
|
* if (!atomic_inc_not_zero(p->refcnt))
|
|
|
|
* long_lived = false;
|
|
|
|
* else
|
|
|
|
* p = rcu_pointer_handoff(p);
|
|
|
|
* }
|
|
|
|
* rcu_read_unlock();
|
|
|
|
*/
|
|
|
|
#define rcu_pointer_handoff(p) (p)
|
|
|
|
|
2010-04-28 21:39:09 +00:00
|
|
|
/**
|
|
|
|
* rcu_read_lock() - mark the beginning of an RCU read-side critical section
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2005-05-01 15:59:04 +00:00
|
|
|
* When synchronize_rcu() is invoked on one CPU while other CPUs
|
2005-04-16 22:20:36 +00:00
|
|
|
* are within RCU read-side critical sections, then the
|
2005-05-01 15:59:04 +00:00
|
|
|
* synchronize_rcu() is guaranteed to block until after all the other
|
2005-04-16 22:20:36 +00:00
|
|
|
* CPUs exit their critical sections. Similarly, if call_rcu() is invoked
|
|
|
|
* on one CPU while other CPUs are within RCU read-side critical
|
|
|
|
* sections, invocation of the corresponding RCU callback is deferred
|
|
|
|
* until after the all the other CPUs exit their critical sections.
|
|
|
|
*
|
2021-04-29 18:18:01 +00:00
|
|
|
* In v5.0 and later kernels, synchronize_rcu() and call_rcu() also
|
|
|
|
* wait for regions of code with preemption disabled, including regions of
|
|
|
|
* code with interrupts or softirqs disabled. In pre-v5.0 kernels, which
|
|
|
|
* define synchronize_sched(), only code enclosed within rcu_read_lock()
|
|
|
|
* and rcu_read_unlock() are guaranteed to be waited for.
|
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Note, however, that RCU callbacks are permitted to run concurrently
|
2010-07-09 00:38:59 +00:00
|
|
|
* with new RCU read-side critical sections. One way that this can happen
|
2005-04-16 22:20:36 +00:00
|
|
|
* is via the following sequence of events: (1) CPU 0 enters an RCU
|
|
|
|
* read-side critical section, (2) CPU 1 invokes call_rcu() to register
|
|
|
|
* an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
|
|
|
|
* (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
|
|
|
|
* callback is invoked. This is legal, because the RCU read-side critical
|
|
|
|
* section that was running concurrently with the call_rcu() (and which
|
|
|
|
* therefore might be referencing something that the corresponding RCU
|
|
|
|
* callback would free up) has completed before the corresponding
|
|
|
|
* RCU callback is invoked.
|
|
|
|
*
|
|
|
|
* RCU read-side critical sections may be nested. Any deferred actions
|
|
|
|
* will be deferred until the outermost RCU read-side critical section
|
|
|
|
* completes.
|
|
|
|
*
|
2010-08-08 04:59:54 +00:00
|
|
|
* You can avoid reading and understanding the next paragraph by
|
|
|
|
* following this rule: don't put anything in an rcu_read_lock() RCU
|
2019-10-15 19:18:14 +00:00
|
|
|
* read-side critical section that would block in a !PREEMPTION kernel.
|
2010-08-08 04:59:54 +00:00
|
|
|
* But if you want the full story, read on!
|
|
|
|
*
|
2019-10-15 02:55:57 +00:00
|
|
|
* In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
|
2014-05-04 22:41:21 +00:00
|
|
|
* it is illegal to block while in an RCU read-side critical section.
|
2019-07-26 21:19:38 +00:00
|
|
|
* In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
|
2014-05-04 22:41:21 +00:00
|
|
|
* kernel builds, RCU read-side critical sections may be preempted,
|
|
|
|
* but explicit blocking is illegal. Finally, in preemptible RCU
|
|
|
|
* implementations in real-time (with -rt patchset) kernel builds, RCU
|
|
|
|
* read-side critical sections may be preempted and they may also block, but
|
|
|
|
* only when acquiring spinlocks that are subject to priority inheritance.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2019-05-21 20:48:43 +00:00
|
|
|
static __always_inline void rcu_read_lock(void)
|
2009-08-22 20:56:47 +00:00
|
|
|
{
|
|
|
|
__rcu_read_lock();
|
|
|
|
__acquire(RCU);
|
2011-10-07 16:22:03 +00:00
|
|
|
rcu_lock_acquire(&rcu_lock_map);
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_lock() used illegally while idle");
|
2009-08-22 20:56:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* So where is rcu_write_lock()? It does not exist, as there is no
|
|
|
|
* way for writers to lock out RCU readers. This is a feature, not
|
|
|
|
* a bug -- this property is what provides RCU's performance benefits.
|
|
|
|
* Of course, writers must coordinate with each other. The normal
|
|
|
|
* spinlock primitives work well for this, but any other technique may be
|
|
|
|
* used as well. RCU does not care how the writers keep out of each
|
|
|
|
* others' way, as long as they do so.
|
|
|
|
*/
|
2009-09-28 14:46:32 +00:00
|
|
|
|
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_read_unlock() - marks the end of an RCU read-side critical section.
|
2009-09-28 14:46:32 +00:00
|
|
|
*
|
2021-04-29 18:30:49 +00:00
|
|
|
* In almost all situations, rcu_read_unlock() is immune from deadlock.
|
|
|
|
* In recent kernels that have consolidated synchronize_sched() and
|
|
|
|
* synchronize_rcu_bh() into synchronize_rcu(), this deadlock immunity
|
|
|
|
* also extends to the scheduler's runqueue and priority-inheritance
|
|
|
|
* spinlocks, courtesy of the quiescent-state deferral that is carried
|
|
|
|
* out when rcu_read_unlock() is invoked with interrupts disabled.
|
2014-05-04 22:38:38 +00:00
|
|
|
*
|
2009-09-28 14:46:32 +00:00
|
|
|
* See rcu_read_lock() for more information.
|
|
|
|
*/
|
2009-08-22 20:56:47 +00:00
|
|
|
static inline void rcu_read_unlock(void)
|
|
|
|
{
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_unlock() used illegally while idle");
|
2009-08-22 20:56:47 +00:00
|
|
|
__release(RCU);
|
|
|
|
__rcu_read_unlock();
|
2015-01-21 23:26:03 +00:00
|
|
|
rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
|
2009-08-22 20:56:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
2021-04-29 18:18:01 +00:00
|
|
|
* This is equivalent to rcu_read_lock(), but also disables softirqs.
|
|
|
|
* Note that anything else that disables softirqs can also serve as an RCU
|
|
|
|
* read-side critical section. However, please note that this equivalence
|
|
|
|
* applies only to v5.0 and later. Before v5.0, rcu_read_lock() and
|
|
|
|
* rcu_read_lock_bh() were unrelated.
|
2011-11-28 18:42:42 +00:00
|
|
|
*
|
|
|
|
* Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
|
|
|
|
* must occur in the same context, for example, it is illegal to invoke
|
|
|
|
* rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
|
|
|
|
* was invoked from some other task.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-08-22 20:56:47 +00:00
|
|
|
static inline void rcu_read_lock_bh(void)
|
|
|
|
{
|
2011-08-01 13:22:11 +00:00
|
|
|
local_bh_disable();
|
2009-08-22 20:56:47 +00:00
|
|
|
__acquire(RCU_BH);
|
2011-10-07 16:22:03 +00:00
|
|
|
rcu_lock_acquire(&rcu_bh_lock_map);
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_lock_bh() used illegally while idle");
|
2009-08-22 20:56:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-07-09 13:05:59 +00:00
|
|
|
/**
|
|
|
|
* rcu_read_unlock_bh() - marks the end of a softirq-only RCU critical section
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* See rcu_read_lock_bh() for more information.
|
|
|
|
*/
|
2009-08-22 20:56:47 +00:00
|
|
|
static inline void rcu_read_unlock_bh(void)
|
|
|
|
{
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_unlock_bh() used illegally while idle");
|
2011-10-07 16:22:03 +00:00
|
|
|
rcu_lock_release(&rcu_bh_lock_map);
|
2009-08-22 20:56:47 +00:00
|
|
|
__release(RCU_BH);
|
2011-08-01 13:22:11 +00:00
|
|
|
local_bh_enable();
|
2009-08-22 20:56:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-09-29 15:06:46 +00:00
|
|
|
/**
|
2010-04-28 21:39:09 +00:00
|
|
|
* rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
|
2008-09-29 15:06:46 +00:00
|
|
|
*
|
2021-04-29 18:18:01 +00:00
|
|
|
* This is equivalent to rcu_read_lock(), but also disables preemption.
|
|
|
|
* Read-side critical sections can also be introduced by anything else that
|
|
|
|
* disables preemption, including local_irq_disable() and friends. However,
|
|
|
|
* please note that the equivalence to rcu_read_lock() applies only to
|
|
|
|
* v5.0 and later. Before v5.0, rcu_read_lock() and rcu_read_lock_sched()
|
|
|
|
* were unrelated.
|
2011-11-28 18:42:42 +00:00
|
|
|
*
|
|
|
|
* Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
|
|
|
|
* must occur in the same context, for example, it is illegal to invoke
|
|
|
|
* rcu_read_unlock_sched() from process context if the matching
|
|
|
|
* rcu_read_lock_sched() was invoked from an NMI handler.
|
2008-09-29 15:06:46 +00:00
|
|
|
*/
|
2009-08-22 20:56:46 +00:00
|
|
|
static inline void rcu_read_lock_sched(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
2009-08-22 20:56:47 +00:00
|
|
|
__acquire(RCU_SCHED);
|
2011-10-07 16:22:03 +00:00
|
|
|
rcu_lock_acquire(&rcu_sched_lock_map);
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_lock_sched() used illegally while idle");
|
2009-08-22 20:56:46 +00:00
|
|
|
}
|
2009-09-23 16:50:42 +00:00
|
|
|
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
2009-08-24 16:42:00 +00:00
|
|
|
static inline notrace void rcu_read_lock_sched_notrace(void)
|
2009-08-22 20:56:46 +00:00
|
|
|
{
|
|
|
|
preempt_disable_notrace();
|
2009-08-22 20:56:47 +00:00
|
|
|
__acquire(RCU_SCHED);
|
2009-08-22 20:56:46 +00:00
|
|
|
}
|
2008-09-29 15:06:46 +00:00
|
|
|
|
2020-07-09 13:05:59 +00:00
|
|
|
/**
|
|
|
|
* rcu_read_unlock_sched() - marks the end of a RCU-classic critical section
|
2008-09-29 15:06:46 +00:00
|
|
|
*
|
2020-07-09 13:05:59 +00:00
|
|
|
* See rcu_read_lock_sched() for more information.
|
2008-09-29 15:06:46 +00:00
|
|
|
*/
|
2009-08-22 20:56:46 +00:00
|
|
|
static inline void rcu_read_unlock_sched(void)
|
|
|
|
{
|
2015-06-18 22:50:02 +00:00
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
"rcu_read_unlock_sched() used illegally while idle");
|
2011-10-07 16:22:03 +00:00
|
|
|
rcu_lock_release(&rcu_sched_lock_map);
|
2009-08-22 20:56:47 +00:00
|
|
|
__release(RCU_SCHED);
|
2009-08-22 20:56:46 +00:00
|
|
|
preempt_enable();
|
|
|
|
}
|
2009-09-23 16:50:42 +00:00
|
|
|
|
|
|
|
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
|
2009-08-24 16:42:00 +00:00
|
|
|
static inline notrace void rcu_read_unlock_sched_notrace(void)
|
2009-08-22 20:56:46 +00:00
|
|
|
{
|
2009-08-22 20:56:47 +00:00
|
|
|
__release(RCU_SCHED);
|
2009-08-22 20:56:46 +00:00
|
|
|
preempt_enable_notrace();
|
|
|
|
}
|
2008-09-29 15:06:46 +00:00
|
|
|
|
2010-04-28 21:39:09 +00:00
|
|
|
/**
|
|
|
|
* RCU_INIT_POINTER() - initialize an RCU protected pointer
|
2017-10-19 21:26:21 +00:00
|
|
|
* @p: The pointer to be initialized.
|
|
|
|
* @v: The value to initialized the pointer to.
|
2010-04-28 21:39:09 +00:00
|
|
|
*
|
2011-08-01 05:33:02 +00:00
|
|
|
* Initialize an RCU-protected pointer in special cases where readers
|
|
|
|
* do not need ordering constraints on the CPU or the compiler. These
|
|
|
|
* special cases are:
|
|
|
|
*
|
2017-10-19 21:26:21 +00:00
|
|
|
* 1. This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
|
2011-08-01 05:33:02 +00:00
|
|
|
* 2. The caller has taken whatever steps are required to prevent
|
2017-10-19 21:26:21 +00:00
|
|
|
* RCU readers from concurrently accessing this pointer *or*
|
2011-08-01 05:33:02 +00:00
|
|
|
* 3. The referenced data structure has already been exposed to
|
2017-10-19 21:26:21 +00:00
|
|
|
* readers either at compile time or via rcu_assign_pointer() *and*
|
|
|
|
*
|
|
|
|
* a. You have not made *any* reader-visible changes to
|
|
|
|
* this structure since then *or*
|
2011-08-01 05:33:02 +00:00
|
|
|
* b. It is OK for readers accessing this structure from its
|
|
|
|
* new location to see the old state of the structure. (For
|
|
|
|
* example, the changes were to statistical counters or to
|
|
|
|
* other state where exact synchronization is not required.)
|
|
|
|
*
|
|
|
|
* Failure to follow these rules governing use of RCU_INIT_POINTER() will
|
|
|
|
* result in impossible-to-diagnose memory corruption. As in the structures
|
|
|
|
* will look OK in crash dumps, but any concurrent RCU readers might
|
|
|
|
* see pre-initialized values of the referenced data structure. So
|
|
|
|
* please be very careful how you use RCU_INIT_POINTER()!!!
|
|
|
|
*
|
|
|
|
* If you are creating an RCU-protected linked structure that is accessed
|
|
|
|
* by a single external-to-structure RCU-protected pointer, then you may
|
|
|
|
* use RCU_INIT_POINTER() to initialize the internal RCU-protected
|
|
|
|
* pointers, but you must use rcu_assign_pointer() to initialize the
|
2017-10-19 21:26:21 +00:00
|
|
|
* external-to-structure pointer *after* you have completely initialized
|
2011-08-01 05:33:02 +00:00
|
|
|
* the reader-accessible portions of the linked structure.
|
2014-03-31 20:13:02 +00:00
|
|
|
*
|
|
|
|
* Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
|
|
|
|
* ordering guarantees for either the CPU or the compiler.
|
2010-04-28 21:39:09 +00:00
|
|
|
*/
|
|
|
|
#define RCU_INIT_POINTER(p, v) \
|
2012-05-16 22:42:30 +00:00
|
|
|
do { \
|
2018-12-12 22:37:10 +00:00
|
|
|
rcu_check_sparse(p, __rcu); \
|
2015-06-02 15:26:48 +00:00
|
|
|
WRITE_ONCE(p, RCU_INITIALIZER(v)); \
|
2012-05-16 22:42:30 +00:00
|
|
|
} while (0)
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
|
2012-05-16 22:23:45 +00:00
|
|
|
/**
|
|
|
|
* RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
|
2017-10-19 21:26:21 +00:00
|
|
|
* @p: The pointer to be initialized.
|
|
|
|
* @v: The value to initialized the pointer to.
|
2012-05-16 22:23:45 +00:00
|
|
|
*
|
|
|
|
* GCC-style initialization for an RCU-protected pointer in a structure field.
|
|
|
|
*/
|
|
|
|
#define RCU_POINTER_INITIALIZER(p, v) \
|
2013-11-11 17:59:34 +00:00
|
|
|
.p = RCU_INITIALIZER(v)
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
|
2012-04-19 18:44:39 +00:00
|
|
|
/*
|
|
|
|
* Does the specified offset indicate that the corresponding rcu_head
|
2020-05-25 21:47:55 +00:00
|
|
|
* structure can be handled by kvfree_rcu()?
|
2012-04-19 18:44:39 +00:00
|
|
|
*/
|
2020-05-25 21:47:55 +00:00
|
|
|
#define __is_kvfree_rcu_offset(offset) ((offset) < 4096)
|
2012-04-19 18:44:39 +00:00
|
|
|
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
/**
|
|
|
|
* kfree_rcu() - kfree an object after a grace period.
|
2020-11-20 11:49:16 +00:00
|
|
|
* @ptr: pointer to kfree for both single- and double-argument invocations.
|
|
|
|
* @rhf: the name of the struct rcu_head within the type of @ptr,
|
|
|
|
* but only for double-argument invocations.
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
*
|
|
|
|
* Many rcu callbacks functions just call kfree() on the base structure.
|
|
|
|
* These functions are trivial, but their size adds up, and furthermore
|
|
|
|
* when they are used in a kernel module, that module must invoke the
|
|
|
|
* high-latency rcu_barrier() function at module-unload time.
|
|
|
|
*
|
|
|
|
* The kfree_rcu() function handles this issue. Rather than encoding a
|
|
|
|
* function address in the embedded rcu_head structure, kfree_rcu() instead
|
|
|
|
* encodes the offset of the rcu_head structure within the base structure.
|
|
|
|
* Because the functions are not allowed in the low-order 4096 bytes of
|
|
|
|
* kernel virtual memory, offsets up to 4095 bytes can be accommodated.
|
|
|
|
* If the offset is larger than 4095 bytes, a compile-time error will
|
2020-11-20 11:49:17 +00:00
|
|
|
* be generated in kvfree_rcu_arg_2(). If this error is triggered, you can
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
* either fall back to use of call_rcu() or rearrange the structure to
|
|
|
|
* position the rcu_head structure into the first 4096 bytes.
|
|
|
|
*
|
|
|
|
* Note that the allowable offset might decrease in the future, for example,
|
|
|
|
* to allow something like kmem_cache_free_rcu().
|
2012-04-19 18:44:39 +00:00
|
|
|
*
|
|
|
|
* The BUILD_BUG_ON check must not involve any function calls, hence the
|
|
|
|
* checks are done in macros here.
|
rcu: introduce kfree_rcu()
Many rcu callbacks functions just call kfree() on the base structure.
These functions are trivial, but their size adds up, and furthermore
when they are used in a kernel module, that module must invoke the
high-latency rcu_barrier() function at module-unload time.
The kfree_rcu() function introduced by this commit addresses this issue.
Rather than encoding a function address in the embedded rcu_head
structure, kfree_rcu() instead encodes the offset of the rcu_head
structure within the base structure. Because the functions are not
allowed in the low-order 4096 bytes of kernel virtual memory, offsets
up to 4095 bytes can be accommodated. If the offset is larger than
4095 bytes, a compile-time error will be generated in __kfree_rcu().
If this error is triggered, you can either fall back to use of call_rcu()
or rearrange the structure to position the rcu_head structure into the
first 4096 bytes.
Note that the allowable offset might decrease in the future, for example,
to allow something like kmem_cache_free_rcu().
The new kfree_rcu() function can replace code as follows:
call_rcu(&p->rcu, simple_kfree_callback);
where "simple_kfree_callback()" might be defined as follows:
void simple_kfree_callback(struct rcu_head *p)
{
struct foo *q = container_of(p, struct foo, rcu);
kfree(q);
}
with the following:
kfree_rcu(&p->rcu, rcu);
Note that the "rcu" is the name of a field in the structure being
freed. The reason for using this rather than passing in a pointer
to the base structure is that the above approach allows better type
checking.
This commit is based on earlier work by Lai Jiangshan and Manfred Spraul:
Lai's V1 patch: http://lkml.org/lkml/2008/9/18/1
Manfred's patch: http://lkml.org/lkml/2009/1/2/115
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Manfred Spraul <manfred@colorfullife.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: David Howells <dhowells@redhat.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
2011-03-18 03:15:47 +00:00
|
|
|
*/
|
2021-01-14 07:22:02 +00:00
|
|
|
#define kfree_rcu(ptr, rhf...) kvfree_rcu(ptr, ## rhf)
|
2013-06-21 23:37:22 +00:00
|
|
|
|
2020-05-25 21:47:57 +00:00
|
|
|
/**
|
|
|
|
* kvfree_rcu() - kvfree an object after a grace period.
|
|
|
|
*
|
2020-05-25 21:47:59 +00:00
|
|
|
* This macro consists of one or two arguments and it is
|
|
|
|
* based on whether an object is head-less or not. If it
|
|
|
|
* has a head then a semantic stays the same as it used
|
|
|
|
* to be before:
|
|
|
|
*
|
|
|
|
* kvfree_rcu(ptr, rhf);
|
|
|
|
*
|
|
|
|
* where @ptr is a pointer to kvfree(), @rhf is the name
|
|
|
|
* of the rcu_head structure within the type of @ptr.
|
|
|
|
*
|
|
|
|
* When it comes to head-less variant, only one argument
|
|
|
|
* is passed and that is just a pointer which has to be
|
|
|
|
* freed after a grace period. Therefore the semantic is
|
|
|
|
*
|
|
|
|
* kvfree_rcu(ptr);
|
|
|
|
*
|
|
|
|
* where @ptr is a pointer to kvfree().
|
|
|
|
*
|
|
|
|
* Please note, head-less way of freeing is permitted to
|
|
|
|
* use from a context that has to follow might_sleep()
|
|
|
|
* annotation. Otherwise, please switch and embed the
|
|
|
|
* rcu_head structure within the type of @ptr.
|
2020-05-25 21:47:57 +00:00
|
|
|
*/
|
2020-05-25 21:47:59 +00:00
|
|
|
#define kvfree_rcu(...) KVFREE_GET_MACRO(__VA_ARGS__, \
|
|
|
|
kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__)
|
|
|
|
|
|
|
|
#define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME
|
2020-11-20 11:49:16 +00:00
|
|
|
#define kvfree_rcu_arg_2(ptr, rhf) \
|
|
|
|
do { \
|
|
|
|
typeof (ptr) ___p = (ptr); \
|
|
|
|
\
|
2020-11-20 11:49:17 +00:00
|
|
|
if (___p) { \
|
|
|
|
BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \
|
|
|
|
kvfree_call_rcu(&((___p)->rhf), (rcu_callback_t)(unsigned long) \
|
|
|
|
(offsetof(typeof(*(ptr)), rhf))); \
|
|
|
|
} \
|
2020-11-20 11:49:16 +00:00
|
|
|
} while (0)
|
|
|
|
|
2020-05-25 21:47:59 +00:00
|
|
|
#define kvfree_rcu_arg_1(ptr) \
|
|
|
|
do { \
|
|
|
|
typeof(ptr) ___p = (ptr); \
|
|
|
|
\
|
|
|
|
if (___p) \
|
|
|
|
kvfree_call_rcu(NULL, (rcu_callback_t) (___p)); \
|
|
|
|
} while (0)
|
2020-05-25 21:47:57 +00:00
|
|
|
|
2016-11-28 20:08:49 +00:00
|
|
|
/*
|
|
|
|
* Place this after a lock-acquisition primitive to guarantee that
|
|
|
|
* an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies
|
|
|
|
* if the UNLOCK and LOCK are executed by the same CPU or if the
|
|
|
|
* UNLOCK and LOCK operate on the same lock variable.
|
|
|
|
*/
|
2017-01-14 21:32:50 +00:00
|
|
|
#ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
|
2016-11-28 20:08:49 +00:00
|
|
|
#define smp_mb__after_unlock_lock() smp_mb() /* Full ordering for lock. */
|
2017-01-14 21:32:50 +00:00
|
|
|
#else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
|
2016-11-28 20:08:49 +00:00
|
|
|
#define smp_mb__after_unlock_lock() do { } while (0)
|
2017-01-14 21:32:50 +00:00
|
|
|
#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
|
2016-11-28 20:08:49 +00:00
|
|
|
|
2016-03-22 02:46:04 +00:00
|
|
|
|
2018-07-24 22:28:09 +00:00
|
|
|
/* Has the specified rcu_head structure been handed to call_rcu()? */
|
|
|
|
|
2018-11-20 16:29:35 +00:00
|
|
|
/**
|
2018-07-24 22:28:09 +00:00
|
|
|
* rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
|
|
|
|
* @rhp: The rcu_head structure to initialize.
|
|
|
|
*
|
|
|
|
* If you intend to invoke rcu_head_after_call_rcu() to test whether a
|
|
|
|
* given rcu_head structure has already been passed to call_rcu(), then
|
|
|
|
* you must also invoke this rcu_head_init() function on it just after
|
|
|
|
* allocating that structure. Calls to this function must not race with
|
|
|
|
* calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
|
|
|
|
*/
|
|
|
|
static inline void rcu_head_init(struct rcu_head *rhp)
|
|
|
|
{
|
|
|
|
rhp->func = (rcu_callback_t)~0L;
|
|
|
|
}
|
|
|
|
|
2018-11-20 16:29:35 +00:00
|
|
|
/**
|
2020-07-09 13:05:59 +00:00
|
|
|
* rcu_head_after_call_rcu() - Has this rcu_head been passed to call_rcu()?
|
2018-07-24 22:28:09 +00:00
|
|
|
* @rhp: The rcu_head structure to test.
|
2018-11-20 16:29:35 +00:00
|
|
|
* @f: The function passed to call_rcu() along with @rhp.
|
2018-07-24 22:28:09 +00:00
|
|
|
*
|
|
|
|
* Returns @true if the @rhp has been passed to call_rcu() with @func,
|
|
|
|
* and @false otherwise. Emits a warning in any other case, including
|
|
|
|
* the case where @rhp has already been invoked after a grace period.
|
|
|
|
* Calls to this function must not race with callback invocation. One way
|
|
|
|
* to avoid such races is to enclose the call to rcu_head_after_call_rcu()
|
|
|
|
* in an RCU read-side critical section that includes a read-side fetch
|
|
|
|
* of the pointer to the structure containing @rhp.
|
|
|
|
*/
|
|
|
|
static inline bool
|
|
|
|
rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
|
|
|
|
{
|
2019-03-11 11:58:03 +00:00
|
|
|
rcu_callback_t func = READ_ONCE(rhp->func);
|
|
|
|
|
|
|
|
if (func == f)
|
2018-07-24 22:28:09 +00:00
|
|
|
return true;
|
2019-03-11 11:58:03 +00:00
|
|
|
WARN_ON_ONCE(func != (rcu_callback_t)~0L);
|
2018-07-24 22:28:09 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2019-10-15 13:48:22 +00:00
|
|
|
/* kernel/ksysfs.c definitions */
|
|
|
|
extern int rcu_expedited;
|
|
|
|
extern int rcu_normal;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif /* __LINUX_RCUPDATE_H */
|