mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-13 22:25:03 +00:00
rcu: Inline __call_rcu() into call_rcu()
Because __call_rcu() is invoked only by call_rcu(), this commit inlines the former into the latter. Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
parent
218b957a69
commit
1fe09ebe7a
1 changed files with 42 additions and 49 deletions
|
@ -2995,9 +2995,47 @@ static void check_cb_ovld(struct rcu_data *rdp)
|
||||||
raw_spin_unlock_rcu_node(rnp);
|
raw_spin_unlock_rcu_node(rnp);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Helper function for call_rcu() and friends. */
|
/**
|
||||||
static void
|
* call_rcu() - Queue an RCU callback for invocation after a grace period.
|
||||||
__call_rcu(struct rcu_head *head, rcu_callback_t func)
|
* @head: structure to be used for queueing the RCU updates.
|
||||||
|
* @func: actual callback function to be invoked after the grace period
|
||||||
|
*
|
||||||
|
* The callback function will be invoked some time after a full grace
|
||||||
|
* period elapses, in other words after all pre-existing RCU read-side
|
||||||
|
* critical sections have completed. However, the callback function
|
||||||
|
* might well execute concurrently with RCU read-side critical sections
|
||||||
|
* that started after call_rcu() was invoked.
|
||||||
|
*
|
||||||
|
* RCU read-side critical sections are delimited by rcu_read_lock()
|
||||||
|
* and rcu_read_unlock(), and may be nested. In addition, but only in
|
||||||
|
* v5.0 and later, regions of code across which interrupts, preemption,
|
||||||
|
* or softirqs have been disabled also serve as RCU read-side critical
|
||||||
|
* sections. This includes hardware interrupt handlers, softirq handlers,
|
||||||
|
* and NMI handlers.
|
||||||
|
*
|
||||||
|
* Note that all CPUs must agree that the grace period extended beyond
|
||||||
|
* all pre-existing RCU read-side critical section. On systems with more
|
||||||
|
* than one CPU, this means that when "func()" is invoked, each CPU is
|
||||||
|
* guaranteed to have executed a full memory barrier since the end of its
|
||||||
|
* last RCU read-side critical section whose beginning preceded the call
|
||||||
|
* to call_rcu(). It also means that each CPU executing an RCU read-side
|
||||||
|
* critical section that continues beyond the start of "func()" must have
|
||||||
|
* executed a memory barrier after the call_rcu() but before the beginning
|
||||||
|
* of that RCU read-side critical section. Note that these guarantees
|
||||||
|
* include CPUs that are offline, idle, or executing in user mode, as
|
||||||
|
* well as CPUs that are executing in the kernel.
|
||||||
|
*
|
||||||
|
* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
|
||||||
|
* resulting RCU callback function "func()", then both CPU A and CPU B are
|
||||||
|
* guaranteed to execute a full memory barrier during the time interval
|
||||||
|
* between the call to call_rcu() and the invocation of "func()" -- even
|
||||||
|
* if CPU A and CPU B are the same CPU (but again only if the system has
|
||||||
|
* more than one CPU).
|
||||||
|
*
|
||||||
|
* Implementation of these memory-ordering guarantees is described here:
|
||||||
|
* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
|
||||||
|
*/
|
||||||
|
void call_rcu(struct rcu_head *head, rcu_callback_t func)
|
||||||
{
|
{
|
||||||
static atomic_t doublefrees;
|
static atomic_t doublefrees;
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
@ -3011,7 +3049,7 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func)
|
||||||
/*
|
/*
|
||||||
* Probable double call_rcu(), so leak the callback.
|
* Probable double call_rcu(), so leak the callback.
|
||||||
* Use rcu:rcu_callback trace event to find the previous
|
* Use rcu:rcu_callback trace event to find the previous
|
||||||
* time callback was passed to __call_rcu().
|
* time callback was passed to call_rcu().
|
||||||
*/
|
*/
|
||||||
if (atomic_inc_return(&doublefrees) < 4) {
|
if (atomic_inc_return(&doublefrees) < 4) {
|
||||||
pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func);
|
pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func);
|
||||||
|
@ -3060,51 +3098,6 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func)
|
||||||
local_irq_restore(flags);
|
local_irq_restore(flags);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* call_rcu() - Queue an RCU callback for invocation after a grace period.
|
|
||||||
* @head: structure to be used for queueing the RCU updates.
|
|
||||||
* @func: actual callback function to be invoked after the grace period
|
|
||||||
*
|
|
||||||
* The callback function will be invoked some time after a full grace
|
|
||||||
* period elapses, in other words after all pre-existing RCU read-side
|
|
||||||
* critical sections have completed. However, the callback function
|
|
||||||
* might well execute concurrently with RCU read-side critical sections
|
|
||||||
* that started after call_rcu() was invoked.
|
|
||||||
*
|
|
||||||
* RCU read-side critical sections are delimited by rcu_read_lock()
|
|
||||||
* and rcu_read_unlock(), and may be nested. In addition, but only in
|
|
||||||
* v5.0 and later, regions of code across which interrupts, preemption,
|
|
||||||
* or softirqs have been disabled also serve as RCU read-side critical
|
|
||||||
* sections. This includes hardware interrupt handlers, softirq handlers,
|
|
||||||
* and NMI handlers.
|
|
||||||
*
|
|
||||||
* Note that all CPUs must agree that the grace period extended beyond
|
|
||||||
* all pre-existing RCU read-side critical section. On systems with more
|
|
||||||
* than one CPU, this means that when "func()" is invoked, each CPU is
|
|
||||||
* guaranteed to have executed a full memory barrier since the end of its
|
|
||||||
* last RCU read-side critical section whose beginning preceded the call
|
|
||||||
* to call_rcu(). It also means that each CPU executing an RCU read-side
|
|
||||||
* critical section that continues beyond the start of "func()" must have
|
|
||||||
* executed a memory barrier after the call_rcu() but before the beginning
|
|
||||||
* of that RCU read-side critical section. Note that these guarantees
|
|
||||||
* include CPUs that are offline, idle, or executing in user mode, as
|
|
||||||
* well as CPUs that are executing in the kernel.
|
|
||||||
*
|
|
||||||
* Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
|
|
||||||
* resulting RCU callback function "func()", then both CPU A and CPU B are
|
|
||||||
* guaranteed to execute a full memory barrier during the time interval
|
|
||||||
* between the call to call_rcu() and the invocation of "func()" -- even
|
|
||||||
* if CPU A and CPU B are the same CPU (but again only if the system has
|
|
||||||
* more than one CPU).
|
|
||||||
*
|
|
||||||
* Implementation of these memory-ordering guarantees is described here:
|
|
||||||
* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.
|
|
||||||
*/
|
|
||||||
void call_rcu(struct rcu_head *head, rcu_callback_t func)
|
|
||||||
{
|
|
||||||
__call_rcu(head, func);
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(call_rcu);
|
EXPORT_SYMBOL_GPL(call_rcu);
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue