mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 20:02:17 +00:00
fbb94cbd70
On PREEMPT_RT, if rcu_core() is preempted by the de-offloading process, some work, such as callbacks acceleration and invocation, may be left unattended due to the volatile checks on the offloaded state. In the worst case this work is postponed until the next rcu_pending() check that can take a jiffy to reach, which can be a problem in case of callbacks flooding. Solve that with invoking rcu_core() early in the de-offloading process. This way any work dismissed by an ongoing rcu_core() call fooled by a preempting deoffloading process will be caught up by a nearby future recall to rcu_core(), this time fully aware of the de-offloading state. Tested-by: Valentin Schneider <valentin.schneider@arm.com> Tested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: Frederic Weisbecker <frederic@kernel.org> Cc: Valentin Schneider <valentin.schneider@arm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: Josh Triplett <josh@joshtriplett.org> Cc: Joel Fernandes <joel@joelfernandes.org> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Neeraj Upadhyay <neeraju@codeaurora.org> Cc: Uladzislau Rezki <urezki@gmail.com> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
228 lines
12 KiB
C
228 lines
12 KiB
C
/* SPDX-License-Identifier: GPL-2.0+ */
|
|
/*
|
|
* RCU segmented callback lists
|
|
*
|
|
* This seemingly RCU-private file must be available to SRCU users
|
|
* because the size of the TREE SRCU srcu_struct structure depends
|
|
* on these definitions.
|
|
*
|
|
* Copyright IBM Corporation, 2017
|
|
*
|
|
* Authors: Paul E. McKenney <paulmck@linux.net.ibm.com>
|
|
*/
|
|
|
|
#ifndef __INCLUDE_LINUX_RCU_SEGCBLIST_H
|
|
#define __INCLUDE_LINUX_RCU_SEGCBLIST_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/atomic.h>
|
|
|
|
/* Simple unsegmented callback lists. */
|
|
struct rcu_cblist {
|
|
struct rcu_head *head;
|
|
struct rcu_head **tail;
|
|
long len;
|
|
};
|
|
|
|
#define RCU_CBLIST_INITIALIZER(n) { .head = NULL, .tail = &n.head }
|
|
|
|
/* Complicated segmented callback lists. ;-) */
|
|
|
|
/*
|
|
* Index values for segments in rcu_segcblist structure.
|
|
*
|
|
* The segments are as follows:
|
|
*
|
|
* [head, *tails[RCU_DONE_TAIL]):
|
|
* Callbacks whose grace period has elapsed, and thus can be invoked.
|
|
* [*tails[RCU_DONE_TAIL], *tails[RCU_WAIT_TAIL]):
|
|
* Callbacks waiting for the current GP from the current CPU's viewpoint.
|
|
* [*tails[RCU_WAIT_TAIL], *tails[RCU_NEXT_READY_TAIL]):
|
|
* Callbacks that arrived before the next GP started, again from
|
|
* the current CPU's viewpoint. These can be handled by the next GP.
|
|
* [*tails[RCU_NEXT_READY_TAIL], *tails[RCU_NEXT_TAIL]):
|
|
* Callbacks that might have arrived after the next GP started.
|
|
* There is some uncertainty as to when a given GP starts and
|
|
* ends, but a CPU knows the exact times if it is the one starting
|
|
* or ending the GP. Other CPUs know that the previous GP ends
|
|
* before the next one starts.
|
|
*
|
|
* Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also
|
|
* empty.
|
|
*
|
|
* The ->gp_seq[] array contains the grace-period number at which the
|
|
* corresponding segment of callbacks will be ready to invoke. A given
|
|
* element of this array is meaningful only when the corresponding segment
|
|
* is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks
|
|
* are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have
|
|
* not yet been assigned a grace-period number).
|
|
*/
|
|
#define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */
|
|
#define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */
|
|
#define RCU_NEXT_READY_TAIL 2 /* Also RCU_NEXT head. */
|
|
#define RCU_NEXT_TAIL 3
|
|
#define RCU_CBLIST_NSEGS 4
|
|
|
|
|
|
/*
|
|
* ==NOCB Offloading state machine==
|
|
*
|
|
*
|
|
* ----------------------------------------------------------------------------
|
|
* | SEGCBLIST_RCU_CORE |
|
|
* | |
|
|
* | Callbacks processed by rcu_core() from softirqs or local |
|
|
* | rcuc kthread, without holding nocb_lock. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* ----------------------------------------------------------------------------
|
|
* | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED |
|
|
* | |
|
|
* | Callbacks processed by rcu_core() from softirqs or local |
|
|
* | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |
|
|
* | allowing nocb_timer to be armed. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* -----------------------------------
|
|
* | |
|
|
* v v
|
|
* --------------------------------------- ----------------------------------|
|
|
* | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | |
|
|
* | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
|
|
* | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |
|
|
* | | | |
|
|
* | | | |
|
|
* | CB kthread woke up and | | GP kthread woke up and |
|
|
* | acknowledged SEGCBLIST_OFFLOADED. | | acknowledged SEGCBLIST_OFFLOADED|
|
|
* | Processes callbacks concurrently | | |
|
|
* | with rcu_core(), holding | | |
|
|
* | nocb_lock. | | |
|
|
* --------------------------------------- -----------------------------------
|
|
* | |
|
|
* -----------------------------------
|
|
* |
|
|
* v
|
|
* |--------------------------------------------------------------------------|
|
|
* | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_OFFLOADED | |
|
|
* | SEGCBLIST_KTHREAD_GP | |
|
|
* | SEGCBLIST_KTHREAD_CB |
|
|
* | |
|
|
* | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |
|
|
* | handling callbacks. Enable bypass queueing. |
|
|
* ----------------------------------------------------------------------------
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
* ==NOCB De-Offloading state machine==
|
|
*
|
|
*
|
|
* |--------------------------------------------------------------------------|
|
|
* | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_OFFLOADED | |
|
|
* | SEGCBLIST_KTHREAD_CB | |
|
|
* | SEGCBLIST_KTHREAD_GP |
|
|
* | |
|
|
* | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
|
|
* | ignores callbacks. Bypass enqueue is enabled. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* |--------------------------------------------------------------------------|
|
|
* | SEGCBLIST_RCU_CORE | |
|
|
* | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_OFFLOADED | |
|
|
* | SEGCBLIST_KTHREAD_CB | |
|
|
* | SEGCBLIST_KTHREAD_GP |
|
|
* | |
|
|
* | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
|
|
* | handles callbacks concurrently. Bypass enqueue is enabled. |
|
|
* | Invoke RCU core so we make sure not to preempt it in the middle with |
|
|
* | leaving some urgent work unattended within a jiffy. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* |--------------------------------------------------------------------------|
|
|
* | SEGCBLIST_RCU_CORE | |
|
|
* | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_KTHREAD_CB | |
|
|
* | SEGCBLIST_KTHREAD_GP |
|
|
* | |
|
|
* | CB/GP kthreads and local rcu_core() handle callbacks concurrently |
|
|
* | holding nocb_lock. Wake up CB and GP kthreads if necessary. Disable |
|
|
* | bypass enqueue. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* -----------------------------------
|
|
* | |
|
|
* v v
|
|
* ---------------------------------------------------------------------------|
|
|
* | | |
|
|
* | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | |
|
|
* | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | |
|
|
* | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
|
|
* | | |
|
|
* | GP kthread woke up and | CB kthread woke up and |
|
|
* | acknowledged the fact that | acknowledged the fact that |
|
|
* | SEGCBLIST_OFFLOADED got cleared. | SEGCBLIST_OFFLOADED got cleared. |
|
|
* | | The CB kthread goes to sleep |
|
|
* | The callbacks from the target CPU | until it ever gets re-offloaded. |
|
|
* | will be ignored from the GP kthread | |
|
|
* | loop. | |
|
|
* ----------------------------------------------------------------------------
|
|
* | |
|
|
* -----------------------------------
|
|
* |
|
|
* v
|
|
* ----------------------------------------------------------------------------
|
|
* | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING |
|
|
* | |
|
|
* | Callbacks processed by rcu_core() from softirqs or local |
|
|
* | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |
|
|
* | Flush pending nocb_timer. Flush nocb bypass callbacks. |
|
|
* ----------------------------------------------------------------------------
|
|
* |
|
|
* v
|
|
* ----------------------------------------------------------------------------
|
|
* | SEGCBLIST_RCU_CORE |
|
|
* | |
|
|
* | Callbacks processed by rcu_core() from softirqs or local |
|
|
* | rcuc kthread, without holding nocb_lock. |
|
|
* ----------------------------------------------------------------------------
|
|
*/
|
|
#define SEGCBLIST_ENABLED BIT(0)
|
|
#define SEGCBLIST_RCU_CORE BIT(1)
|
|
#define SEGCBLIST_LOCKING BIT(2)
|
|
#define SEGCBLIST_KTHREAD_CB BIT(3)
|
|
#define SEGCBLIST_KTHREAD_GP BIT(4)
|
|
#define SEGCBLIST_OFFLOADED BIT(5)
|
|
|
|
struct rcu_segcblist {
|
|
struct rcu_head *head;
|
|
struct rcu_head **tails[RCU_CBLIST_NSEGS];
|
|
unsigned long gp_seq[RCU_CBLIST_NSEGS];
|
|
#ifdef CONFIG_RCU_NOCB_CPU
|
|
atomic_long_t len;
|
|
#else
|
|
long len;
|
|
#endif
|
|
long seglen[RCU_CBLIST_NSEGS];
|
|
u8 flags;
|
|
};
|
|
|
|
#define RCU_SEGCBLIST_INITIALIZER(n) \
|
|
{ \
|
|
.head = NULL, \
|
|
.tails[RCU_DONE_TAIL] = &n.head, \
|
|
.tails[RCU_WAIT_TAIL] = &n.head, \
|
|
.tails[RCU_NEXT_READY_TAIL] = &n.head, \
|
|
.tails[RCU_NEXT_TAIL] = &n.head, \
|
|
}
|
|
|
|
#endif /* __INCLUDE_LINUX_RCU_SEGCBLIST_H */
|