mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 11:49:32 +00:00
rcu: Accelerate RCU callbacks at grace-period end
Now that callback acceleration is idempotent, it is safe to accelerate callbacks during grace-period cleanup on any CPUs that the kthread happens to be running on. This commit therefore propagates the completion of the grace period to the per-CPU data structures, and also adds an rcu_advance_cbs() just before the cpu_needs_another_gp() check in order to reduce false-positive grace periods. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
parent
5e44ce35a6
commit
b11cc5760a
1 changed files with 13 additions and 8 deletions
|
@ -1396,6 +1396,9 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
|||
rcu_for_each_node_breadth_first(rsp, rnp) {
|
||||
raw_spin_lock_irq(&rnp->lock);
|
||||
rnp->completed = rsp->gpnum;
|
||||
rdp = this_cpu_ptr(rsp->rda);
|
||||
if (rnp == rdp->mynode)
|
||||
__rcu_process_gp_end(rsp, rnp, rdp);
|
||||
nocb += rcu_nocb_gp_cleanup(rsp, rnp);
|
||||
raw_spin_unlock_irq(&rnp->lock);
|
||||
cond_resched();
|
||||
|
@ -1408,6 +1411,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
|||
trace_rcu_grace_period(rsp->name, rsp->completed, "end");
|
||||
rsp->fqs_state = RCU_GP_IDLE;
|
||||
rdp = this_cpu_ptr(rsp->rda);
|
||||
rcu_advance_cbs(rsp, rnp, rdp); /* Reduce false positives below. */
|
||||
if (cpu_needs_another_gp(rsp, rdp))
|
||||
rsp->gp_flags = 1;
|
||||
raw_spin_unlock_irq(&rnp->lock);
|
||||
|
@ -1497,6 +1501,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
|
|||
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
|
||||
struct rcu_node *rnp = rcu_get_root(rsp);
|
||||
|
||||
/*
|
||||
* If there is no grace period in progress right now, any
|
||||
* callbacks we have up to this point will be satisfied by the
|
||||
* next grace period. Also, advancing the callbacks reduces the
|
||||
* probability of false positives from cpu_needs_another_gp()
|
||||
* resulting in pointless grace periods. So, advance callbacks!
|
||||
*/
|
||||
rcu_advance_cbs(rsp, rnp, rdp);
|
||||
|
||||
if (!rsp->gp_kthread ||
|
||||
!cpu_needs_another_gp(rsp, rdp)) {
|
||||
/*
|
||||
|
@ -1509,14 +1522,6 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
|
|||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Because there is no grace period in progress right now,
|
||||
* any callbacks we have up to this point will be satisfied
|
||||
* by the next grace period. So this is a good place to
|
||||
* assign a grace period number to recently posted callbacks.
|
||||
*/
|
||||
rcu_accelerate_cbs(rsp, rnp, rdp);
|
||||
|
||||
rsp->gp_flags = RCU_GP_FLAG_INIT;
|
||||
raw_spin_unlock(&rnp->lock); /* Interrupts remain disabled. */
|
||||
|
||||
|
|
Loading…
Reference in a new issue