mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 11:49:32 +00:00
rcu: Make rcu_gp_cleanup() more accurately predict need for new GP
Currently, rcu_gp_cleanup() scans the rcu_node tree in order to reset state to reflect the end of the grace period. It also checks to see whether a new grace period is needed, but in a number of cases, rather than directly cause the new grace period to be immediately started, it instead leaves the grace-period-needed state where various fail-safes can find it. This works fine, but results in higher contention on the root rcu_node structure's ->lock, which is undesirable, and contention on that lock has recently become noticeable. This commit therefore makes rcu_gp_cleanup() immediately start a new grace period if there is any need for one. It is quite possible that it will later be necessary to throttle the grace-period rate, but that can be dealt with when and if. Reported-by: Nicholas Piggin <npiggin@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Nicholas Piggin <npiggin@gmail.com>
This commit is contained in:
parent
5fe0a56298
commit
fb31340f8a
3 changed files with 10 additions and 24 deletions
|
@ -1763,14 +1763,14 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
|
|||
* Clean up any old requests for the just-ended grace period. Also return
|
||||
* whether any additional grace periods have been requested.
|
||||
*/
|
||||
static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
|
||||
static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
|
||||
{
|
||||
int c = rnp->completed;
|
||||
int needmore;
|
||||
bool needmore;
|
||||
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
|
||||
|
||||
need_future_gp_element(rnp, c) = 0;
|
||||
needmore = need_future_gp_element(rnp, c + 1);
|
||||
needmore = need_any_future_gp(rnp);
|
||||
trace_rcu_future_gp(rnp, rdp, c,
|
||||
needmore ? TPS("CleanupMore") : TPS("Cleanup"));
|
||||
return needmore;
|
||||
|
@ -2113,7 +2113,6 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
|||
{
|
||||
unsigned long gp_duration;
|
||||
bool needgp = false;
|
||||
int nocb = 0;
|
||||
struct rcu_data *rdp;
|
||||
struct rcu_node *rnp = rcu_get_root(rsp);
|
||||
struct swait_queue_head *sq;
|
||||
|
@ -2152,7 +2151,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
|||
if (rnp == rdp->mynode)
|
||||
needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
|
||||
/* smp_mb() provided by prior unlock-lock pair. */
|
||||
nocb += rcu_future_gp_cleanup(rsp, rnp);
|
||||
needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp;
|
||||
sq = rcu_nocb_gp_get(rnp);
|
||||
raw_spin_unlock_irq_rcu_node(rnp);
|
||||
rcu_nocb_gp_cleanup(sq);
|
||||
|
@ -2162,13 +2161,18 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
|||
}
|
||||
rnp = rcu_get_root(rsp);
|
||||
raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
|
||||
rcu_nocb_gp_set(rnp, nocb);
|
||||
|
||||
/* Declare grace period done. */
|
||||
WRITE_ONCE(rsp->completed, rsp->gpnum);
|
||||
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
|
||||
rsp->gp_state = RCU_GP_IDLE;
|
||||
/* Check for GP requests since above loop. */
|
||||
rdp = this_cpu_ptr(rsp->rda);
|
||||
if (need_any_future_gp(rnp)) {
|
||||
trace_rcu_future_gp(rnp, rdp, rsp->completed - 1,
|
||||
TPS("CleanupMore"));
|
||||
needgp = true;
|
||||
}
|
||||
/* Advance CBs to reduce false positives below. */
|
||||
needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
|
||||
if (needgp || cpu_needs_another_gp(rsp, rdp)) {
|
||||
|
|
|
@ -469,7 +469,6 @@ static void print_cpu_stall_info_end(void);
|
|||
static void zero_cpu_stall_ticks(struct rcu_data *rdp);
|
||||
static void increment_cpu_stall_ticks(void);
|
||||
static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu);
|
||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq);
|
||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
|
||||
static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
|
||||
static void rcu_init_one_nocb(struct rcu_node *rnp);
|
||||
|
|
|
@ -1780,19 +1780,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
|
|||
swake_up_all(sq);
|
||||
}
|
||||
|
||||
/*
|
||||
* Set the root rcu_node structure's ->need_future_gp field
|
||||
* based on the sum of those of all rcu_node structures. This does
|
||||
* double-count the root rcu_node structure's requests, but this
|
||||
* is necessary to handle the possibility of a rcu_nocb_kthread()
|
||||
* having awakened during the time that the rcu_node structures
|
||||
* were being updated for the end of the previous grace period.
|
||||
*/
|
||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
|
||||
{
|
||||
need_future_gp_element(rnp, rnp->completed + 1) += nrq;
|
||||
}
|
||||
|
||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
||||
{
|
||||
return &rnp->nocb_gp_wq[rnp->completed & 0x1];
|
||||
|
@ -2495,10 +2482,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
|
|||
{
|
||||
}
|
||||
|
||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
|
||||
{
|
||||
}
|
||||
|
||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
||||
{
|
||||
return NULL;
|
||||
|
|
Loading…
Reference in a new issue