mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-13 22:25:03 +00:00
rcu: Make rcu_gp_cleanup() more accurately predict need for new GP
Currently, rcu_gp_cleanup() scans the rcu_node tree in order to reset state to reflect the end of the grace period. It also checks to see whether a new grace period is needed, but in a number of cases, rather than directly cause the new grace period to be immediately started, it instead leaves the grace-period-needed state where various fail-safes can find it. This works fine, but results in higher contention on the root rcu_node structure's ->lock, which is undesirable, and contention on that lock has recently become noticeable. This commit therefore makes rcu_gp_cleanup() immediately start a new grace period if there is any need for one. It is quite possible that it will later be necessary to throttle the grace-period rate, but that can be dealt with when and if. Reported-by: Nicholas Piggin <npiggin@gmail.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Nicholas Piggin <npiggin@gmail.com>
This commit is contained in:
parent
5fe0a56298
commit
fb31340f8a
3 changed files with 10 additions and 24 deletions
|
@ -1763,14 +1763,14 @@ rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp,
|
||||||
* Clean up any old requests for the just-ended grace period. Also return
|
* Clean up any old requests for the just-ended grace period. Also return
|
||||||
* whether any additional grace periods have been requested.
|
* whether any additional grace periods have been requested.
|
||||||
*/
|
*/
|
||||||
static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
|
static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp)
|
||||||
{
|
{
|
||||||
int c = rnp->completed;
|
int c = rnp->completed;
|
||||||
int needmore;
|
bool needmore;
|
||||||
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
|
struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
|
||||||
|
|
||||||
need_future_gp_element(rnp, c) = 0;
|
need_future_gp_element(rnp, c) = 0;
|
||||||
needmore = need_future_gp_element(rnp, c + 1);
|
needmore = need_any_future_gp(rnp);
|
||||||
trace_rcu_future_gp(rnp, rdp, c,
|
trace_rcu_future_gp(rnp, rdp, c,
|
||||||
needmore ? TPS("CleanupMore") : TPS("Cleanup"));
|
needmore ? TPS("CleanupMore") : TPS("Cleanup"));
|
||||||
return needmore;
|
return needmore;
|
||||||
|
@ -2113,7 +2113,6 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
{
|
{
|
||||||
unsigned long gp_duration;
|
unsigned long gp_duration;
|
||||||
bool needgp = false;
|
bool needgp = false;
|
||||||
int nocb = 0;
|
|
||||||
struct rcu_data *rdp;
|
struct rcu_data *rdp;
|
||||||
struct rcu_node *rnp = rcu_get_root(rsp);
|
struct rcu_node *rnp = rcu_get_root(rsp);
|
||||||
struct swait_queue_head *sq;
|
struct swait_queue_head *sq;
|
||||||
|
@ -2152,7 +2151,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
if (rnp == rdp->mynode)
|
if (rnp == rdp->mynode)
|
||||||
needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
|
needgp = __note_gp_changes(rsp, rnp, rdp) || needgp;
|
||||||
/* smp_mb() provided by prior unlock-lock pair. */
|
/* smp_mb() provided by prior unlock-lock pair. */
|
||||||
nocb += rcu_future_gp_cleanup(rsp, rnp);
|
needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp;
|
||||||
sq = rcu_nocb_gp_get(rnp);
|
sq = rcu_nocb_gp_get(rnp);
|
||||||
raw_spin_unlock_irq_rcu_node(rnp);
|
raw_spin_unlock_irq_rcu_node(rnp);
|
||||||
rcu_nocb_gp_cleanup(sq);
|
rcu_nocb_gp_cleanup(sq);
|
||||||
|
@ -2162,13 +2161,18 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
}
|
}
|
||||||
rnp = rcu_get_root(rsp);
|
rnp = rcu_get_root(rsp);
|
||||||
raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
|
raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */
|
||||||
rcu_nocb_gp_set(rnp, nocb);
|
|
||||||
|
|
||||||
/* Declare grace period done. */
|
/* Declare grace period done. */
|
||||||
WRITE_ONCE(rsp->completed, rsp->gpnum);
|
WRITE_ONCE(rsp->completed, rsp->gpnum);
|
||||||
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
|
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
|
||||||
rsp->gp_state = RCU_GP_IDLE;
|
rsp->gp_state = RCU_GP_IDLE;
|
||||||
|
/* Check for GP requests since above loop. */
|
||||||
rdp = this_cpu_ptr(rsp->rda);
|
rdp = this_cpu_ptr(rsp->rda);
|
||||||
|
if (need_any_future_gp(rnp)) {
|
||||||
|
trace_rcu_future_gp(rnp, rdp, rsp->completed - 1,
|
||||||
|
TPS("CleanupMore"));
|
||||||
|
needgp = true;
|
||||||
|
}
|
||||||
/* Advance CBs to reduce false positives below. */
|
/* Advance CBs to reduce false positives below. */
|
||||||
needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
|
needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp;
|
||||||
if (needgp || cpu_needs_another_gp(rsp, rdp)) {
|
if (needgp || cpu_needs_another_gp(rsp, rdp)) {
|
||||||
|
|
|
@ -469,7 +469,6 @@ static void print_cpu_stall_info_end(void);
|
||||||
static void zero_cpu_stall_ticks(struct rcu_data *rdp);
|
static void zero_cpu_stall_ticks(struct rcu_data *rdp);
|
||||||
static void increment_cpu_stall_ticks(void);
|
static void increment_cpu_stall_ticks(void);
|
||||||
static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu);
|
static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu);
|
||||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq);
|
|
||||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
|
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp);
|
||||||
static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
|
static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq);
|
||||||
static void rcu_init_one_nocb(struct rcu_node *rnp);
|
static void rcu_init_one_nocb(struct rcu_node *rnp);
|
||||||
|
|
|
@ -1780,19 +1780,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
|
||||||
swake_up_all(sq);
|
swake_up_all(sq);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Set the root rcu_node structure's ->need_future_gp field
|
|
||||||
* based on the sum of those of all rcu_node structures. This does
|
|
||||||
* double-count the root rcu_node structure's requests, but this
|
|
||||||
* is necessary to handle the possibility of a rcu_nocb_kthread()
|
|
||||||
* having awakened during the time that the rcu_node structures
|
|
||||||
* were being updated for the end of the previous grace period.
|
|
||||||
*/
|
|
||||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
|
|
||||||
{
|
|
||||||
need_future_gp_element(rnp, rnp->completed + 1) += nrq;
|
|
||||||
}
|
|
||||||
|
|
||||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
||||||
{
|
{
|
||||||
return &rnp->nocb_gp_wq[rnp->completed & 0x1];
|
return &rnp->nocb_gp_wq[rnp->completed & 0x1];
|
||||||
|
@ -2495,10 +2482,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
|
||||||
{
|
{
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
Loading…
Reference in a new issue