mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

rcu: More aggressively enlist scheduler aid for nohz_full CPUs 

Because nohz_full CPUs can leave the scheduler-clock interrupt disabled
even when in kernel mode, RCU cannot rely on rcu_check_callbacks() to
enlist the scheduler's aid in extracting a quiescent state from such CPUs.
This commit therefore more aggressively uses resched_cpu() on nohz_full
CPUs that fail to pass through a quiescent state in a timely manner.
By default, the resched_cpu() beating starts 300 milliseconds into the
quiescent state.

While in the neighborhood, add a ->last_fqs_resched field to the rcu_data
structure in order to rate-limit resched_cpu() calls from the RCU
grace-period kthread.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney 2018-07-25 11:49:47 -07:00
parent c06aed0e31
commit d3052109c0
3 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
kernel/rcu/tree.c
View File

@ -61,6 +61,7 @@
#include <linux/trace_events.h>
#include <linux/suspend.h>
#include <linux/ftrace.h>
#include <linux/tick.h>
#include "tree.h"
#include "rcu.h"
@ -1088,19 +1089,38 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
WRITE_ONCE(*rnhqp, true);
/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
smp_store_release(ruqp, true);
rcu_state.jiffies_resched += jtsq; /* Re-enable beating. */
} else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
WRITE_ONCE(*ruqp, true);
}
/*
* If more than halfway to RCU CPU stall-warning time, do a
* resched_cpu() to try to loosen things up a bit. Also check to
* see if the CPU is getting hammered with interrupts, but only
* once per grace period, just to keep the IPIs down to a dull roar.
* NO_HZ_FULL CPUs can run in-kernel without rcu_check_callbacks!
* The above code handles this, but only for straight cond_resched().
* And some in-kernel loops check need_resched() before calling
* cond_resched(), which defeats the above code for CPUs that are
* running in-kernel with scheduling-clock interrupts disabled.
* So hit them over the head with the resched_cpu() hammer!
*/
if (tick_nohz_full_cpu(rdp->cpu) &&
time_after(jiffies,
READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {
resched_cpu(rdp->cpu);
WRITE_ONCE(rdp->last_fqs_resched, jiffies);
}
/*
* If more than halfway to RCU CPU stall-warning time, invoke
* resched_cpu() more frequently to try to loosen things up a bit.
* Also check to see if the CPU is getting hammered with interrupts,
* but only once per grace period, just to keep the IPIs down to
* a dull roar.
*/
if (time_after(jiffies, rcu_state.jiffies_resched)) {
resched_cpu(rdp->cpu);
if (time_after(jiffies,
READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
resched_cpu(rdp->cpu);
WRITE_ONCE(rdp->last_fqs_resched, jiffies);
}
if (IS_ENABLED(CONFIG_IRQ_WORK) &&
!rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
(rnp->ffmask & rdp->grpmask)) {

1
kernel/rcu/tree.h
View File

@ -260,6 +260,7 @@ struct rcu_data {
short rcu_ofl_gp_flags; /* ->gp_flags at last offline. */
unsigned long rcu_onl_gp_seq; /* ->gp_seq at last online. */
short rcu_onl_gp_flags; /* ->gp_flags at last online. */
unsigned long last_fqs_resched; /* Time of last rcu_resched(). */
int cpu;
};

1
kernel/rcu/tree_plugin.h
View File

@ -1850,6 +1850,7 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp)
{
rdp->ticks_this_gp = 0;
rdp->softirq_snap = kstat_softirqs_cpu(RCU_SOFTIRQ, smp_processor_id());
WRITE_ONCE(rdp->last_fqs_resched, jiffies);
}
#ifdef CONFIG_RCU_NOCB_CPU