mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-13 22:25:03 +00:00
rcu: Apply callbacks processing time limit only on softirq
Time limit only makes sense when callbacks are serviced in softirq mode because: _ In case we need to get back to the scheduler, cond_resched_tasks_rcu_qs() is called after each callback. _ In case some other softirq vector needs the CPU, the call to local_bh_enable() before cond_resched_tasks_rcu_qs() takes care about them via a call to do_softirq(). Therefore, make sure the time limit only applies to softirq mode. Reviewed-by: Valentin Schneider <valentin.schneider@arm.com> Tested-by: Valentin Schneider <valentin.schneider@arm.com> Tested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: Frederic Weisbecker <frederic@kernel.org> Cc: Valentin Schneider <valentin.schneider@arm.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: Josh Triplett <josh@joshtriplett.org> Cc: Joel Fernandes <joel@joelfernandes.org> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Neeraj Upadhyay <neeraju@codeaurora.org> Cc: Uladzislau Rezki <urezki@gmail.com> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
parent
3e61e95e2d
commit
a554ba2888
1 changed files with 12 additions and 13 deletions
|
@ -2488,7 +2488,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||||
div = READ_ONCE(rcu_divisor);
|
div = READ_ONCE(rcu_divisor);
|
||||||
div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
|
div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
|
||||||
bl = max(rdp->blimit, pending >> div);
|
bl = max(rdp->blimit, pending >> div);
|
||||||
if (unlikely(bl > 100)) {
|
if (in_serving_softirq() && unlikely(bl > 100)) {
|
||||||
long rrn = READ_ONCE(rcu_resched_ns);
|
long rrn = READ_ONCE(rcu_resched_ns);
|
||||||
|
|
||||||
rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
|
rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
|
||||||
|
@ -2528,6 +2528,17 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||||
if (in_serving_softirq()) {
|
if (in_serving_softirq()) {
|
||||||
if (count >= bl && (need_resched() || !is_idle_task(current)))
|
if (count >= bl && (need_resched() || !is_idle_task(current)))
|
||||||
break;
|
break;
|
||||||
|
/*
|
||||||
|
* Make sure we don't spend too much time here and deprive other
|
||||||
|
* softirq vectors of CPU cycles.
|
||||||
|
*/
|
||||||
|
if (unlikely(tlimit)) {
|
||||||
|
/* only call local_clock() every 32 callbacks */
|
||||||
|
if (likely((count & 31) || local_clock() < tlimit))
|
||||||
|
continue;
|
||||||
|
/* Exceeded the time limit, so leave. */
|
||||||
|
break;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
local_bh_enable();
|
local_bh_enable();
|
||||||
lockdep_assert_irqs_enabled();
|
lockdep_assert_irqs_enabled();
|
||||||
|
@ -2535,18 +2546,6 @@ static void rcu_do_batch(struct rcu_data *rdp)
|
||||||
lockdep_assert_irqs_enabled();
|
lockdep_assert_irqs_enabled();
|
||||||
local_bh_disable();
|
local_bh_disable();
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Make sure we don't spend too much time here and deprive other
|
|
||||||
* softirq vectors of CPU cycles.
|
|
||||||
*/
|
|
||||||
if (unlikely(tlimit)) {
|
|
||||||
/* only call local_clock() every 32 callbacks */
|
|
||||||
if (likely((count & 31) || local_clock() < tlimit))
|
|
||||||
continue;
|
|
||||||
/* Exceeded the time limit, so leave. */
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rcu_nocb_lock_irqsave(rdp, flags);
|
rcu_nocb_lock_irqsave(rdp, flags);
|
||||||
|
|
Loading…
Reference in a new issue