mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-24 17:59:32 +00:00
rcu: Reduce latency of rcu_prepare_for_idle()
Re-enable interrupts across calls to quiescent-state functions and also across force_quiescent_state() to reduce latency. Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
parent
f535a607c1
commit
3ad0decf98
1 changed files with 7 additions and 5 deletions
|
@ -2009,7 +2009,6 @@ int rcu_needs_cpu(int cpu)
|
||||||
*/
|
*/
|
||||||
static void rcu_prepare_for_idle(int cpu)
|
static void rcu_prepare_for_idle(int cpu)
|
||||||
{
|
{
|
||||||
int c = 0;
|
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
|
|
||||||
local_irq_save(flags);
|
local_irq_save(flags);
|
||||||
|
@ -2055,27 +2054,30 @@ static void rcu_prepare_for_idle(int cpu)
|
||||||
*/
|
*/
|
||||||
#ifdef CONFIG_TREE_PREEMPT_RCU
|
#ifdef CONFIG_TREE_PREEMPT_RCU
|
||||||
if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
|
if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
|
||||||
|
local_irq_restore(flags);
|
||||||
rcu_preempt_qs(cpu);
|
rcu_preempt_qs(cpu);
|
||||||
force_quiescent_state(&rcu_preempt_state, 0);
|
force_quiescent_state(&rcu_preempt_state, 0);
|
||||||
c = c || per_cpu(rcu_preempt_data, cpu).nxtlist;
|
local_irq_save(flags);
|
||||||
}
|
}
|
||||||
#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
|
#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
|
||||||
if (per_cpu(rcu_sched_data, cpu).nxtlist) {
|
if (per_cpu(rcu_sched_data, cpu).nxtlist) {
|
||||||
|
local_irq_restore(flags);
|
||||||
rcu_sched_qs(cpu);
|
rcu_sched_qs(cpu);
|
||||||
force_quiescent_state(&rcu_sched_state, 0);
|
force_quiescent_state(&rcu_sched_state, 0);
|
||||||
c = c || per_cpu(rcu_sched_data, cpu).nxtlist;
|
local_irq_save(flags);
|
||||||
}
|
}
|
||||||
if (per_cpu(rcu_bh_data, cpu).nxtlist) {
|
if (per_cpu(rcu_bh_data, cpu).nxtlist) {
|
||||||
|
local_irq_restore(flags);
|
||||||
rcu_bh_qs(cpu);
|
rcu_bh_qs(cpu);
|
||||||
force_quiescent_state(&rcu_bh_state, 0);
|
force_quiescent_state(&rcu_bh_state, 0);
|
||||||
c = c || per_cpu(rcu_bh_data, cpu).nxtlist;
|
local_irq_save(flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If RCU callbacks are still pending, RCU still needs this CPU.
|
* If RCU callbacks are still pending, RCU still needs this CPU.
|
||||||
* So try forcing the callbacks through the grace period.
|
* So try forcing the callbacks through the grace period.
|
||||||
*/
|
*/
|
||||||
if (c) {
|
if (rcu_cpu_has_callbacks(cpu)) {
|
||||||
local_irq_restore(flags);
|
local_irq_restore(flags);
|
||||||
trace_rcu_prep_idle("More callbacks");
|
trace_rcu_prep_idle("More callbacks");
|
||||||
invoke_rcu_core();
|
invoke_rcu_core();
|
||||||
|
|
Loading…
Reference in a new issue