mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-28 07:13:34 +00:00
Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull RCU fix from Ingo Molnar: "A boot hang fix for the offloaded callback RCU model (RCU_NOCB_CPU=y && (TREE_CPU=y || TREE_PREEMPT_RC)) in certain bootup scenarios" * 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: rcu: Make nocb leader kthreads process pending callbacks after spawning
This commit is contained in:
commit
81368f8bb8
2 changed files with 12 additions and 12 deletions
|
@ -358,7 +358,7 @@ struct rcu_data {
|
||||||
struct rcu_head **nocb_gp_tail;
|
struct rcu_head **nocb_gp_tail;
|
||||||
long nocb_gp_count;
|
long nocb_gp_count;
|
||||||
long nocb_gp_count_lazy;
|
long nocb_gp_count_lazy;
|
||||||
bool nocb_leader_wake; /* Is the nocb leader thread awake? */
|
bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
|
||||||
struct rcu_data *nocb_next_follower;
|
struct rcu_data *nocb_next_follower;
|
||||||
/* Next follower in wakeup chain. */
|
/* Next follower in wakeup chain. */
|
||||||
|
|
||||||
|
|
|
@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
|
||||||
|
|
||||||
if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
|
if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
|
||||||
return;
|
return;
|
||||||
if (!ACCESS_ONCE(rdp_leader->nocb_leader_wake) || force) {
|
if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
|
||||||
/* Prior xchg orders against prior callback enqueue. */
|
/* Prior xchg orders against prior callback enqueue. */
|
||||||
ACCESS_ONCE(rdp_leader->nocb_leader_wake) = true;
|
ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
|
||||||
wake_up(&rdp_leader->nocb_wq);
|
wake_up(&rdp_leader->nocb_wq);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -2253,7 +2253,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
|
||||||
if (!rcu_nocb_poll) {
|
if (!rcu_nocb_poll) {
|
||||||
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
|
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
|
||||||
wait_event_interruptible(my_rdp->nocb_wq,
|
wait_event_interruptible(my_rdp->nocb_wq,
|
||||||
ACCESS_ONCE(my_rdp->nocb_leader_wake));
|
!ACCESS_ONCE(my_rdp->nocb_leader_sleep));
|
||||||
/* Memory barrier handled by smp_mb() calls below and repoll. */
|
/* Memory barrier handled by smp_mb() calls below and repoll. */
|
||||||
} else if (firsttime) {
|
} else if (firsttime) {
|
||||||
firsttime = false; /* Don't drown trace log with "Poll"! */
|
firsttime = false; /* Don't drown trace log with "Poll"! */
|
||||||
|
@ -2292,12 +2292,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
|
||||||
schedule_timeout_interruptible(1);
|
schedule_timeout_interruptible(1);
|
||||||
|
|
||||||
/* Rescan in case we were a victim of memory ordering. */
|
/* Rescan in case we were a victim of memory ordering. */
|
||||||
my_rdp->nocb_leader_wake = false;
|
my_rdp->nocb_leader_sleep = true;
|
||||||
smp_mb(); /* Ensure _wake false before scan. */
|
smp_mb(); /* Ensure _sleep true before scan. */
|
||||||
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
|
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
|
||||||
if (ACCESS_ONCE(rdp->nocb_head)) {
|
if (ACCESS_ONCE(rdp->nocb_head)) {
|
||||||
/* Found CB, so short-circuit next wait. */
|
/* Found CB, so short-circuit next wait. */
|
||||||
my_rdp->nocb_leader_wake = true;
|
my_rdp->nocb_leader_sleep = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
goto wait_again;
|
goto wait_again;
|
||||||
|
@ -2307,17 +2307,17 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
|
||||||
rcu_nocb_wait_gp(my_rdp);
|
rcu_nocb_wait_gp(my_rdp);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We left ->nocb_leader_wake set to reduce cache thrashing.
|
* We left ->nocb_leader_sleep unset to reduce cache thrashing.
|
||||||
* We clear it now, but recheck for new callbacks while
|
* We set it now, but recheck for new callbacks while
|
||||||
* traversing our follower list.
|
* traversing our follower list.
|
||||||
*/
|
*/
|
||||||
my_rdp->nocb_leader_wake = false;
|
my_rdp->nocb_leader_sleep = true;
|
||||||
smp_mb(); /* Ensure _wake false before scan of ->nocb_head. */
|
smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
|
||||||
|
|
||||||
/* Each pass through the following loop wakes a follower, if needed. */
|
/* Each pass through the following loop wakes a follower, if needed. */
|
||||||
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
|
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
|
||||||
if (ACCESS_ONCE(rdp->nocb_head))
|
if (ACCESS_ONCE(rdp->nocb_head))
|
||||||
my_rdp->nocb_leader_wake = true; /* No need to wait. */
|
my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
|
||||||
if (!rdp->nocb_gp_head)
|
if (!rdp->nocb_gp_head)
|
||||||
continue; /* No CBs, so no need to wake follower. */
|
continue; /* No CBs, so no need to wake follower. */
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue