mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-03 07:38:10 +00:00
rcu: Convert rcu_grace_period tracepoint to gp_seq
This commit makes the rcu_grace_period tracepoint use gp_seq instead of ->gpnum or ->completed. It also introduces a "cpuofl-bgp" string to less obscurely indicate when a CPU has gone offline while a grace period is waiting on it. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
parent
ab5e869c1f
commit
477351f782
3 changed files with 29 additions and 29 deletions
|
@ -52,6 +52,7 @@ TRACE_EVENT(rcu_utilization,
|
||||||
* "cpuqs": CPU passes through a quiescent state.
|
* "cpuqs": CPU passes through a quiescent state.
|
||||||
* "cpuonl": CPU comes online.
|
* "cpuonl": CPU comes online.
|
||||||
* "cpuofl": CPU goes offline.
|
* "cpuofl": CPU goes offline.
|
||||||
|
* "cpuofl-bgp": CPU goes offline while blocking a grace period.
|
||||||
* "reqwait": GP kthread sleeps waiting for grace-period request.
|
* "reqwait": GP kthread sleeps waiting for grace-period request.
|
||||||
* "reqwaitsig": GP kthread awakened by signal from reqwait state.
|
* "reqwaitsig": GP kthread awakened by signal from reqwait state.
|
||||||
* "fqswait": GP kthread waiting until time to force quiescent states.
|
* "fqswait": GP kthread waiting until time to force quiescent states.
|
||||||
|
@ -63,24 +64,24 @@ TRACE_EVENT(rcu_utilization,
|
||||||
*/
|
*/
|
||||||
TRACE_EVENT(rcu_grace_period,
|
TRACE_EVENT(rcu_grace_period,
|
||||||
|
|
||||||
TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent),
|
TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent),
|
||||||
|
|
||||||
TP_ARGS(rcuname, gpnum, gpevent),
|
TP_ARGS(rcuname, gp_seq, gpevent),
|
||||||
|
|
||||||
TP_STRUCT__entry(
|
TP_STRUCT__entry(
|
||||||
__field(const char *, rcuname)
|
__field(const char *, rcuname)
|
||||||
__field(unsigned long, gpnum)
|
__field(unsigned long, gp_seq)
|
||||||
__field(const char *, gpevent)
|
__field(const char *, gpevent)
|
||||||
),
|
),
|
||||||
|
|
||||||
TP_fast_assign(
|
TP_fast_assign(
|
||||||
__entry->rcuname = rcuname;
|
__entry->rcuname = rcuname;
|
||||||
__entry->gpnum = gpnum;
|
__entry->gp_seq = gp_seq;
|
||||||
__entry->gpevent = gpevent;
|
__entry->gpevent = gpevent;
|
||||||
),
|
),
|
||||||
|
|
||||||
TP_printk("%s %lu %s",
|
TP_printk("%s %lu %s",
|
||||||
__entry->rcuname, __entry->gpnum, __entry->gpevent)
|
__entry->rcuname, __entry->gp_seq, __entry->gpevent)
|
||||||
);
|
);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -753,7 +754,7 @@ TRACE_EVENT(rcu_barrier,
|
||||||
|
|
||||||
#else /* #ifdef CONFIG_RCU_TRACE */
|
#else /* #ifdef CONFIG_RCU_TRACE */
|
||||||
|
|
||||||
#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0)
|
#define trace_rcu_grace_period(rcuname, gp_seq, gpevent) do { } while (0)
|
||||||
#define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \
|
#define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \
|
||||||
level, grplo, grphi, event) \
|
level, grplo, grphi, event) \
|
||||||
do { } while (0)
|
do { } while (0)
|
||||||
|
|
|
@ -234,7 +234,7 @@ void rcu_sched_qs(void)
|
||||||
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
|
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.s))
|
||||||
return;
|
return;
|
||||||
trace_rcu_grace_period(TPS("rcu_sched"),
|
trace_rcu_grace_period(TPS("rcu_sched"),
|
||||||
__this_cpu_read(rcu_sched_data.gpnum),
|
__this_cpu_read(rcu_sched_data.gp_seq),
|
||||||
TPS("cpuqs"));
|
TPS("cpuqs"));
|
||||||
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
|
__this_cpu_write(rcu_sched_data.cpu_no_qs.b.norm, false);
|
||||||
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
|
if (!__this_cpu_read(rcu_sched_data.cpu_no_qs.b.exp))
|
||||||
|
@ -249,7 +249,7 @@ void rcu_bh_qs(void)
|
||||||
RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
|
RCU_LOCKDEP_WARN(preemptible(), "rcu_bh_qs() invoked with preemption enabled!!!");
|
||||||
if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
|
if (__this_cpu_read(rcu_bh_data.cpu_no_qs.s)) {
|
||||||
trace_rcu_grace_period(TPS("rcu_bh"),
|
trace_rcu_grace_period(TPS("rcu_bh"),
|
||||||
__this_cpu_read(rcu_bh_data.gpnum),
|
__this_cpu_read(rcu_bh_data.gp_seq),
|
||||||
TPS("cpuqs"));
|
TPS("cpuqs"));
|
||||||
__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
|
__this_cpu_write(rcu_bh_data.cpu_no_qs.b.norm, false);
|
||||||
}
|
}
|
||||||
|
@ -1615,7 +1615,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
|
||||||
trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread"));
|
trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread"));
|
||||||
goto unlock_out;
|
goto unlock_out;
|
||||||
}
|
}
|
||||||
trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), TPS("newreq"));
|
trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq), TPS("newreq"));
|
||||||
ret = true; /* Caller must wake GP kthread. */
|
ret = true; /* Caller must wake GP kthread. */
|
||||||
unlock_out:
|
unlock_out:
|
||||||
/* Push furthest requested GP to leaf node and rcu_data structure. */
|
/* Push furthest requested GP to leaf node and rcu_data structure. */
|
||||||
|
@ -1702,9 +1702,9 @@ static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp,
|
||||||
|
|
||||||
/* Trace depending on how much we were able to accelerate. */
|
/* Trace depending on how much we were able to accelerate. */
|
||||||
if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
|
if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
|
||||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccWaitCB"));
|
trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccWaitCB"));
|
||||||
else
|
else
|
||||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("AccReadyCB"));
|
trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("AccReadyCB"));
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1774,7 +1774,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
|
||||||
* go looking for one.
|
* go looking for one.
|
||||||
*/
|
*/
|
||||||
rdp->gpnum = rnp->gpnum;
|
rdp->gpnum = rnp->gpnum;
|
||||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
|
trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpustart"));
|
||||||
need_gp = !!(rnp->qsmask & rdp->grpmask);
|
need_gp = !!(rnp->qsmask & rdp->grpmask);
|
||||||
rdp->cpu_no_qs.b.norm = need_gp;
|
rdp->cpu_no_qs.b.norm = need_gp;
|
||||||
rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
|
rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_dynticks.rcu_qs_ctr);
|
||||||
|
@ -1851,7 +1851,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
|
||||||
rcu_seq_start(&rsp->gp_seq);
|
rcu_seq_start(&rsp->gp_seq);
|
||||||
if (WARN_ON_ONCE(((rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT) != rcu_seq_ctr(rnp->gp_seq))) /* Catch any ->completed/->gp_seq mismatches. */
|
if (WARN_ON_ONCE(((rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT) != rcu_seq_ctr(rnp->gp_seq))) /* Catch any ->completed/->gp_seq mismatches. */
|
||||||
pr_info("%s ->completed: %#lx (%#lx) ->gp_seq %#lx (%#lx)\n", __func__, rnp->completed, (rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT, rnp->gp_seq, rcu_seq_ctr(rnp->gp_seq));
|
pr_info("%s ->completed: %#lx (%#lx) ->gp_seq %#lx (%#lx)\n", __func__, rnp->completed, (rnp->completed << RCU_SEQ_CTR_SHIFT) >> RCU_SEQ_CTR_SHIFT, rnp->gp_seq, rcu_seq_ctr(rnp->gp_seq));
|
||||||
trace_rcu_grace_period(rsp->name, rsp->gpnum, TPS("start"));
|
trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("start"));
|
||||||
raw_spin_unlock_irq_rcu_node(rnp);
|
raw_spin_unlock_irq_rcu_node(rnp);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1928,7 +1928,7 @@ static bool rcu_gp_init(struct rcu_state *rsp)
|
||||||
if (rnp == rdp->mynode)
|
if (rnp == rdp->mynode)
|
||||||
(void)__note_gp_changes(rsp, rnp, rdp);
|
(void)__note_gp_changes(rsp, rnp, rdp);
|
||||||
rcu_preempt_boost_start_gp(rnp);
|
rcu_preempt_boost_start_gp(rnp);
|
||||||
trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
|
trace_rcu_grace_period_init(rsp->name, rnp->gp_seq,
|
||||||
rnp->level, rnp->grplo,
|
rnp->level, rnp->grplo,
|
||||||
rnp->grphi, rnp->qsmask);
|
rnp->grphi, rnp->qsmask);
|
||||||
raw_spin_unlock_irq_rcu_node(rnp);
|
raw_spin_unlock_irq_rcu_node(rnp);
|
||||||
|
@ -2048,7 +2048,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
/* Declare grace period done. */
|
/* Declare grace period done. */
|
||||||
WRITE_ONCE(rsp->completed, rsp->gpnum);
|
WRITE_ONCE(rsp->completed, rsp->gpnum);
|
||||||
rcu_seq_end(&rsp->gp_seq);
|
rcu_seq_end(&rsp->gp_seq);
|
||||||
trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end"));
|
trace_rcu_grace_period(rsp->name, rsp->gp_seq, TPS("end"));
|
||||||
rsp->gp_state = RCU_GP_IDLE;
|
rsp->gp_state = RCU_GP_IDLE;
|
||||||
/* Check for GP requests since above loop. */
|
/* Check for GP requests since above loop. */
|
||||||
rdp = this_cpu_ptr(rsp->rda);
|
rdp = this_cpu_ptr(rsp->rda);
|
||||||
|
@ -2061,7 +2061,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
|
||||||
if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
|
if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) {
|
||||||
WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
|
WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT);
|
||||||
rsp->gp_req_activity = jiffies;
|
rsp->gp_req_activity = jiffies;
|
||||||
trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum),
|
trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gp_seq),
|
||||||
TPS("newreq"));
|
TPS("newreq"));
|
||||||
} else {
|
} else {
|
||||||
WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
|
WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT);
|
||||||
|
@ -2087,7 +2087,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
/* Handle grace-period start. */
|
/* Handle grace-period start. */
|
||||||
for (;;) {
|
for (;;) {
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("reqwait"));
|
TPS("reqwait"));
|
||||||
rsp->gp_state = RCU_GP_WAIT_GPS;
|
rsp->gp_state = RCU_GP_WAIT_GPS;
|
||||||
swait_event_idle(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
|
swait_event_idle(rsp->gp_wq, READ_ONCE(rsp->gp_flags) &
|
||||||
|
@ -2100,7 +2100,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
WRITE_ONCE(rsp->gp_activity, jiffies);
|
WRITE_ONCE(rsp->gp_activity, jiffies);
|
||||||
WARN_ON(signal_pending(current));
|
WARN_ON(signal_pending(current));
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("reqwaitsig"));
|
TPS("reqwaitsig"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -2119,7 +2119,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
jiffies + 3 * j);
|
jiffies + 3 * j);
|
||||||
}
|
}
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("fqswait"));
|
TPS("fqswait"));
|
||||||
rsp->gp_state = RCU_GP_WAIT_FQS;
|
rsp->gp_state = RCU_GP_WAIT_FQS;
|
||||||
ret = swait_event_idle_timeout(rsp->gp_wq,
|
ret = swait_event_idle_timeout(rsp->gp_wq,
|
||||||
|
@ -2134,12 +2134,12 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
|
if (ULONG_CMP_GE(jiffies, rsp->jiffies_force_qs) ||
|
||||||
(gf & RCU_GP_FLAG_FQS)) {
|
(gf & RCU_GP_FLAG_FQS)) {
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("fqsstart"));
|
TPS("fqsstart"));
|
||||||
rcu_gp_fqs(rsp, first_gp_fqs);
|
rcu_gp_fqs(rsp, first_gp_fqs);
|
||||||
first_gp_fqs = false;
|
first_gp_fqs = false;
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("fqsend"));
|
TPS("fqsend"));
|
||||||
cond_resched_tasks_rcu_qs();
|
cond_resched_tasks_rcu_qs();
|
||||||
WRITE_ONCE(rsp->gp_activity, jiffies);
|
WRITE_ONCE(rsp->gp_activity, jiffies);
|
||||||
|
@ -2158,7 +2158,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
|
||||||
WRITE_ONCE(rsp->gp_activity, jiffies);
|
WRITE_ONCE(rsp->gp_activity, jiffies);
|
||||||
WARN_ON(signal_pending(current));
|
WARN_ON(signal_pending(current));
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name,
|
||||||
READ_ONCE(rsp->gpnum),
|
READ_ONCE(rsp->gp_seq),
|
||||||
TPS("fqswaitsig"));
|
TPS("fqswaitsig"));
|
||||||
ret = 1; /* Keep old FQS timing. */
|
ret = 1; /* Keep old FQS timing. */
|
||||||
j = jiffies;
|
j = jiffies;
|
||||||
|
@ -2388,17 +2388,16 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
|
||||||
*/
|
*/
|
||||||
static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
|
static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
|
||||||
{
|
{
|
||||||
RCU_TRACE(unsigned long mask;)
|
RCU_TRACE(bool blkd;)
|
||||||
RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
|
RCU_TRACE(struct rcu_data *rdp = this_cpu_ptr(rsp->rda);)
|
||||||
RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
|
RCU_TRACE(struct rcu_node *rnp = rdp->mynode;)
|
||||||
|
|
||||||
if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
|
if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
RCU_TRACE(mask = rdp->grpmask;)
|
RCU_TRACE(blkd = !!(rnp->qsmask & rdp->grpmask);)
|
||||||
trace_rcu_grace_period(rsp->name,
|
trace_rcu_grace_period(rsp->name, rnp->gp_seq,
|
||||||
rnp->gpnum + 1 - !!(rnp->qsmask & mask),
|
blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
|
||||||
TPS("cpuofl"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -3538,7 +3537,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
|
||||||
rdp->core_needs_qs = false;
|
rdp->core_needs_qs = false;
|
||||||
rdp->rcu_iw_pending = false;
|
rdp->rcu_iw_pending = false;
|
||||||
rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
|
rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
|
||||||
trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
|
trace_rcu_grace_period(rsp->name, rdp->gp_seq, TPS("cpuonl"));
|
||||||
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
|
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -301,7 +301,7 @@ static void rcu_preempt_qs(void)
|
||||||
RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_qs() invoked with preemption enabled!!!\n");
|
RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_qs() invoked with preemption enabled!!!\n");
|
||||||
if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
|
if (__this_cpu_read(rcu_data_p->cpu_no_qs.s)) {
|
||||||
trace_rcu_grace_period(TPS("rcu_preempt"),
|
trace_rcu_grace_period(TPS("rcu_preempt"),
|
||||||
__this_cpu_read(rcu_data_p->gpnum),
|
__this_cpu_read(rcu_data_p->gp_seq),
|
||||||
TPS("cpuqs"));
|
TPS("cpuqs"));
|
||||||
__this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
|
__this_cpu_write(rcu_data_p->cpu_no_qs.b.norm, false);
|
||||||
barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
|
barrier(); /* Coordinate with rcu_preempt_check_callbacks(). */
|
||||||
|
|
Loading…
Reference in a new issue