mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-29 22:02:02 +00:00
srcu: Move heuristics fields from srcu_struct to srcu_usage
This commit moves the ->srcu_size_jiffies, ->srcu_n_lock_retries, and ->srcu_n_exp_nodelay fields from the srcu_struct structure to the srcu_usage structure to reduce the size of the former in order to improve cache locality. Suggested-by: Christoph Hellwig <hch@lst.de> Tested-by: Sachin Sant <sachinp@linux.ibm.com> Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com> Tested-by: Joel Fernandes (Google) <joel@joelfernandes.org> Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
parent
03200b5ca3
commit
3b46679c62
2 changed files with 12 additions and 12 deletions
|
@ -73,6 +73,9 @@ struct srcu_usage {
|
|||
unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */
|
||||
unsigned long srcu_gp_start; /* Last GP start timestamp (jiffies) */
|
||||
unsigned long srcu_last_gp_end; /* Last GP end timestamp (ns) */
|
||||
unsigned long srcu_size_jiffies; /* Current contention-measurement interval. */
|
||||
unsigned long srcu_n_lock_retries; /* Contention events in current interval. */
|
||||
unsigned long srcu_n_exp_nodelay; /* # expedited no-delays in current GP phase. */
|
||||
};
|
||||
|
||||
/*
|
||||
|
@ -80,9 +83,6 @@ struct srcu_usage {
|
|||
*/
|
||||
struct srcu_struct {
|
||||
unsigned int srcu_idx; /* Current rdr array element. */
|
||||
unsigned long srcu_size_jiffies; /* Current contention-measurement interval. */
|
||||
unsigned long srcu_n_lock_retries; /* Contention events in current interval. */
|
||||
unsigned long srcu_n_exp_nodelay; /* # expedited no-delays in current GP phase. */
|
||||
struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */
|
||||
bool sda_is_static; /* May ->sda be passed to free_percpu()? */
|
||||
unsigned long srcu_barrier_seq; /* srcu_barrier seq #. */
|
||||
|
|
|
@ -348,11 +348,11 @@ static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
|
|||
if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)
|
||||
return;
|
||||
j = jiffies;
|
||||
if (ssp->srcu_size_jiffies != j) {
|
||||
ssp->srcu_size_jiffies = j;
|
||||
ssp->srcu_n_lock_retries = 0;
|
||||
if (ssp->srcu_sup->srcu_size_jiffies != j) {
|
||||
ssp->srcu_sup->srcu_size_jiffies = j;
|
||||
ssp->srcu_sup->srcu_n_lock_retries = 0;
|
||||
}
|
||||
if (++ssp->srcu_n_lock_retries <= small_contention_lim)
|
||||
if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)
|
||||
return;
|
||||
__srcu_transition_to_big(ssp);
|
||||
}
|
||||
|
@ -624,8 +624,8 @@ static unsigned long srcu_get_delay(struct srcu_struct *ssp)
|
|||
if (time_after(j, gpstart))
|
||||
jbase += j - gpstart;
|
||||
if (!jbase) {
|
||||
WRITE_ONCE(ssp->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_n_exp_nodelay) + 1);
|
||||
if (READ_ONCE(ssp->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)
|
||||
WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) + 1);
|
||||
if (READ_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase)
|
||||
jbase = 1;
|
||||
}
|
||||
}
|
||||
|
@ -783,7 +783,7 @@ static void srcu_gp_start(struct srcu_struct *ssp)
|
|||
rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq));
|
||||
spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */
|
||||
WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);
|
||||
WRITE_ONCE(ssp->srcu_n_exp_nodelay, 0);
|
||||
WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);
|
||||
smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
|
||||
rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);
|
||||
state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);
|
||||
|
@ -1625,7 +1625,7 @@ static void srcu_advance_state(struct srcu_struct *ssp)
|
|||
srcu_flip(ssp);
|
||||
spin_lock_irq_rcu_node(ssp->srcu_sup);
|
||||
rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);
|
||||
ssp->srcu_n_exp_nodelay = 0;
|
||||
ssp->srcu_sup->srcu_n_exp_nodelay = 0;
|
||||
spin_unlock_irq_rcu_node(ssp->srcu_sup);
|
||||
}
|
||||
|
||||
|
@ -1640,7 +1640,7 @@ static void srcu_advance_state(struct srcu_struct *ssp)
|
|||
mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
|
||||
return; /* readers present, retry later. */
|
||||
}
|
||||
ssp->srcu_n_exp_nodelay = 0;
|
||||
ssp->srcu_sup->srcu_n_exp_nodelay = 0;
|
||||
srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue