mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 05:48:40 +00:00
![Paul E. McKenney](/assets/img/avatar_default.png)
Currently, IPIs are used to force other CPUs to invalidate their TLBs in response to a kernel virtual-memory mapping change. This works, but degrades both battery lifetime (for idle CPUs) and real-time response (for nohz_full CPUs), and in addition results in unnecessary IPIs due to the fact that CPUs executing in usermode are unaffected by stale kernel mappings. It would be better to cause a CPU executing in usermode to wait until it is entering kernel mode to do the flush, first to avoid interrupting usemode tasks and second to handle multiple flush requests with a single flush in the case of a long-running user task. This commit therefore reserves a bit at the bottom of the ->dynticks counter, which is checked upon exit from extended quiescent states. If it is set, it is cleared and then a new rcu_eqs_special_exit() macro is invoked, which, if not supplied, is an empty single-pass do-while loop. If this bottom bit is set on -entry- to an extended quiescent state, then a WARN_ON_ONCE() triggers. This bottom bit may be set using a new rcu_eqs_special_set() function, which returns true if the bit was set, or false if the CPU turned out to not be in an extended quiescent state. Please note that this function refuses to set the bit for a non-nohz_full CPU when that CPU is executing in usermode because usermode execution is tracked by RCU as a dyntick-idle extended quiescent state only for nohz_full CPUs. Reported-by: Andy Lutomirski <luto@amacapital.net> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
257 lines
5 KiB
C
257 lines
5 KiB
C
/*
|
|
* Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, you can access it online at
|
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
|
*
|
|
* Copyright IBM Corporation, 2008
|
|
*
|
|
* Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
|
|
*
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
|
* Documentation/RCU
|
|
*/
|
|
#ifndef __LINUX_TINY_H
|
|
#define __LINUX_TINY_H
|
|
|
|
#include <linux/cache.h>
|
|
|
|
struct rcu_dynticks;
|
|
static inline int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline bool rcu_eqs_special_set(int cpu)
|
|
{
|
|
return false; /* Never flag non-existent other CPUs! */
|
|
}
|
|
|
|
static inline unsigned long get_state_synchronize_rcu(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void cond_synchronize_rcu(unsigned long oldstate)
|
|
{
|
|
might_sleep();
|
|
}
|
|
|
|
static inline unsigned long get_state_synchronize_sched(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void cond_synchronize_sched(unsigned long oldstate)
|
|
{
|
|
might_sleep();
|
|
}
|
|
|
|
extern void rcu_barrier_bh(void);
|
|
extern void rcu_barrier_sched(void);
|
|
|
|
static inline void synchronize_rcu_expedited(void)
|
|
{
|
|
synchronize_sched(); /* Only one CPU, so pretty fast anyway!!! */
|
|
}
|
|
|
|
static inline void rcu_barrier(void)
|
|
{
|
|
rcu_barrier_sched(); /* Only one CPU, so only one list of callbacks! */
|
|
}
|
|
|
|
static inline void synchronize_rcu_bh(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void synchronize_rcu_bh_expedited(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void synchronize_sched_expedited(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void kfree_call_rcu(struct rcu_head *head,
|
|
rcu_callback_t func)
|
|
{
|
|
call_rcu(head, func);
|
|
}
|
|
|
|
static inline void rcu_note_context_switch(void)
|
|
{
|
|
rcu_sched_qs();
|
|
}
|
|
|
|
/*
|
|
* Take advantage of the fact that there is only one CPU, which
|
|
* allows us to ignore virtualization-based context switches.
|
|
*/
|
|
static inline void rcu_virt_note_context_switch(int cpu)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* Return the number of grace periods started.
|
|
*/
|
|
static inline unsigned long rcu_batches_started(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of bottom-half grace periods started.
|
|
*/
|
|
static inline unsigned long rcu_batches_started_bh(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of sched grace periods started.
|
|
*/
|
|
static inline unsigned long rcu_batches_started_sched(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of grace periods completed.
|
|
*/
|
|
static inline unsigned long rcu_batches_completed(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of bottom-half grace periods completed.
|
|
*/
|
|
static inline unsigned long rcu_batches_completed_bh(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of sched grace periods completed.
|
|
*/
|
|
static inline unsigned long rcu_batches_completed_sched(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of expedited grace periods completed.
|
|
*/
|
|
static inline unsigned long rcu_exp_batches_completed(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Return the number of expedited sched grace periods completed.
|
|
*/
|
|
static inline unsigned long rcu_exp_batches_completed_sched(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void rcu_force_quiescent_state(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_bh_force_quiescent_state(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_sched_force_quiescent_state(void)
|
|
{
|
|
}
|
|
|
|
static inline void show_rcu_gp_kthreads(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_cpu_stall_reset(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_idle_enter(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_idle_exit(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_irq_enter(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_irq_exit_irqson(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_irq_enter_irqson(void)
|
|
{
|
|
}
|
|
|
|
static inline void rcu_irq_exit(void)
|
|
{
|
|
}
|
|
|
|
static inline void exit_rcu(void)
|
|
{
|
|
}
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
extern int rcu_scheduler_active __read_mostly;
|
|
void rcu_scheduler_starting(void);
|
|
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
static inline void rcu_scheduler_starting(void)
|
|
{
|
|
}
|
|
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE)
|
|
|
|
static inline bool rcu_is_watching(void)
|
|
{
|
|
return __rcu_is_watching();
|
|
}
|
|
|
|
#else /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
|
|
|
|
static inline bool rcu_is_watching(void)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
#endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
|
|
|
|
static inline void rcu_all_qs(void)
|
|
{
|
|
barrier(); /* Avoid RCU read-side critical sections leaking across. */
|
|
}
|
|
|
|
/* RCUtree hotplug events */
|
|
#define rcutree_prepare_cpu NULL
|
|
#define rcutree_online_cpu NULL
|
|
#define rcutree_offline_cpu NULL
|
|
#define rcutree_dead_cpu NULL
|
|
#define rcutree_dying_cpu NULL
|
|
|
|
#endif /* __LINUX_RCUTINY_H */
|