linux-stable/include/linux/hardirq.h
Thomas Gleixner 728b478d2d softirq: Add RT specific softirq accounting
RT requires the softirq processing and local bottomhalf disabled regions to
be preemptible. Using the normal preempt count based serialization is
therefore not possible because this implicitely disables preemption.

RT kernels use a per CPU local lock to serialize bottomhalfs. As
local_bh_disable() can nest the lock can only be acquired on the outermost
invocation of local_bh_disable() and released when the nest count becomes
zero. Tasks which hold the local lock can be preempted so its required to
keep track of the nest count per task.

Add a RT only counter to task struct and adjust the relevant macros in
preempt.h.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210309085726.983627589@linutronix.de
2021-03-17 16:34:08 +01:00

153 lines
3.5 KiB
C

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_HARDIRQ_H
#define LINUX_HARDIRQ_H
#include <linux/context_tracking_state.h>
#include <linux/preempt.h>
#include <linux/lockdep.h>
#include <linux/ftrace_irq.h>
#include <linux/sched.h>
#include <linux/vtime.h>
#include <asm/hardirq.h>
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
#ifdef CONFIG_NO_HZ_FULL
void __rcu_irq_enter_check_tick(void);
#else
static inline void __rcu_irq_enter_check_tick(void) { }
#endif
static __always_inline void rcu_irq_enter_check_tick(void)
{
if (context_tracking_enabled())
__rcu_irq_enter_check_tick();
}
/*
* It is safe to do non-atomic ops on ->hardirq_context,
* because NMI handlers may not preempt and the ops are
* always balanced, so the interrupted value of ->hardirq_context
* will always be restored.
*/
#define __irq_enter() \
do { \
preempt_count_add(HARDIRQ_OFFSET); \
lockdep_hardirq_enter(); \
account_hardirq_enter(current); \
} while (0)
/*
* Like __irq_enter() without time accounting for fast
* interrupts, e.g. reschedule IPI where time accounting
* is more expensive than the actual interrupt.
*/
#define __irq_enter_raw() \
do { \
preempt_count_add(HARDIRQ_OFFSET); \
lockdep_hardirq_enter(); \
} while (0)
/*
* Enter irq context (on NO_HZ, update jiffies):
*/
void irq_enter(void);
/*
* Like irq_enter(), but RCU is already watching.
*/
void irq_enter_rcu(void);
/*
* Exit irq context without processing softirqs:
*/
#define __irq_exit() \
do { \
account_hardirq_exit(current); \
lockdep_hardirq_exit(); \
preempt_count_sub(HARDIRQ_OFFSET); \
} while (0)
/*
* Like __irq_exit() without time accounting
*/
#define __irq_exit_raw() \
do { \
lockdep_hardirq_exit(); \
preempt_count_sub(HARDIRQ_OFFSET); \
} while (0)
/*
* Exit irq context and process softirqs if needed:
*/
void irq_exit(void);
/*
* Like irq_exit(), but return with RCU watching.
*/
void irq_exit_rcu(void);
#ifndef arch_nmi_enter
#define arch_nmi_enter() do { } while (0)
#define arch_nmi_exit() do { } while (0)
#endif
#ifdef CONFIG_TINY_RCU
static inline void rcu_nmi_enter(void) { }
static inline void rcu_nmi_exit(void) { }
#else
extern void rcu_nmi_enter(void);
extern void rcu_nmi_exit(void);
#endif
/*
* NMI vs Tracing
* --------------
*
* We must not land in a tracer until (or after) we've changed preempt_count
* such that in_nmi() becomes true. To that effect all NMI C entry points must
* be marked 'notrace' and call nmi_enter() as soon as possible.
*/
/*
* nmi_enter() can nest up to 15 times; see NMI_BITS.
*/
#define __nmi_enter() \
do { \
lockdep_off(); \
arch_nmi_enter(); \
printk_nmi_enter(); \
BUG_ON(in_nmi() == NMI_MASK); \
__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
} while (0)
#define nmi_enter() \
do { \
__nmi_enter(); \
lockdep_hardirq_enter(); \
rcu_nmi_enter(); \
instrumentation_begin(); \
ftrace_nmi_enter(); \
instrumentation_end(); \
} while (0)
#define __nmi_exit() \
do { \
BUG_ON(!in_nmi()); \
__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
printk_nmi_exit(); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
#define nmi_exit() \
do { \
instrumentation_begin(); \
ftrace_nmi_exit(); \
instrumentation_end(); \
rcu_nmi_exit(); \
lockdep_hardirq_exit(); \
__nmi_exit(); \
} while (0)
#endif /* LINUX_HARDIRQ_H */