mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 02:39:48 +00:00
![Linus Torvalds](/assets/img/avatar_default.png)
Pull percpu consistent-ops changes from Tejun Heo: "Way back, before the current percpu allocator was implemented, static and dynamic percpu memory areas were allocated and handled separately and had their own accessors. The distinction has been gone for many years now; however, the now duplicate two sets of accessors remained with the pointer based ones - this_cpu_*() - evolving various other operations over time. During the process, we also accumulated other inconsistent operations. This pull request contains Christoph's patches to clean up the duplicate accessor situation. __get_cpu_var() uses are replaced with with this_cpu_ptr() and __this_cpu_ptr() with raw_cpu_ptr(). Unfortunately, the former sometimes is tricky thanks to C being a bit messy with the distinction between lvalues and pointers, which led to a rather ugly solution for cpumask_var_t involving the introduction of this_cpu_cpumask_var_ptr(). This converts most of the uses but not all. Christoph will follow up with the remaining conversions in this merge window and hopefully remove the obsolete accessors" * 'for-3.18-consistent-ops' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: (38 commits) irqchip: Properly fetch the per cpu offset percpu: Resolve ambiguities in __get_cpu_var/cpumask_var_t -fix ia64: sn_nodepda cannot be assigned to after this_cpu conversion. Use __this_cpu_write. percpu: Resolve ambiguities in __get_cpu_var/cpumask_var_t Revert "powerpc: Replace __get_cpu_var uses" percpu: Remove __this_cpu_ptr clocksource: Replace __this_cpu_ptr with raw_cpu_ptr sparc: Replace __get_cpu_var uses avr32: Replace __get_cpu_var with __this_cpu_write blackfin: Replace __get_cpu_var uses tile: Use this_cpu_ptr() for hardware counters tile: Replace __get_cpu_var uses powerpc: Replace __get_cpu_var uses alpha: Replace __get_cpu_var ia64: Replace __get_cpu_var uses s390: cio driver &__get_cpu_var replacements s390: Replace __get_cpu_var uses mips: Replace __get_cpu_var uses MIPS: Replace __get_cpu_var uses in FPU emulator. arm: Replace __this_cpu_ptr with raw_cpu_ptr ...
784 lines
19 KiB
C
784 lines
19 KiB
C
/*
|
|
* linux/kernel/softirq.c
|
|
*
|
|
* Copyright (C) 1992 Linus Torvalds
|
|
*
|
|
* Distribute under GPLv2.
|
|
*
|
|
* Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
|
|
*/
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
#include <linux/export.h>
|
|
#include <linux/kernel_stat.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/init.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/notifier.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/cpu.h>
|
|
#include <linux/freezer.h>
|
|
#include <linux/kthread.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/ftrace.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/smpboot.h>
|
|
#include <linux/tick.h>
|
|
#include <linux/irq.h>
|
|
|
|
#define CREATE_TRACE_POINTS
|
|
#include <trace/events/irq.h>
|
|
|
|
/*
|
|
- No shared variables, all the data are CPU local.
|
|
- If a softirq needs serialization, let it serialize itself
|
|
by its own spinlocks.
|
|
- Even if softirq is serialized, only local cpu is marked for
|
|
execution. Hence, we get something sort of weak cpu binding.
|
|
Though it is still not clear, will it result in better locality
|
|
or will not.
|
|
|
|
Examples:
|
|
- NET RX softirq. It is multithreaded and does not require
|
|
any global serialization.
|
|
- NET TX softirq. It kicks software netdevice queues, hence
|
|
it is logically serialized per device, but this serialization
|
|
is invisible to common code.
|
|
- Tasklets: serialized wrt itself.
|
|
*/
|
|
|
|
#ifndef __ARCH_IRQ_STAT
|
|
irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
|
|
EXPORT_SYMBOL(irq_stat);
|
|
#endif
|
|
|
|
static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
|
|
|
|
DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
|
|
|
|
const char * const softirq_to_name[NR_SOFTIRQS] = {
|
|
"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
|
|
"TASKLET", "SCHED", "HRTIMER", "RCU"
|
|
};
|
|
|
|
/*
|
|
* we cannot loop indefinitely here to avoid userspace starvation,
|
|
* but we also don't want to introduce a worst case 1/HZ latency
|
|
* to the pending events, so lets the scheduler to balance
|
|
* the softirq load for us.
|
|
*/
|
|
static void wakeup_softirqd(void)
|
|
{
|
|
/* Interrupts are disabled: no need to stop preemption */
|
|
struct task_struct *tsk = __this_cpu_read(ksoftirqd);
|
|
|
|
if (tsk && tsk->state != TASK_RUNNING)
|
|
wake_up_process(tsk);
|
|
}
|
|
|
|
/*
|
|
* preempt_count and SOFTIRQ_OFFSET usage:
|
|
* - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
|
|
* softirq processing.
|
|
* - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
|
|
* on local_bh_disable or local_bh_enable.
|
|
* This lets us distinguish between whether we are currently processing
|
|
* softirq and whether we just have bh disabled.
|
|
*/
|
|
|
|
/*
|
|
* This one is for softirq.c-internal use,
|
|
* where hardirqs are disabled legitimately:
|
|
*/
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
|
|
{
|
|
unsigned long flags;
|
|
|
|
WARN_ON_ONCE(in_irq());
|
|
|
|
raw_local_irq_save(flags);
|
|
/*
|
|
* The preempt tracer hooks into preempt_count_add and will break
|
|
* lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
|
|
* is set and before current->softirq_enabled is cleared.
|
|
* We must manually increment preempt_count here and manually
|
|
* call the trace_preempt_off later.
|
|
*/
|
|
__preempt_count_add(cnt);
|
|
/*
|
|
* Were softirqs turned off above:
|
|
*/
|
|
if (softirq_count() == (cnt & SOFTIRQ_MASK))
|
|
trace_softirqs_off(ip);
|
|
raw_local_irq_restore(flags);
|
|
|
|
if (preempt_count() == cnt)
|
|
trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
|
|
}
|
|
EXPORT_SYMBOL(__local_bh_disable_ip);
|
|
#endif /* CONFIG_TRACE_IRQFLAGS */
|
|
|
|
static void __local_bh_enable(unsigned int cnt)
|
|
{
|
|
WARN_ON_ONCE(!irqs_disabled());
|
|
|
|
if (softirq_count() == (cnt & SOFTIRQ_MASK))
|
|
trace_softirqs_on(_RET_IP_);
|
|
preempt_count_sub(cnt);
|
|
}
|
|
|
|
/*
|
|
* Special-case - softirqs can safely be enabled in
|
|
* cond_resched_softirq(), or by __do_softirq(),
|
|
* without processing still-pending softirqs:
|
|
*/
|
|
void _local_bh_enable(void)
|
|
{
|
|
WARN_ON_ONCE(in_irq());
|
|
__local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
|
|
}
|
|
EXPORT_SYMBOL(_local_bh_enable);
|
|
|
|
void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
|
|
{
|
|
WARN_ON_ONCE(in_irq() || irqs_disabled());
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
local_irq_disable();
|
|
#endif
|
|
/*
|
|
* Are softirqs going to be turned on now:
|
|
*/
|
|
if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
|
|
trace_softirqs_on(ip);
|
|
/*
|
|
* Keep preemption disabled until we are done with
|
|
* softirq processing:
|
|
*/
|
|
preempt_count_sub(cnt - 1);
|
|
|
|
if (unlikely(!in_interrupt() && local_softirq_pending())) {
|
|
/*
|
|
* Run softirq if any pending. And do it in its own stack
|
|
* as we may be calling this deep in a task call stack already.
|
|
*/
|
|
do_softirq();
|
|
}
|
|
|
|
preempt_count_dec();
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
local_irq_enable();
|
|
#endif
|
|
preempt_check_resched();
|
|
}
|
|
EXPORT_SYMBOL(__local_bh_enable_ip);
|
|
|
|
/*
|
|
* We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
|
|
* but break the loop if need_resched() is set or after 2 ms.
|
|
* The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
|
|
* certain cases, such as stop_machine(), jiffies may cease to
|
|
* increment and so we need the MAX_SOFTIRQ_RESTART limit as
|
|
* well to make sure we eventually return from this method.
|
|
*
|
|
* These limits have been established via experimentation.
|
|
* The two things to balance is latency against fairness -
|
|
* we want to handle softirqs as soon as possible, but they
|
|
* should not be able to lock up the box.
|
|
*/
|
|
#define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
|
|
#define MAX_SOFTIRQ_RESTART 10
|
|
|
|
#ifdef CONFIG_TRACE_IRQFLAGS
|
|
/*
|
|
* When we run softirqs from irq_exit() and thus on the hardirq stack we need
|
|
* to keep the lockdep irq context tracking as tight as possible in order to
|
|
* not miss-qualify lock contexts and miss possible deadlocks.
|
|
*/
|
|
|
|
static inline bool lockdep_softirq_start(void)
|
|
{
|
|
bool in_hardirq = false;
|
|
|
|
if (trace_hardirq_context(current)) {
|
|
in_hardirq = true;
|
|
trace_hardirq_exit();
|
|
}
|
|
|
|
lockdep_softirq_enter();
|
|
|
|
return in_hardirq;
|
|
}
|
|
|
|
static inline void lockdep_softirq_end(bool in_hardirq)
|
|
{
|
|
lockdep_softirq_exit();
|
|
|
|
if (in_hardirq)
|
|
trace_hardirq_enter();
|
|
}
|
|
#else
|
|
static inline bool lockdep_softirq_start(void) { return false; }
|
|
static inline void lockdep_softirq_end(bool in_hardirq) { }
|
|
#endif
|
|
|
|
asmlinkage __visible void __do_softirq(void)
|
|
{
|
|
unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
|
|
unsigned long old_flags = current->flags;
|
|
int max_restart = MAX_SOFTIRQ_RESTART;
|
|
struct softirq_action *h;
|
|
bool in_hardirq;
|
|
__u32 pending;
|
|
int softirq_bit;
|
|
|
|
/*
|
|
* Mask out PF_MEMALLOC s current task context is borrowed for the
|
|
* softirq. A softirq handled such as network RX might set PF_MEMALLOC
|
|
* again if the socket is related to swap
|
|
*/
|
|
current->flags &= ~PF_MEMALLOC;
|
|
|
|
pending = local_softirq_pending();
|
|
account_irq_enter_time(current);
|
|
|
|
__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
|
|
in_hardirq = lockdep_softirq_start();
|
|
|
|
restart:
|
|
/* Reset the pending bitmask before enabling irqs */
|
|
set_softirq_pending(0);
|
|
|
|
local_irq_enable();
|
|
|
|
h = softirq_vec;
|
|
|
|
while ((softirq_bit = ffs(pending))) {
|
|
unsigned int vec_nr;
|
|
int prev_count;
|
|
|
|
h += softirq_bit - 1;
|
|
|
|
vec_nr = h - softirq_vec;
|
|
prev_count = preempt_count();
|
|
|
|
kstat_incr_softirqs_this_cpu(vec_nr);
|
|
|
|
trace_softirq_entry(vec_nr);
|
|
h->action(h);
|
|
trace_softirq_exit(vec_nr);
|
|
if (unlikely(prev_count != preempt_count())) {
|
|
pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
|
|
vec_nr, softirq_to_name[vec_nr], h->action,
|
|
prev_count, preempt_count());
|
|
preempt_count_set(prev_count);
|
|
}
|
|
h++;
|
|
pending >>= softirq_bit;
|
|
}
|
|
|
|
rcu_bh_qs();
|
|
local_irq_disable();
|
|
|
|
pending = local_softirq_pending();
|
|
if (pending) {
|
|
if (time_before(jiffies, end) && !need_resched() &&
|
|
--max_restart)
|
|
goto restart;
|
|
|
|
wakeup_softirqd();
|
|
}
|
|
|
|
lockdep_softirq_end(in_hardirq);
|
|
account_irq_exit_time(current);
|
|
__local_bh_enable(SOFTIRQ_OFFSET);
|
|
WARN_ON_ONCE(in_interrupt());
|
|
tsk_restore_flags(current, old_flags, PF_MEMALLOC);
|
|
}
|
|
|
|
asmlinkage __visible void do_softirq(void)
|
|
{
|
|
__u32 pending;
|
|
unsigned long flags;
|
|
|
|
if (in_interrupt())
|
|
return;
|
|
|
|
local_irq_save(flags);
|
|
|
|
pending = local_softirq_pending();
|
|
|
|
if (pending)
|
|
do_softirq_own_stack();
|
|
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
/*
|
|
* Enter an interrupt context.
|
|
*/
|
|
void irq_enter(void)
|
|
{
|
|
rcu_irq_enter();
|
|
if (is_idle_task(current) && !in_interrupt()) {
|
|
/*
|
|
* Prevent raise_softirq from needlessly waking up ksoftirqd
|
|
* here, as softirq will be serviced on return from interrupt.
|
|
*/
|
|
local_bh_disable();
|
|
tick_irq_enter();
|
|
_local_bh_enable();
|
|
}
|
|
|
|
__irq_enter();
|
|
}
|
|
|
|
static inline void invoke_softirq(void)
|
|
{
|
|
if (!force_irqthreads) {
|
|
#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
|
|
/*
|
|
* We can safely execute softirq on the current stack if
|
|
* it is the irq stack, because it should be near empty
|
|
* at this stage.
|
|
*/
|
|
__do_softirq();
|
|
#else
|
|
/*
|
|
* Otherwise, irq_exit() is called on the task stack that can
|
|
* be potentially deep already. So call softirq in its own stack
|
|
* to prevent from any overrun.
|
|
*/
|
|
do_softirq_own_stack();
|
|
#endif
|
|
} else {
|
|
wakeup_softirqd();
|
|
}
|
|
}
|
|
|
|
static inline void tick_irq_exit(void)
|
|
{
|
|
#ifdef CONFIG_NO_HZ_COMMON
|
|
int cpu = smp_processor_id();
|
|
|
|
/* Make sure that timer wheel updates are propagated */
|
|
if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
|
|
if (!in_interrupt())
|
|
tick_nohz_irq_exit();
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/*
|
|
* Exit an interrupt context. Process softirqs if needed and possible:
|
|
*/
|
|
void irq_exit(void)
|
|
{
|
|
#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
|
|
local_irq_disable();
|
|
#else
|
|
WARN_ON_ONCE(!irqs_disabled());
|
|
#endif
|
|
|
|
account_irq_exit_time(current);
|
|
preempt_count_sub(HARDIRQ_OFFSET);
|
|
if (!in_interrupt() && local_softirq_pending())
|
|
invoke_softirq();
|
|
|
|
tick_irq_exit();
|
|
rcu_irq_exit();
|
|
trace_hardirq_exit(); /* must be last! */
|
|
}
|
|
|
|
/*
|
|
* This function must run with irqs disabled!
|
|
*/
|
|
inline void raise_softirq_irqoff(unsigned int nr)
|
|
{
|
|
__raise_softirq_irqoff(nr);
|
|
|
|
/*
|
|
* If we're in an interrupt or softirq, we're done
|
|
* (this also catches softirq-disabled code). We will
|
|
* actually run the softirq once we return from
|
|
* the irq or softirq.
|
|
*
|
|
* Otherwise we wake up ksoftirqd to make sure we
|
|
* schedule the softirq soon.
|
|
*/
|
|
if (!in_interrupt())
|
|
wakeup_softirqd();
|
|
}
|
|
|
|
void raise_softirq(unsigned int nr)
|
|
{
|
|
unsigned long flags;
|
|
|
|
local_irq_save(flags);
|
|
raise_softirq_irqoff(nr);
|
|
local_irq_restore(flags);
|
|
}
|
|
|
|
void __raise_softirq_irqoff(unsigned int nr)
|
|
{
|
|
trace_softirq_raise(nr);
|
|
or_softirq_pending(1UL << nr);
|
|
}
|
|
|
|
void open_softirq(int nr, void (*action)(struct softirq_action *))
|
|
{
|
|
softirq_vec[nr].action = action;
|
|
}
|
|
|
|
/*
|
|
* Tasklets
|
|
*/
|
|
struct tasklet_head {
|
|
struct tasklet_struct *head;
|
|
struct tasklet_struct **tail;
|
|
};
|
|
|
|
static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
|
|
static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
|
|
|
|
void __tasklet_schedule(struct tasklet_struct *t)
|
|
{
|
|
unsigned long flags;
|
|
|
|
local_irq_save(flags);
|
|
t->next = NULL;
|
|
*__this_cpu_read(tasklet_vec.tail) = t;
|
|
__this_cpu_write(tasklet_vec.tail, &(t->next));
|
|
raise_softirq_irqoff(TASKLET_SOFTIRQ);
|
|
local_irq_restore(flags);
|
|
}
|
|
EXPORT_SYMBOL(__tasklet_schedule);
|
|
|
|
void __tasklet_hi_schedule(struct tasklet_struct *t)
|
|
{
|
|
unsigned long flags;
|
|
|
|
local_irq_save(flags);
|
|
t->next = NULL;
|
|
*__this_cpu_read(tasklet_hi_vec.tail) = t;
|
|
__this_cpu_write(tasklet_hi_vec.tail, &(t->next));
|
|
raise_softirq_irqoff(HI_SOFTIRQ);
|
|
local_irq_restore(flags);
|
|
}
|
|
EXPORT_SYMBOL(__tasklet_hi_schedule);
|
|
|
|
void __tasklet_hi_schedule_first(struct tasklet_struct *t)
|
|
{
|
|
BUG_ON(!irqs_disabled());
|
|
|
|
t->next = __this_cpu_read(tasklet_hi_vec.head);
|
|
__this_cpu_write(tasklet_hi_vec.head, t);
|
|
__raise_softirq_irqoff(HI_SOFTIRQ);
|
|
}
|
|
EXPORT_SYMBOL(__tasklet_hi_schedule_first);
|
|
|
|
static void tasklet_action(struct softirq_action *a)
|
|
{
|
|
struct tasklet_struct *list;
|
|
|
|
local_irq_disable();
|
|
list = __this_cpu_read(tasklet_vec.head);
|
|
__this_cpu_write(tasklet_vec.head, NULL);
|
|
__this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
|
|
local_irq_enable();
|
|
|
|
while (list) {
|
|
struct tasklet_struct *t = list;
|
|
|
|
list = list->next;
|
|
|
|
if (tasklet_trylock(t)) {
|
|
if (!atomic_read(&t->count)) {
|
|
if (!test_and_clear_bit(TASKLET_STATE_SCHED,
|
|
&t->state))
|
|
BUG();
|
|
t->func(t->data);
|
|
tasklet_unlock(t);
|
|
continue;
|
|
}
|
|
tasklet_unlock(t);
|
|
}
|
|
|
|
local_irq_disable();
|
|
t->next = NULL;
|
|
*__this_cpu_read(tasklet_vec.tail) = t;
|
|
__this_cpu_write(tasklet_vec.tail, &(t->next));
|
|
__raise_softirq_irqoff(TASKLET_SOFTIRQ);
|
|
local_irq_enable();
|
|
}
|
|
}
|
|
|
|
static void tasklet_hi_action(struct softirq_action *a)
|
|
{
|
|
struct tasklet_struct *list;
|
|
|
|
local_irq_disable();
|
|
list = __this_cpu_read(tasklet_hi_vec.head);
|
|
__this_cpu_write(tasklet_hi_vec.head, NULL);
|
|
__this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head));
|
|
local_irq_enable();
|
|
|
|
while (list) {
|
|
struct tasklet_struct *t = list;
|
|
|
|
list = list->next;
|
|
|
|
if (tasklet_trylock(t)) {
|
|
if (!atomic_read(&t->count)) {
|
|
if (!test_and_clear_bit(TASKLET_STATE_SCHED,
|
|
&t->state))
|
|
BUG();
|
|
t->func(t->data);
|
|
tasklet_unlock(t);
|
|
continue;
|
|
}
|
|
tasklet_unlock(t);
|
|
}
|
|
|
|
local_irq_disable();
|
|
t->next = NULL;
|
|
*__this_cpu_read(tasklet_hi_vec.tail) = t;
|
|
__this_cpu_write(tasklet_hi_vec.tail, &(t->next));
|
|
__raise_softirq_irqoff(HI_SOFTIRQ);
|
|
local_irq_enable();
|
|
}
|
|
}
|
|
|
|
void tasklet_init(struct tasklet_struct *t,
|
|
void (*func)(unsigned long), unsigned long data)
|
|
{
|
|
t->next = NULL;
|
|
t->state = 0;
|
|
atomic_set(&t->count, 0);
|
|
t->func = func;
|
|
t->data = data;
|
|
}
|
|
EXPORT_SYMBOL(tasklet_init);
|
|
|
|
void tasklet_kill(struct tasklet_struct *t)
|
|
{
|
|
if (in_interrupt())
|
|
pr_notice("Attempt to kill tasklet from interrupt\n");
|
|
|
|
while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
|
|
do {
|
|
yield();
|
|
} while (test_bit(TASKLET_STATE_SCHED, &t->state));
|
|
}
|
|
tasklet_unlock_wait(t);
|
|
clear_bit(TASKLET_STATE_SCHED, &t->state);
|
|
}
|
|
EXPORT_SYMBOL(tasklet_kill);
|
|
|
|
/*
|
|
* tasklet_hrtimer
|
|
*/
|
|
|
|
/*
|
|
* The trampoline is called when the hrtimer expires. It schedules a tasklet
|
|
* to run __tasklet_hrtimer_trampoline() which in turn will call the intended
|
|
* hrtimer callback, but from softirq context.
|
|
*/
|
|
static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
|
|
{
|
|
struct tasklet_hrtimer *ttimer =
|
|
container_of(timer, struct tasklet_hrtimer, timer);
|
|
|
|
tasklet_hi_schedule(&ttimer->tasklet);
|
|
return HRTIMER_NORESTART;
|
|
}
|
|
|
|
/*
|
|
* Helper function which calls the hrtimer callback from
|
|
* tasklet/softirq context
|
|
*/
|
|
static void __tasklet_hrtimer_trampoline(unsigned long data)
|
|
{
|
|
struct tasklet_hrtimer *ttimer = (void *)data;
|
|
enum hrtimer_restart restart;
|
|
|
|
restart = ttimer->function(&ttimer->timer);
|
|
if (restart != HRTIMER_NORESTART)
|
|
hrtimer_restart(&ttimer->timer);
|
|
}
|
|
|
|
/**
|
|
* tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
|
|
* @ttimer: tasklet_hrtimer which is initialized
|
|
* @function: hrtimer callback function which gets called from softirq context
|
|
* @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
|
|
* @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
|
|
*/
|
|
void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
|
|
enum hrtimer_restart (*function)(struct hrtimer *),
|
|
clockid_t which_clock, enum hrtimer_mode mode)
|
|
{
|
|
hrtimer_init(&ttimer->timer, which_clock, mode);
|
|
ttimer->timer.function = __hrtimer_tasklet_trampoline;
|
|
tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
|
|
(unsigned long)ttimer);
|
|
ttimer->function = function;
|
|
}
|
|
EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
|
|
|
|
void __init softirq_init(void)
|
|
{
|
|
int cpu;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
per_cpu(tasklet_vec, cpu).tail =
|
|
&per_cpu(tasklet_vec, cpu).head;
|
|
per_cpu(tasklet_hi_vec, cpu).tail =
|
|
&per_cpu(tasklet_hi_vec, cpu).head;
|
|
}
|
|
|
|
open_softirq(TASKLET_SOFTIRQ, tasklet_action);
|
|
open_softirq(HI_SOFTIRQ, tasklet_hi_action);
|
|
}
|
|
|
|
static int ksoftirqd_should_run(unsigned int cpu)
|
|
{
|
|
return local_softirq_pending();
|
|
}
|
|
|
|
static void run_ksoftirqd(unsigned int cpu)
|
|
{
|
|
local_irq_disable();
|
|
if (local_softirq_pending()) {
|
|
/*
|
|
* We can safely run softirq on inline stack, as we are not deep
|
|
* in the task stack here.
|
|
*/
|
|
__do_softirq();
|
|
rcu_note_context_switch(cpu);
|
|
local_irq_enable();
|
|
cond_resched();
|
|
return;
|
|
}
|
|
local_irq_enable();
|
|
}
|
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
/*
|
|
* tasklet_kill_immediate is called to remove a tasklet which can already be
|
|
* scheduled for execution on @cpu.
|
|
*
|
|
* Unlike tasklet_kill, this function removes the tasklet
|
|
* _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
|
|
*
|
|
* When this function is called, @cpu must be in the CPU_DEAD state.
|
|
*/
|
|
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
|
|
{
|
|
struct tasklet_struct **i;
|
|
|
|
BUG_ON(cpu_online(cpu));
|
|
BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
|
|
|
|
if (!test_bit(TASKLET_STATE_SCHED, &t->state))
|
|
return;
|
|
|
|
/* CPU is dead, so no lock needed. */
|
|
for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
|
|
if (*i == t) {
|
|
*i = t->next;
|
|
/* If this was the tail element, move the tail ptr */
|
|
if (*i == NULL)
|
|
per_cpu(tasklet_vec, cpu).tail = i;
|
|
return;
|
|
}
|
|
}
|
|
BUG();
|
|
}
|
|
|
|
static void takeover_tasklets(unsigned int cpu)
|
|
{
|
|
/* CPU is dead, so no lock needed. */
|
|
local_irq_disable();
|
|
|
|
/* Find end, append list for that CPU. */
|
|
if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
|
|
*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
|
|
this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
|
|
per_cpu(tasklet_vec, cpu).head = NULL;
|
|
per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
|
|
}
|
|
raise_softirq_irqoff(TASKLET_SOFTIRQ);
|
|
|
|
if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
|
|
*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
|
|
__this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
|
|
per_cpu(tasklet_hi_vec, cpu).head = NULL;
|
|
per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
|
|
}
|
|
raise_softirq_irqoff(HI_SOFTIRQ);
|
|
|
|
local_irq_enable();
|
|
}
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
|
|
static int cpu_callback(struct notifier_block *nfb, unsigned long action,
|
|
void *hcpu)
|
|
{
|
|
switch (action) {
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
case CPU_DEAD:
|
|
case CPU_DEAD_FROZEN:
|
|
takeover_tasklets((unsigned long)hcpu);
|
|
break;
|
|
#endif /* CONFIG_HOTPLUG_CPU */
|
|
}
|
|
return NOTIFY_OK;
|
|
}
|
|
|
|
static struct notifier_block cpu_nfb = {
|
|
.notifier_call = cpu_callback
|
|
};
|
|
|
|
static struct smp_hotplug_thread softirq_threads = {
|
|
.store = &ksoftirqd,
|
|
.thread_should_run = ksoftirqd_should_run,
|
|
.thread_fn = run_ksoftirqd,
|
|
.thread_comm = "ksoftirqd/%u",
|
|
};
|
|
|
|
static __init int spawn_ksoftirqd(void)
|
|
{
|
|
register_cpu_notifier(&cpu_nfb);
|
|
|
|
BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
|
|
|
|
return 0;
|
|
}
|
|
early_initcall(spawn_ksoftirqd);
|
|
|
|
/*
|
|
* [ These __weak aliases are kept in a separate compilation unit, so that
|
|
* GCC does not inline them incorrectly. ]
|
|
*/
|
|
|
|
int __init __weak early_irq_init(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
int __init __weak arch_probe_nr_irqs(void)
|
|
{
|
|
return NR_IRQS_LEGACY;
|
|
}
|
|
|
|
int __init __weak arch_early_irq_init(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
|
|
{
|
|
return from;
|
|
}
|