mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-25 10:19:47 +00:00
![Thomas Gleixner](/assets/img/avatar_default.png)
Based on 1 normalized pattern(s): this program is free software you can redistribute it and or modify it under the terms of the gnu general public license as published by the free software foundation version 2 of the license extracted by the scancode license scanner the SPDX license identifier GPL-2.0-only has been chosen to replace the boilerplate/reference in 315 file(s). Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Reviewed-by: Allison Randal <allison@lohutok.net> Reviewed-by: Armijn Hemel <armijn@tjaldur.nl> Cc: linux-spdx@vger.kernel.org Link: https://lkml.kernel.org/r/20190531190115.503150771@linutronix.de Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
334 lines
10 KiB
C
334 lines
10 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* async.c: Asynchronous function calls for boot performance
|
|
*
|
|
* (C) Copyright 2009 Intel Corporation
|
|
* Author: Arjan van de Ven <arjan@linux.intel.com>
|
|
*/
|
|
|
|
|
|
/*
|
|
|
|
Goals and Theory of Operation
|
|
|
|
The primary goal of this feature is to reduce the kernel boot time,
|
|
by doing various independent hardware delays and discovery operations
|
|
decoupled and not strictly serialized.
|
|
|
|
More specifically, the asynchronous function call concept allows
|
|
certain operations (primarily during system boot) to happen
|
|
asynchronously, out of order, while these operations still
|
|
have their externally visible parts happen sequentially and in-order.
|
|
(not unlike how out-of-order CPUs retire their instructions in order)
|
|
|
|
Key to the asynchronous function call implementation is the concept of
|
|
a "sequence cookie" (which, although it has an abstracted type, can be
|
|
thought of as a monotonically incrementing number).
|
|
|
|
The async core will assign each scheduled event such a sequence cookie and
|
|
pass this to the called functions.
|
|
|
|
The asynchronously called function should before doing a globally visible
|
|
operation, such as registering device numbers, call the
|
|
async_synchronize_cookie() function and pass in its own cookie. The
|
|
async_synchronize_cookie() function will make sure that all asynchronous
|
|
operations that were scheduled prior to the operation corresponding with the
|
|
cookie have completed.
|
|
|
|
Subsystem/driver initialization code that scheduled asynchronous probe
|
|
functions, but which shares global resources with other drivers/subsystems
|
|
that do not use the asynchronous call feature, need to do a full
|
|
synchronization with the async_synchronize_full() function, before returning
|
|
from their init function. This is to maintain strict ordering between the
|
|
asynchronous and synchronous parts of the kernel.
|
|
|
|
*/
|
|
|
|
#include <linux/async.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/ktime.h>
|
|
#include <linux/export.h>
|
|
#include <linux/wait.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/workqueue.h>
|
|
|
|
#include "workqueue_internal.h"
|
|
|
|
static async_cookie_t next_cookie = 1;
|
|
|
|
#define MAX_WORK 32768
|
|
#define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
|
|
|
|
static LIST_HEAD(async_global_pending); /* pending from all registered doms */
|
|
static ASYNC_DOMAIN(async_dfl_domain);
|
|
static DEFINE_SPINLOCK(async_lock);
|
|
|
|
struct async_entry {
|
|
struct list_head domain_list;
|
|
struct list_head global_list;
|
|
struct work_struct work;
|
|
async_cookie_t cookie;
|
|
async_func_t func;
|
|
void *data;
|
|
struct async_domain *domain;
|
|
};
|
|
|
|
static DECLARE_WAIT_QUEUE_HEAD(async_done);
|
|
|
|
static atomic_t entry_count;
|
|
|
|
static async_cookie_t lowest_in_progress(struct async_domain *domain)
|
|
{
|
|
struct async_entry *first = NULL;
|
|
async_cookie_t ret = ASYNC_COOKIE_MAX;
|
|
unsigned long flags;
|
|
|
|
spin_lock_irqsave(&async_lock, flags);
|
|
|
|
if (domain) {
|
|
if (!list_empty(&domain->pending))
|
|
first = list_first_entry(&domain->pending,
|
|
struct async_entry, domain_list);
|
|
} else {
|
|
if (!list_empty(&async_global_pending))
|
|
first = list_first_entry(&async_global_pending,
|
|
struct async_entry, global_list);
|
|
}
|
|
|
|
if (first)
|
|
ret = first->cookie;
|
|
|
|
spin_unlock_irqrestore(&async_lock, flags);
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* pick the first pending entry and run it
|
|
*/
|
|
static void async_run_entry_fn(struct work_struct *work)
|
|
{
|
|
struct async_entry *entry =
|
|
container_of(work, struct async_entry, work);
|
|
unsigned long flags;
|
|
ktime_t uninitialized_var(calltime), delta, rettime;
|
|
|
|
/* 1) run (and print duration) */
|
|
if (initcall_debug && system_state < SYSTEM_RUNNING) {
|
|
pr_debug("calling %lli_%pS @ %i\n",
|
|
(long long)entry->cookie,
|
|
entry->func, task_pid_nr(current));
|
|
calltime = ktime_get();
|
|
}
|
|
entry->func(entry->data, entry->cookie);
|
|
if (initcall_debug && system_state < SYSTEM_RUNNING) {
|
|
rettime = ktime_get();
|
|
delta = ktime_sub(rettime, calltime);
|
|
pr_debug("initcall %lli_%pS returned 0 after %lld usecs\n",
|
|
(long long)entry->cookie,
|
|
entry->func,
|
|
(long long)ktime_to_ns(delta) >> 10);
|
|
}
|
|
|
|
/* 2) remove self from the pending queues */
|
|
spin_lock_irqsave(&async_lock, flags);
|
|
list_del_init(&entry->domain_list);
|
|
list_del_init(&entry->global_list);
|
|
|
|
/* 3) free the entry */
|
|
kfree(entry);
|
|
atomic_dec(&entry_count);
|
|
|
|
spin_unlock_irqrestore(&async_lock, flags);
|
|
|
|
/* 4) wake up any waiters */
|
|
wake_up(&async_done);
|
|
}
|
|
|
|
/**
|
|
* async_schedule_node_domain - NUMA specific version of async_schedule_domain
|
|
* @func: function to execute asynchronously
|
|
* @data: data pointer to pass to the function
|
|
* @node: NUMA node that we want to schedule this on or close to
|
|
* @domain: the domain
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* @domain may be used in the async_synchronize_*_domain() functions to
|
|
* wait within a certain synchronization domain rather than globally.
|
|
*
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*
|
|
* The node requested will be honored on a best effort basis. If the node
|
|
* has no CPUs associated with it then the work is distributed among all
|
|
* available CPUs.
|
|
*/
|
|
async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
|
|
int node, struct async_domain *domain)
|
|
{
|
|
struct async_entry *entry;
|
|
unsigned long flags;
|
|
async_cookie_t newcookie;
|
|
|
|
/* allow irq-off callers */
|
|
entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
|
|
|
|
/*
|
|
* If we're out of memory or if there's too much work
|
|
* pending already, we execute synchronously.
|
|
*/
|
|
if (!entry || atomic_read(&entry_count) > MAX_WORK) {
|
|
kfree(entry);
|
|
spin_lock_irqsave(&async_lock, flags);
|
|
newcookie = next_cookie++;
|
|
spin_unlock_irqrestore(&async_lock, flags);
|
|
|
|
/* low on memory.. run synchronously */
|
|
func(data, newcookie);
|
|
return newcookie;
|
|
}
|
|
INIT_LIST_HEAD(&entry->domain_list);
|
|
INIT_LIST_HEAD(&entry->global_list);
|
|
INIT_WORK(&entry->work, async_run_entry_fn);
|
|
entry->func = func;
|
|
entry->data = data;
|
|
entry->domain = domain;
|
|
|
|
spin_lock_irqsave(&async_lock, flags);
|
|
|
|
/* allocate cookie and queue */
|
|
newcookie = entry->cookie = next_cookie++;
|
|
|
|
list_add_tail(&entry->domain_list, &domain->pending);
|
|
if (domain->registered)
|
|
list_add_tail(&entry->global_list, &async_global_pending);
|
|
|
|
atomic_inc(&entry_count);
|
|
spin_unlock_irqrestore(&async_lock, flags);
|
|
|
|
/* mark that this task has queued an async job, used by module init */
|
|
current->flags |= PF_USED_ASYNC;
|
|
|
|
/* schedule for execution */
|
|
queue_work_node(node, system_unbound_wq, &entry->work);
|
|
|
|
return newcookie;
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_schedule_node_domain);
|
|
|
|
/**
|
|
* async_schedule_node - NUMA specific version of async_schedule
|
|
* @func: function to execute asynchronously
|
|
* @data: data pointer to pass to the function
|
|
* @node: NUMA node that we want to schedule this on or close to
|
|
*
|
|
* Returns an async_cookie_t that may be used for checkpointing later.
|
|
* Note: This function may be called from atomic or non-atomic contexts.
|
|
*
|
|
* The node requested will be honored on a best effort basis. If the node
|
|
* has no CPUs associated with it then the work is distributed among all
|
|
* available CPUs.
|
|
*/
|
|
async_cookie_t async_schedule_node(async_func_t func, void *data, int node)
|
|
{
|
|
return async_schedule_node_domain(func, data, node, &async_dfl_domain);
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_schedule_node);
|
|
|
|
/**
|
|
* async_synchronize_full - synchronize all asynchronous function calls
|
|
*
|
|
* This function waits until all asynchronous function calls have been done.
|
|
*/
|
|
void async_synchronize_full(void)
|
|
{
|
|
async_synchronize_full_domain(NULL);
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_synchronize_full);
|
|
|
|
/**
|
|
* async_unregister_domain - ensure no more anonymous waiters on this domain
|
|
* @domain: idle domain to flush out of any async_synchronize_full instances
|
|
*
|
|
* async_synchronize_{cookie|full}_domain() are not flushed since callers
|
|
* of these routines should know the lifetime of @domain
|
|
*
|
|
* Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
|
|
*/
|
|
void async_unregister_domain(struct async_domain *domain)
|
|
{
|
|
spin_lock_irq(&async_lock);
|
|
WARN_ON(!domain->registered || !list_empty(&domain->pending));
|
|
domain->registered = 0;
|
|
spin_unlock_irq(&async_lock);
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_unregister_domain);
|
|
|
|
/**
|
|
* async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
|
|
* @domain: the domain to synchronize
|
|
*
|
|
* This function waits until all asynchronous function calls for the
|
|
* synchronization domain specified by @domain have been done.
|
|
*/
|
|
void async_synchronize_full_domain(struct async_domain *domain)
|
|
{
|
|
async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
|
|
|
|
/**
|
|
* async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
|
|
* @cookie: async_cookie_t to use as checkpoint
|
|
* @domain: the domain to synchronize (%NULL for all registered domains)
|
|
*
|
|
* This function waits until all asynchronous function calls for the
|
|
* synchronization domain specified by @domain submitted prior to @cookie
|
|
* have been done.
|
|
*/
|
|
void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
|
|
{
|
|
ktime_t uninitialized_var(starttime), delta, endtime;
|
|
|
|
if (initcall_debug && system_state < SYSTEM_RUNNING) {
|
|
pr_debug("async_waiting @ %i\n", task_pid_nr(current));
|
|
starttime = ktime_get();
|
|
}
|
|
|
|
wait_event(async_done, lowest_in_progress(domain) >= cookie);
|
|
|
|
if (initcall_debug && system_state < SYSTEM_RUNNING) {
|
|
endtime = ktime_get();
|
|
delta = ktime_sub(endtime, starttime);
|
|
|
|
pr_debug("async_continuing @ %i after %lli usec\n",
|
|
task_pid_nr(current),
|
|
(long long)ktime_to_ns(delta) >> 10);
|
|
}
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
|
|
|
|
/**
|
|
* async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
|
|
* @cookie: async_cookie_t to use as checkpoint
|
|
*
|
|
* This function waits until all asynchronous function calls prior to @cookie
|
|
* have been done.
|
|
*/
|
|
void async_synchronize_cookie(async_cookie_t cookie)
|
|
{
|
|
async_synchronize_cookie_domain(cookie, &async_dfl_domain);
|
|
}
|
|
EXPORT_SYMBOL_GPL(async_synchronize_cookie);
|
|
|
|
/**
|
|
* current_is_async - is %current an async worker task?
|
|
*
|
|
* Returns %true if %current is an async worker task.
|
|
*/
|
|
bool current_is_async(void)
|
|
{
|
|
struct worker *worker = current_wq_worker();
|
|
|
|
return worker && worker->current_func == async_run_entry_fn;
|
|
}
|
|
EXPORT_SYMBOL_GPL(current_is_async);
|