2019-05-19 13:51:37 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2006-05-24 00:18:44 +00:00
|
|
|
/*
|
|
|
|
* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This code implements the DMA subsystem. It provides a HW-neutral interface
|
|
|
|
* for other kernel code to use asynchronous memory copy capabilities,
|
|
|
|
* if present, and allows different HW DMA drivers to register as providing
|
|
|
|
* this capability.
|
|
|
|
*
|
|
|
|
* Due to the fact we are accelerating what is already a relatively fast
|
|
|
|
* operation, the code goes to great lengths to avoid additional overhead,
|
|
|
|
* such as locking.
|
|
|
|
*
|
|
|
|
* LOCKING:
|
|
|
|
*
|
2009-01-06 18:38:17 +00:00
|
|
|
* The subsystem keeps a global list of dma_device structs it is protected by a
|
|
|
|
* mutex, dma_list_mutex.
|
2006-05-24 00:18:44 +00:00
|
|
|
*
|
2009-01-06 18:38:18 +00:00
|
|
|
* A subsystem can get access to a channel by calling dmaengine_get() followed
|
|
|
|
* by dma_find_channel(), or if it has need for an exclusive channel it can call
|
|
|
|
* dma_request_channel(). Once a channel is allocated a reference is taken
|
|
|
|
* against its corresponding driver to disable removal.
|
|
|
|
*
|
2006-05-24 00:18:44 +00:00
|
|
|
* Each device has a channels list, which runs unlocked but is never modified
|
|
|
|
* once the device is registered, it's just setup by the driver.
|
|
|
|
*
|
2018-06-14 15:34:32 +00:00
|
|
|
* See Documentation/driver-api/dmaengine for more details
|
2006-05-24 00:18:44 +00:00
|
|
|
*/
|
|
|
|
|
2012-07-18 16:51:28 +00:00
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
#include <linux/platform_device.h>
|
2011-06-16 11:01:34 +00:00
|
|
|
#include <linux/dma-mapping.h>
|
2006-05-24 00:18:44 +00:00
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/module.h>
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
#include <linux/mm.h>
|
2006-05-24 00:18:44 +00:00
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/dmaengine.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/mutex.h>
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
#include <linux/jiffies.h>
|
2009-01-06 18:38:14 +00:00
|
|
|
#include <linux/rculist.h>
|
2009-01-06 18:38:21 +00:00
|
|
|
#include <linux/idr.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
2013-04-09 11:05:44 +00:00
|
|
|
#include <linux/acpi.h>
|
|
|
|
#include <linux/acpi_dma.h>
|
2012-09-14 22:41:57 +00:00
|
|
|
#include <linux/of_dma.h>
|
2013-10-18 17:35:24 +00:00
|
|
|
#include <linux/mempool.h>
|
2019-03-05 23:42:58 +00:00
|
|
|
#include <linux/numa.h>
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2020-04-29 12:21:50 +00:00
|
|
|
#include "dmaengine.h"
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
static DEFINE_MUTEX(dma_list_mutex);
|
2016-12-15 16:57:51 +00:00
|
|
|
static DEFINE_IDA(dma_ida);
|
2006-05-24 00:18:44 +00:00
|
|
|
static LIST_HEAD(dma_device_list);
|
2009-01-06 18:38:14 +00:00
|
|
|
static long dmaengine_ref_count;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2020-03-06 14:28:37 +00:00
|
|
|
/* --- debugfs implementation --- */
|
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
|
|
#include <linux/debugfs.h>
|
|
|
|
|
2020-03-06 14:28:39 +00:00
|
|
|
static struct dentry *rootdir;
|
|
|
|
|
|
|
|
static void dmaengine_debug_register(struct dma_device *dma_dev)
|
|
|
|
{
|
|
|
|
dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),
|
|
|
|
rootdir);
|
|
|
|
if (IS_ERR(dma_dev->dbg_dev_root))
|
|
|
|
dma_dev->dbg_dev_root = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dmaengine_debug_unregister(struct dma_device *dma_dev)
|
|
|
|
{
|
|
|
|
debugfs_remove_recursive(dma_dev->dbg_dev_root);
|
|
|
|
dma_dev->dbg_dev_root = NULL;
|
|
|
|
}
|
|
|
|
|
2020-03-06 14:28:37 +00:00
|
|
|
static void dmaengine_dbg_summary_show(struct seq_file *s,
|
|
|
|
struct dma_device *dma_dev)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
list_for_each_entry(chan, &dma_dev->channels, device_node) {
|
|
|
|
if (chan->client_count) {
|
|
|
|
seq_printf(s, " %-13s| %s", dma_chan_name(chan),
|
|
|
|
chan->dbg_client_name ?: "in-use");
|
|
|
|
|
|
|
|
if (chan->router)
|
|
|
|
seq_printf(s, " (via router: %s)\n",
|
|
|
|
dev_name(chan->router->dev));
|
|
|
|
else
|
|
|
|
seq_puts(s, "\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int dmaengine_summary_show(struct seq_file *s, void *data)
|
|
|
|
{
|
|
|
|
struct dma_device *dma_dev = NULL;
|
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
list_for_each_entry(dma_dev, &dma_device_list, global_node) {
|
|
|
|
seq_printf(s, "dma%d (%s): number of channels: %u\n",
|
|
|
|
dma_dev->dev_id, dev_name(dma_dev->dev),
|
|
|
|
dma_dev->chancnt);
|
|
|
|
|
|
|
|
if (dma_dev->dbg_summary_show)
|
|
|
|
dma_dev->dbg_summary_show(s, dma_dev);
|
|
|
|
else
|
|
|
|
dmaengine_dbg_summary_show(s, dma_dev);
|
|
|
|
|
|
|
|
if (!list_is_last(&dma_dev->global_node, &dma_device_list))
|
|
|
|
seq_puts(s, "\n");
|
|
|
|
}
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
DEFINE_SHOW_ATTRIBUTE(dmaengine_summary);
|
|
|
|
|
|
|
|
static void __init dmaengine_debugfs_init(void)
|
|
|
|
{
|
2020-03-06 14:28:39 +00:00
|
|
|
rootdir = debugfs_create_dir("dmaengine", NULL);
|
2020-03-06 14:28:37 +00:00
|
|
|
|
|
|
|
/* /sys/kernel/debug/dmaengine/summary */
|
|
|
|
debugfs_create_file("summary", 0444, rootdir, NULL,
|
|
|
|
&dmaengine_summary_fops);
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
static inline void dmaengine_debugfs_init(void) { }
|
2020-03-06 14:28:39 +00:00
|
|
|
static inline int dmaengine_debug_register(struct dma_device *dma_dev)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }
|
2020-03-06 14:28:37 +00:00
|
|
|
#endif /* DEBUG_FS */
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
/* --- sysfs implementation --- */
|
|
|
|
|
2020-01-17 15:30:56 +00:00
|
|
|
#define DMA_SLAVE_NAME "slave"
|
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
/**
|
2019-06-07 11:30:39 +00:00
|
|
|
* dev_to_dma_chan - convert a device pointer to its sysfs container object
|
2020-04-29 12:21:51 +00:00
|
|
|
* @dev: device node
|
2009-01-06 18:38:21 +00:00
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* Must be called under dma_list_mutex.
|
2009-01-06 18:38:21 +00:00
|
|
|
*/
|
|
|
|
static struct dma_chan *dev_to_dma_chan(struct device *dev)
|
|
|
|
{
|
|
|
|
struct dma_chan_dev *chan_dev;
|
|
|
|
|
|
|
|
chan_dev = container_of(dev, typeof(*chan_dev), device);
|
|
|
|
return chan_dev->chan;
|
|
|
|
}
|
|
|
|
|
2013-07-24 22:05:08 +00:00
|
|
|
static ssize_t memcpy_count_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
2006-05-24 00:18:44 +00:00
|
|
|
{
|
2009-01-06 18:38:21 +00:00
|
|
|
struct dma_chan *chan;
|
2006-05-24 00:18:44 +00:00
|
|
|
unsigned long count = 0;
|
|
|
|
int i;
|
2009-01-06 18:38:21 +00:00
|
|
|
int err;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan) {
|
|
|
|
for_each_possible_cpu(i)
|
|
|
|
count += per_cpu_ptr(chan->local, i)->memcpy_count;
|
|
|
|
err = sprintf(buf, "%lu\n", count);
|
|
|
|
} else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
return err;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2013-07-24 22:05:08 +00:00
|
|
|
static DEVICE_ATTR_RO(memcpy_count);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2013-07-24 22:05:08 +00:00
|
|
|
static ssize_t bytes_transferred_show(struct device *dev,
|
|
|
|
struct device_attribute *attr, char *buf)
|
2006-05-24 00:18:44 +00:00
|
|
|
{
|
2009-01-06 18:38:21 +00:00
|
|
|
struct dma_chan *chan;
|
2006-05-24 00:18:44 +00:00
|
|
|
unsigned long count = 0;
|
|
|
|
int i;
|
2009-01-06 18:38:21 +00:00
|
|
|
int err;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan) {
|
|
|
|
for_each_possible_cpu(i)
|
|
|
|
count += per_cpu_ptr(chan->local, i)->bytes_transferred;
|
|
|
|
err = sprintf(buf, "%lu\n", count);
|
|
|
|
} else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
return err;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2013-07-24 22:05:08 +00:00
|
|
|
static DEVICE_ATTR_RO(bytes_transferred);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2013-07-24 22:05:08 +00:00
|
|
|
static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
|
|
|
|
char *buf)
|
2006-05-24 00:18:44 +00:00
|
|
|
{
|
2009-01-06 18:38:21 +00:00
|
|
|
struct dma_chan *chan;
|
|
|
|
int err;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan)
|
|
|
|
err = sprintf(buf, "%d\n", chan->client_count);
|
|
|
|
else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
return err;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2013-07-24 22:05:08 +00:00
|
|
|
static DEVICE_ATTR_RO(in_use);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2013-07-24 22:05:08 +00:00
|
|
|
static struct attribute *dma_dev_attrs[] = {
|
|
|
|
&dev_attr_memcpy_count.attr,
|
|
|
|
&dev_attr_bytes_transferred.attr,
|
|
|
|
&dev_attr_in_use.attr,
|
|
|
|
NULL,
|
2006-05-24 00:18:44 +00:00
|
|
|
};
|
2013-07-24 22:05:08 +00:00
|
|
|
ATTRIBUTE_GROUPS(dma_dev);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
static void chan_dev_release(struct device *dev)
|
|
|
|
{
|
|
|
|
struct dma_chan_dev *chan_dev;
|
|
|
|
|
|
|
|
chan_dev = container_of(dev, typeof(*chan_dev), device);
|
|
|
|
kfree(chan_dev);
|
|
|
|
}
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
static struct class dma_devclass = {
|
2007-09-25 00:03:03 +00:00
|
|
|
.name = "dma",
|
2013-07-24 22:05:08 +00:00
|
|
|
.dev_groups = dma_dev_groups,
|
2009-01-06 18:38:21 +00:00
|
|
|
.dev_release = chan_dev_release,
|
2006-05-24 00:18:44 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* --- client and device registration --- */
|
|
|
|
|
2020-04-29 12:21:51 +00:00
|
|
|
/* enable iteration over all operation types */
|
2019-12-16 19:01:18 +00:00
|
|
|
static dma_cap_mask_t dma_cap_mask_all;
|
|
|
|
|
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* struct dma_chan_tbl_ent - tracks channel allocations per core/operation
|
|
|
|
* @chan: associated channel for this entry
|
2019-12-16 19:01:18 +00:00
|
|
|
*/
|
|
|
|
struct dma_chan_tbl_ent {
|
|
|
|
struct dma_chan *chan;
|
|
|
|
};
|
|
|
|
|
2020-04-29 12:21:51 +00:00
|
|
|
/* percpu lookup table for memory-to-memory offload providers */
|
2019-12-16 19:01:18 +00:00
|
|
|
static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
|
|
|
|
|
|
|
|
static int __init dma_channel_table_init(void)
|
|
|
|
{
|
|
|
|
enum dma_transaction_type cap;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
|
|
|
|
|
|
|
|
/* 'interrupt', 'private', and 'slave' are channel capabilities,
|
|
|
|
* but are not associated with an operation so they do not need
|
|
|
|
* an entry in the channel_table
|
|
|
|
*/
|
|
|
|
clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
|
|
|
|
clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
|
|
|
|
clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
|
|
|
|
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all) {
|
|
|
|
channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
|
|
|
|
if (!channel_table[cap]) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (err) {
|
2019-12-24 04:56:14 +00:00
|
|
|
pr_err("dmaengine dma_channel_table_init failure: %d\n", err);
|
2019-12-16 19:01:18 +00:00
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
free_percpu(channel_table[cap]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
arch_initcall(dma_channel_table_init);
|
|
|
|
|
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
|
|
|
|
* @chan: DMA channel to test
|
|
|
|
* @cpu: CPU index which the channel should be close to
|
|
|
|
*
|
|
|
|
* Returns true if the channel is in the same NUMA-node as the CPU.
|
2019-12-16 19:01:18 +00:00
|
|
|
*/
|
|
|
|
static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
|
|
|
|
{
|
|
|
|
int node = dev_to_node(chan->device->dev);
|
|
|
|
return node == NUMA_NO_NODE ||
|
|
|
|
cpumask_test_cpu(cpu, cpumask_of_node(node));
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* min_chan - finds the channel with min count and in the same NUMA-node as the CPU
|
|
|
|
* @cap: capability to match
|
|
|
|
* @cpu: CPU index which the channel should be close to
|
2019-12-16 19:01:18 +00:00
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* If some channels are close to the given CPU, the one with the lowest
|
|
|
|
* reference count is returned. Otherwise, CPU is ignored and only the
|
2019-12-16 19:01:18 +00:00
|
|
|
* reference count is taken into account.
|
2020-04-29 12:21:51 +00:00
|
|
|
*
|
2019-12-16 19:01:18 +00:00
|
|
|
* Must be called under dma_list_mutex.
|
|
|
|
*/
|
|
|
|
static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
struct dma_chan *min = NULL;
|
|
|
|
struct dma_chan *localmin = NULL;
|
|
|
|
|
|
|
|
list_for_each_entry(device, &dma_device_list, global_node) {
|
|
|
|
if (!dma_has_cap(cap, device->cap_mask) ||
|
|
|
|
dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
if (!chan->client_count)
|
|
|
|
continue;
|
|
|
|
if (!min || chan->table_count < min->table_count)
|
|
|
|
min = chan;
|
|
|
|
|
|
|
|
if (dma_chan_is_local(chan, cpu))
|
|
|
|
if (!localmin ||
|
|
|
|
chan->table_count < localmin->table_count)
|
|
|
|
localmin = chan;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
chan = localmin ? localmin : min;
|
|
|
|
|
|
|
|
if (chan)
|
|
|
|
chan->table_count++;
|
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_channel_rebalance - redistribute the available channels
|
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* Optimize for CPU isolation (each CPU gets a dedicated channel for an
|
|
|
|
* operation type) in the SMP case, and operation isolation (avoid
|
|
|
|
* multi-tasking channels) in the non-SMP case.
|
|
|
|
*
|
|
|
|
* Must be called under dma_list_mutex.
|
2019-12-16 19:01:18 +00:00
|
|
|
*/
|
|
|
|
static void dma_channel_rebalance(void)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
struct dma_device *device;
|
|
|
|
int cpu;
|
|
|
|
int cap;
|
|
|
|
|
|
|
|
/* undo the last distribution */
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
|
|
|
|
|
|
|
|
list_for_each_entry(device, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
chan->table_count = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* don't populate the channel_table if no clients are available */
|
|
|
|
if (!dmaengine_ref_count)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* redistribute available channels */
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
chan = min_chan(cap, cpu);
|
|
|
|
per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-21 09:33:09 +00:00
|
|
|
static int dma_device_satisfies_mask(struct dma_device *device,
|
|
|
|
const dma_cap_mask_t *want)
|
2007-07-09 18:56:42 +00:00
|
|
|
{
|
|
|
|
dma_cap_mask_t has;
|
|
|
|
|
2009-01-06 18:38:15 +00:00
|
|
|
bitmap_and(has.bits, want->bits, device->cap_mask.bits,
|
2007-07-09 18:56:42 +00:00
|
|
|
DMA_TX_TYPE_END);
|
|
|
|
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
|
|
|
|
}
|
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
static struct module *dma_chan_to_owner(struct dma_chan *chan)
|
|
|
|
{
|
2019-12-16 19:01:16 +00:00
|
|
|
return chan->device->owner;
|
2009-01-06 18:38:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* balance_ref_count - catch up the channel reference count
|
2020-04-29 12:21:51 +00:00
|
|
|
* @chan: channel to balance ->client_count versus dmaengine_ref_count
|
2009-01-06 18:38:14 +00:00
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* Must be called under dma_list_mutex.
|
2009-01-06 18:38:14 +00:00
|
|
|
*/
|
|
|
|
static void balance_ref_count(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct module *owner = dma_chan_to_owner(chan);
|
|
|
|
|
|
|
|
while (chan->client_count < dmaengine_ref_count) {
|
|
|
|
__module_get(owner);
|
|
|
|
chan->client_count++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-16 19:01:19 +00:00
|
|
|
static void dma_device_release(struct kref *ref)
|
|
|
|
{
|
|
|
|
struct dma_device *device = container_of(ref, struct dma_device, ref);
|
|
|
|
|
|
|
|
list_del_rcu(&device->global_node);
|
|
|
|
dma_channel_rebalance();
|
|
|
|
|
|
|
|
if (device->device_release)
|
|
|
|
device->device_release(device);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dma_device_put(struct dma_device *device)
|
|
|
|
{
|
|
|
|
lockdep_assert_held(&dma_list_mutex);
|
|
|
|
kref_put(&device->ref, dma_device_release);
|
|
|
|
}
|
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* dma_chan_get - try to grab a DMA channel's parent driver module
|
|
|
|
* @chan: channel to grab
|
2009-01-06 18:38:14 +00:00
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* Must be called under dma_list_mutex.
|
2009-01-06 18:38:14 +00:00
|
|
|
*/
|
|
|
|
static int dma_chan_get(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct module *owner = dma_chan_to_owner(chan);
|
2014-11-17 13:41:58 +00:00
|
|
|
int ret;
|
2009-01-06 18:38:14 +00:00
|
|
|
|
2014-11-17 13:41:58 +00:00
|
|
|
/* The channel is already in use, update client count */
|
2009-01-06 18:38:14 +00:00
|
|
|
if (chan->client_count) {
|
|
|
|
__module_get(owner);
|
2014-11-17 13:41:58 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2009-01-06 18:38:14 +00:00
|
|
|
|
2014-11-17 13:41:58 +00:00
|
|
|
if (!try_module_get(owner))
|
|
|
|
return -ENODEV;
|
2009-01-06 18:38:14 +00:00
|
|
|
|
2019-12-16 19:01:19 +00:00
|
|
|
ret = kref_get_unless_zero(&chan->device->ref);
|
|
|
|
if (!ret) {
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto module_put_out;
|
|
|
|
}
|
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
/* allocate upon first client reference */
|
2014-11-17 13:41:59 +00:00
|
|
|
if (chan->device->device_alloc_chan_resources) {
|
|
|
|
ret = chan->device->device_alloc_chan_resources(chan);
|
|
|
|
if (ret < 0)
|
|
|
|
goto err_out;
|
|
|
|
}
|
2009-01-06 18:38:14 +00:00
|
|
|
|
2014-11-17 13:41:58 +00:00
|
|
|
if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
|
|
|
|
balance_ref_count(chan);
|
|
|
|
|
|
|
|
out:
|
|
|
|
chan->client_count++;
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_out:
|
2019-12-16 19:01:19 +00:00
|
|
|
dma_device_put(chan->device);
|
|
|
|
module_put_out:
|
2014-11-17 13:41:58 +00:00
|
|
|
module_put(owner);
|
|
|
|
return ret;
|
2009-01-06 18:38:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* dma_chan_put - drop a reference to a DMA channel's parent driver module
|
|
|
|
* @chan: channel to release
|
2009-01-06 18:38:14 +00:00
|
|
|
*
|
2020-04-29 12:21:51 +00:00
|
|
|
* Must be called under dma_list_mutex.
|
2009-01-06 18:38:14 +00:00
|
|
|
*/
|
|
|
|
static void dma_chan_put(struct dma_chan *chan)
|
|
|
|
{
|
2014-11-17 13:41:59 +00:00
|
|
|
/* This channel is not in use, bail out */
|
2009-01-06 18:38:14 +00:00
|
|
|
if (!chan->client_count)
|
2014-11-17 13:41:59 +00:00
|
|
|
return;
|
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
chan->client_count--;
|
2014-11-17 13:41:59 +00:00
|
|
|
|
|
|
|
/* This channel is not in use anymore, free it */
|
2015-10-20 09:46:28 +00:00
|
|
|
if (!chan->client_count && chan->device->device_free_chan_resources) {
|
|
|
|
/* Make sure all operations have completed */
|
|
|
|
dmaengine_synchronize(chan);
|
2009-01-06 18:38:14 +00:00
|
|
|
chan->device->device_free_chan_resources(chan);
|
2015-10-20 09:46:28 +00:00
|
|
|
}
|
2015-04-09 09:35:47 +00:00
|
|
|
|
|
|
|
/* If the channel is used via a DMA request router, free the mapping */
|
|
|
|
if (chan->router && chan->router->route_free) {
|
|
|
|
chan->router->route_free(chan->router->dev, chan->route_data);
|
|
|
|
chan->router = NULL;
|
|
|
|
chan->route_data = NULL;
|
|
|
|
}
|
2019-12-24 04:52:15 +00:00
|
|
|
|
|
|
|
dma_device_put(chan->device);
|
|
|
|
module_put(dma_chan_to_owner(chan));
|
2009-01-06 18:38:14 +00:00
|
|
|
}
|
|
|
|
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
|
|
|
|
{
|
|
|
|
enum dma_status status;
|
|
|
|
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
|
|
|
|
|
|
|
|
dma_async_issue_pending(chan);
|
|
|
|
do {
|
|
|
|
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
|
|
|
|
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_err(chan->device->dev, "%s: timeout!\n", __func__);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
return DMA_ERROR;
|
|
|
|
}
|
2012-11-08 10:02:07 +00:00
|
|
|
if (status != DMA_IN_PROGRESS)
|
|
|
|
break;
|
|
|
|
cpu_relax();
|
|
|
|
} while (1);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_sync_wait);
|
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
/**
|
|
|
|
* dma_find_channel - find a channel to carry out the operation
|
2020-04-29 12:21:51 +00:00
|
|
|
* @tx_type: transaction type
|
2009-01-06 18:38:14 +00:00
|
|
|
*/
|
|
|
|
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
|
|
|
|
{
|
2009-10-03 10:48:23 +00:00
|
|
|
return this_cpu_read(channel_table[tx_type]->chan);
|
2009-01-06 18:38:14 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_find_channel);
|
2012-04-04 23:10:46 +00:00
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
/**
|
|
|
|
* dma_issue_pending_all - flush all pending operations across all channels
|
|
|
|
*/
|
|
|
|
void dma_issue_pending_all(void)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2009-01-06 18:38:15 +00:00
|
|
|
list_for_each_entry_rcu(device, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 18:38:14 +00:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
if (chan->client_count)
|
|
|
|
device->device_issue_pending(chan);
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
2009-01-06 18:38:14 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_issue_pending_all);
|
|
|
|
|
2014-10-28 22:30:58 +00:00
|
|
|
int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
|
|
|
|
if (!chan || !caps)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
device = chan->device;
|
|
|
|
|
|
|
|
/* check if the channel supports slave transactions */
|
2016-05-10 17:43:34 +00:00
|
|
|
if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
|
|
|
|
test_bit(DMA_CYCLIC, device->cap_mask.bits)))
|
2014-10-28 22:30:58 +00:00
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Check whether it reports it uses the generic slave
|
|
|
|
* capabilities, if not, that means it doesn't support any
|
|
|
|
* kind of slave capabilities reporting.
|
|
|
|
*/
|
|
|
|
if (!device->directions)
|
|
|
|
return -ENXIO;
|
|
|
|
|
|
|
|
caps->src_addr_widths = device->src_addr_widths;
|
|
|
|
caps->dst_addr_widths = device->dst_addr_widths;
|
|
|
|
caps->directions = device->directions;
|
2020-07-23 00:58:41 +00:00
|
|
|
caps->min_burst = device->min_burst;
|
2016-01-22 11:06:50 +00:00
|
|
|
caps->max_burst = device->max_burst;
|
2014-10-28 22:30:58 +00:00
|
|
|
caps->residue_granularity = device->residue_granularity;
|
2015-10-13 19:54:29 +00:00
|
|
|
caps->descriptor_reuse = device->descriptor_reuse;
|
2018-07-02 13:08:10 +00:00
|
|
|
caps->cmd_pause = !!device->device_pause;
|
|
|
|
caps->cmd_resume = !!device->device_resume;
|
2014-10-28 22:30:58 +00:00
|
|
|
caps->cmd_terminate = !!device->device_terminate_all;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_get_slave_caps);
|
|
|
|
|
2013-03-25 12:23:52 +00:00
|
|
|
static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
|
|
|
|
struct dma_device *dev,
|
2009-01-06 18:38:21 +00:00
|
|
|
dma_filter_fn fn, void *fn_param)
|
2009-01-06 18:38:15 +00:00
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
2020-01-21 09:33:09 +00:00
|
|
|
if (mask && !dma_device_satisfies_mask(dev, mask)) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
|
2009-01-06 18:38:15 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
/* devices with multiple channels need special handling as we need to
|
|
|
|
* ensure that all channels are either private or public.
|
|
|
|
*/
|
|
|
|
if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
|
|
|
|
list_for_each_entry(chan, &dev->channels, device_node) {
|
|
|
|
/* some channels are already publicly allocated */
|
|
|
|
if (chan->client_count)
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(chan, &dev->channels, device_node) {
|
|
|
|
if (chan->client_count) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(dev->dev, "%s: %s busy\n",
|
2009-01-06 18:38:21 +00:00
|
|
|
__func__, dma_chan_name(chan));
|
2009-01-06 18:38:15 +00:00
|
|
|
continue;
|
|
|
|
}
|
2009-01-06 18:38:21 +00:00
|
|
|
if (fn && !fn(chan, fn_param)) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(dev->dev, "%s: %s filter said false\n",
|
2009-01-06 18:38:21 +00:00
|
|
|
__func__, dma_chan_name(chan));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return chan;
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
|
|
|
|
2009-01-06 18:38:21 +00:00
|
|
|
return NULL;
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
|
|
|
|
2015-12-14 20:47:39 +00:00
|
|
|
static struct dma_chan *find_candidate(struct dma_device *device,
|
|
|
|
const dma_cap_mask_t *mask,
|
|
|
|
dma_filter_fn fn, void *fn_param)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (chan) {
|
|
|
|
/* Found a suitable channel, try to grab, prep, and return it.
|
|
|
|
* We first set DMA_PRIVATE to disable balance_ref_count as this
|
|
|
|
* channel will not be published in the general-purpose
|
|
|
|
* allocator
|
|
|
|
*/
|
|
|
|
dma_cap_set(DMA_PRIVATE, device->cap_mask);
|
|
|
|
device->privatecnt++;
|
|
|
|
err = dma_chan_get(chan);
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
if (err == -ENODEV) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(device->dev, "%s: %s module removed\n",
|
|
|
|
__func__, dma_chan_name(chan));
|
2015-12-14 20:47:39 +00:00
|
|
|
list_del_rcu(&device->global_node);
|
|
|
|
} else
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(device->dev,
|
|
|
|
"%s: failed to get %s: (%d)\n",
|
2015-12-14 20:47:39 +00:00
|
|
|
__func__, dma_chan_name(chan), err);
|
|
|
|
|
|
|
|
if (--device->privatecnt == 0)
|
|
|
|
dma_cap_clear(DMA_PRIVATE, device->cap_mask);
|
|
|
|
|
|
|
|
chan = ERR_PTR(err);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return chan ? chan : ERR_PTR(-EPROBE_DEFER);
|
|
|
|
}
|
|
|
|
|
2009-01-06 18:38:15 +00:00
|
|
|
/**
|
2015-06-01 21:53:43 +00:00
|
|
|
* dma_get_slave_channel - try to get specific channel exclusively
|
2020-04-29 12:21:51 +00:00
|
|
|
* @chan: target channel
|
2013-06-28 12:39:12 +00:00
|
|
|
*/
|
|
|
|
struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
int err = -EBUSY;
|
|
|
|
|
|
|
|
/* lock against __dma_request_channel */
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
|
2013-08-19 05:17:26 +00:00
|
|
|
if (chan->client_count == 0) {
|
2015-09-24 09:03:35 +00:00
|
|
|
struct dma_device *device = chan->device;
|
|
|
|
|
|
|
|
dma_cap_set(DMA_PRIVATE, device->cap_mask);
|
|
|
|
device->privatecnt++;
|
2013-06-28 12:39:12 +00:00
|
|
|
err = dma_chan_get(chan);
|
2015-09-24 09:03:35 +00:00
|
|
|
if (err) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(chan->device->dev,
|
|
|
|
"%s: failed to get %s: (%d)\n",
|
2013-08-19 05:17:26 +00:00
|
|
|
__func__, dma_chan_name(chan), err);
|
2015-09-24 09:03:35 +00:00
|
|
|
chan = NULL;
|
|
|
|
if (--device->privatecnt == 0)
|
|
|
|
dma_cap_clear(DMA_PRIVATE, device->cap_mask);
|
|
|
|
}
|
2013-08-19 05:17:26 +00:00
|
|
|
} else
|
2013-06-28 12:39:12 +00:00
|
|
|
chan = NULL;
|
|
|
|
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_get_slave_channel);
|
|
|
|
|
2013-11-26 19:40:51 +00:00
|
|
|
struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
|
|
|
|
{
|
|
|
|
dma_cap_mask_t mask;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
dma_cap_zero(mask);
|
|
|
|
dma_cap_set(DMA_SLAVE, mask);
|
|
|
|
|
|
|
|
/* lock against __dma_request_channel */
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
|
2015-12-14 20:47:39 +00:00
|
|
|
chan = find_candidate(device, &mask, NULL, NULL);
|
2013-11-26 19:40:51 +00:00
|
|
|
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
2015-12-14 20:47:39 +00:00
|
|
|
return IS_ERR(chan) ? NULL : chan;
|
2013-11-26 19:40:51 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
|
|
|
|
|
2009-01-06 18:38:15 +00:00
|
|
|
/**
|
2013-08-14 16:35:03 +00:00
|
|
|
* __dma_request_channel - try to allocate an exclusive channel
|
2020-04-29 12:21:51 +00:00
|
|
|
* @mask: capabilities that the channel must satisfy
|
|
|
|
* @fn: optional callback to disposition available channels
|
|
|
|
* @fn_param: opaque parameter to pass to dma_filter_fn()
|
|
|
|
* @np: device node to look for DMA channels
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
*
|
|
|
|
* Returns pointer to appropriate DMA channel on success or NULL.
|
2009-01-06 18:38:15 +00:00
|
|
|
*/
|
2013-03-25 12:23:52 +00:00
|
|
|
struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
|
2019-05-20 11:32:14 +00:00
|
|
|
dma_filter_fn fn, void *fn_param,
|
|
|
|
struct device_node *np)
|
2009-01-06 18:38:15 +00:00
|
|
|
{
|
|
|
|
struct dma_device *device, *_d;
|
|
|
|
struct dma_chan *chan = NULL;
|
|
|
|
|
|
|
|
/* Find a channel */
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
|
2019-05-20 11:32:14 +00:00
|
|
|
/* Finds a DMA controller with matching device node */
|
|
|
|
if (np && device->dev->of_node && np != device->dev->of_node)
|
|
|
|
continue;
|
|
|
|
|
2015-12-14 20:47:39 +00:00
|
|
|
chan = find_candidate(device, mask, fn, fn_param);
|
|
|
|
if (!IS_ERR(chan))
|
|
|
|
break;
|
2009-01-06 18:38:15 +00:00
|
|
|
|
2015-12-14 20:47:39 +00:00
|
|
|
chan = NULL;
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
2016-04-07 13:49:43 +00:00
|
|
|
pr_debug("%s: %s (%s)\n",
|
2012-07-18 16:51:28 +00:00
|
|
|
__func__,
|
|
|
|
chan ? "success" : "fail",
|
2009-01-06 18:38:21 +00:00
|
|
|
chan ? dma_chan_name(chan) : NULL);
|
2009-01-06 18:38:15 +00:00
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__dma_request_channel);
|
|
|
|
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
|
|
|
|
const char *name,
|
|
|
|
struct device *dev)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!device->filter.mapcnt)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
for (i = 0; i < device->filter.mapcnt; i++) {
|
|
|
|
const struct dma_slave_map *map = &device->filter.map[i];
|
|
|
|
|
|
|
|
if (!strcmp(map->devname, dev_name(dev)) &&
|
|
|
|
!strcmp(map->slave, name))
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2012-09-14 22:41:57 +00:00
|
|
|
/**
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
* dma_request_chan - try to allocate an exclusive slave channel
|
2012-09-14 22:41:57 +00:00
|
|
|
* @dev: pointer to client device structure
|
|
|
|
* @name: slave channel name
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
*
|
|
|
|
* Returns pointer to appropriate DMA channel on success or an error pointer.
|
2012-09-14 22:41:57 +00:00
|
|
|
*/
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
struct dma_chan *dma_request_chan(struct device *dev, const char *name)
|
2012-09-14 22:41:57 +00:00
|
|
|
{
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
struct dma_device *d, *_d;
|
|
|
|
struct dma_chan *chan = NULL;
|
|
|
|
|
2012-09-14 22:41:57 +00:00
|
|
|
/* If device-tree is present get slave info from here */
|
|
|
|
if (dev->of_node)
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
chan = of_dma_request_slave_channel(dev->of_node, name);
|
2012-09-14 22:41:57 +00:00
|
|
|
|
2013-04-09 11:05:44 +00:00
|
|
|
/* If device was enumerated by ACPI get slave info from here */
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
if (has_acpi_companion(dev) && !chan)
|
|
|
|
chan = acpi_dma_request_slave_chan_by_name(dev, name);
|
|
|
|
|
2020-01-17 15:30:56 +00:00
|
|
|
if (PTR_ERR(chan) == -EPROBE_DEFER)
|
|
|
|
return chan;
|
|
|
|
|
|
|
|
if (!IS_ERR_OR_NULL(chan))
|
|
|
|
goto found;
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
|
|
|
|
/* Try to find the channel via the DMA filter map(s) */
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
|
|
|
|
dma_cap_mask_t mask;
|
|
|
|
const struct dma_slave_map *map = dma_filter_match(d, name, dev);
|
2013-04-09 11:05:44 +00:00
|
|
|
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
if (!map)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
dma_cap_zero(mask);
|
|
|
|
dma_cap_set(DMA_SLAVE, mask);
|
2013-04-09 11:05:44 +00:00
|
|
|
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
chan = find_candidate(d, &mask, d->filter.fn, map->param);
|
|
|
|
if (!IS_ERR(chan))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
2020-01-31 09:38:58 +00:00
|
|
|
if (IS_ERR_OR_NULL(chan))
|
|
|
|
return chan ? chan : ERR_PTR(-EPROBE_DEFER);
|
2020-01-17 15:30:56 +00:00
|
|
|
|
|
|
|
found:
|
2020-03-06 14:28:37 +00:00
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
|
|
chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev),
|
|
|
|
name);
|
|
|
|
#endif
|
|
|
|
|
2020-01-17 15:30:56 +00:00
|
|
|
chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);
|
|
|
|
if (!chan->name)
|
2020-01-31 09:38:58 +00:00
|
|
|
return chan;
|
|
|
|
chan->slave = dev;
|
2020-01-17 15:30:56 +00:00
|
|
|
|
|
|
|
if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,
|
|
|
|
DMA_SLAVE_NAME))
|
2020-01-31 09:38:58 +00:00
|
|
|
dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
|
2020-01-17 15:30:56 +00:00
|
|
|
if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))
|
2020-01-31 09:38:58 +00:00
|
|
|
dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);
|
|
|
|
|
2020-01-17 15:30:56 +00:00
|
|
|
return chan;
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
}
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
EXPORT_SYMBOL_GPL(dma_request_chan);
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_request_slave_channel - try to allocate an exclusive slave channel
|
|
|
|
* @dev: pointer to client device structure
|
|
|
|
* @name: slave channel name
|
|
|
|
*
|
|
|
|
* Returns pointer to appropriate DMA channel on success or NULL.
|
|
|
|
*/
|
|
|
|
struct dma_chan *dma_request_slave_channel(struct device *dev,
|
|
|
|
const char *name)
|
|
|
|
{
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
struct dma_chan *ch = dma_request_chan(dev, name);
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
if (IS_ERR(ch))
|
|
|
|
return NULL;
|
2015-08-07 10:26:47 +00:00
|
|
|
|
dma: add channel request API that supports deferred probe
dma_request_slave_channel() simply returns NULL whenever DMA channel
lookup fails. Lookup could fail for two distinct reasons:
a) No DMA specification exists for the channel name.
This includes situations where no DMA specifications exist at all, or
other general lookup problems.
b) A DMA specification does exist, yet the driver for that channel is not
yet registered.
Case (b) should trigger deferred probe in client drivers. However, since
they have no way to differentiate the two situations, it cannot.
Implement new function dma_request_slave_channel_reason(), which performs
identically to dma_request_slave_channel(), except that it returns an
error-pointer rather than NULL, which allows callers to detect when
deferred probe should occur.
Eventually, all drivers should be converted to this new API, the old API
removed, and the new API renamed to the more desirable name. This patch
doesn't convert the existing API and all drivers in one go, since some
drivers call dma_request_slave_channel() then dma_request_channel() if
that fails. That would require either modifying dma_request_channel() in
the same way, or adding extra error-handling code to all affected
drivers, and there are close to 100 drivers using the other API, rather
than just the 15-20 or so that use dma_request_slave_channel(), which
might be tenable in a single patch.
acpi_dma_request_slave_chan_by_name() doesn't currently implement
deferred probe. It should, but this will be addressed later.
Acked-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Stephen Warren <swarren@nvidia.com>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2013-11-26 17:04:22 +00:00
|
|
|
return ch;
|
2012-09-14 22:41:57 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_request_slave_channel);
|
|
|
|
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
/**
|
|
|
|
* dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
|
2020-04-29 12:21:51 +00:00
|
|
|
* @mask: capabilities that the channel must satisfy
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
*
|
|
|
|
* Returns pointer to appropriate DMA channel on success or an error pointer.
|
|
|
|
*/
|
|
|
|
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
if (!mask)
|
|
|
|
return ERR_PTR(-ENODEV);
|
|
|
|
|
2019-05-20 11:32:14 +00:00
|
|
|
chan = __dma_request_channel(mask, NULL, NULL, NULL);
|
2018-07-18 09:29:57 +00:00
|
|
|
if (!chan) {
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
if (list_empty(&dma_device_list))
|
|
|
|
chan = ERR_PTR(-EPROBE_DEFER);
|
|
|
|
else
|
|
|
|
chan = ERR_PTR(-ENODEV);
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
dmaengine: core: Introduce new, universal API to request a channel
The two API function can cover most, if not all current APIs used to
request a channel. With minimal effort dmaengine drivers, platforms and
dmaengine user drivers can be converted to use the two function.
struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask);
To request any channel matching with the requested capabilities, can be
used to request channel for memcpy, memset, xor, etc where no hardware
synchronization is needed.
struct dma_chan *dma_request_chan(struct device *dev, const char *name);
To request a slave channel. The dma_request_chan() will try to find the
channel via DT, ACPI or in case if the kernel booted in non DT/ACPI mode
it will use a filter lookup table and retrieves the needed information from
the dma_slave_map provided by the DMA drivers.
This legacy mode needs changes in platform code, in dmaengine drivers and
finally the dmaengine user drivers can be converted:
For each dmaengine driver an array of DMA device, slave and the parameter
for the filter function needs to be added:
static const struct dma_slave_map da830_edma_map[] = {
{ "davinci-mcasp.0", "rx", EDMA_FILTER_PARAM(0, 0) },
{ "davinci-mcasp.0", "tx", EDMA_FILTER_PARAM(0, 1) },
{ "davinci-mcasp.1", "rx", EDMA_FILTER_PARAM(0, 2) },
{ "davinci-mcasp.1", "tx", EDMA_FILTER_PARAM(0, 3) },
{ "davinci-mcasp.2", "rx", EDMA_FILTER_PARAM(0, 4) },
{ "davinci-mcasp.2", "tx", EDMA_FILTER_PARAM(0, 5) },
{ "spi_davinci.0", "rx", EDMA_FILTER_PARAM(0, 14) },
{ "spi_davinci.0", "tx", EDMA_FILTER_PARAM(0, 15) },
{ "da830-mmc.0", "rx", EDMA_FILTER_PARAM(0, 16) },
{ "da830-mmc.0", "tx", EDMA_FILTER_PARAM(0, 17) },
{ "spi_davinci.1", "rx", EDMA_FILTER_PARAM(0, 18) },
{ "spi_davinci.1", "tx", EDMA_FILTER_PARAM(0, 19) },
};
This information is going to be needed by the dmaengine driver, so
modification to the platform_data is needed, and the driver map should be
added to the pdata of the DMA driver:
da8xx_edma0_pdata.slave_map = da830_edma_map;
da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
The DMA driver then needs to configure the needed device -> filter_fn
mapping before it registers with dma_async_device_register() :
ecc->dma_slave.filter_map.map = info->slave_map;
ecc->dma_slave.filter_map.mapcnt = info->slavecnt;
ecc->dma_slave.filter_map.fn = edma_filter_fn;
When neither DT or ACPI lookup is available the dma_request_chan() will
try to match the requester's device name with the filter_map's list of
device names, when a match found it will use the information from the
dma_slave_map to get the channel with the dma_get_channel() internal
function.
Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2015-12-14 20:47:40 +00:00
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);
|
|
|
|
|
2009-01-06 18:38:15 +00:00
|
|
|
void dma_release_channel(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
WARN_ONCE(chan->client_count != 1,
|
|
|
|
"chan reference count %d != 1\n", chan->client_count);
|
|
|
|
dma_chan_put(chan);
|
2009-03-06 11:07:14 +00:00
|
|
|
/* drop PRIVATE cap enabled by __dma_request_channel() */
|
|
|
|
if (--chan->device->privatecnt == 0)
|
|
|
|
dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
|
2020-01-31 09:38:58 +00:00
|
|
|
|
2020-01-17 15:30:56 +00:00
|
|
|
if (chan->slave) {
|
2020-01-31 09:38:58 +00:00
|
|
|
sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
|
2020-01-17 15:30:56 +00:00
|
|
|
sysfs_remove_link(&chan->slave->kobj, chan->name);
|
|
|
|
kfree(chan->name);
|
|
|
|
chan->name = NULL;
|
|
|
|
chan->slave = NULL;
|
|
|
|
}
|
2020-03-06 14:28:37 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_DEBUG_FS
|
|
|
|
kfree(chan->dbg_client_name);
|
|
|
|
chan->dbg_client_name = NULL;
|
|
|
|
#endif
|
2009-01-06 18:38:15 +00:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_release_channel);
|
|
|
|
|
2007-07-09 18:56:42 +00:00
|
|
|
/**
|
2009-01-06 18:38:17 +00:00
|
|
|
* dmaengine_get - register interest in dma_channels
|
2007-07-09 18:56:42 +00:00
|
|
|
*/
|
2009-01-06 18:38:17 +00:00
|
|
|
void dmaengine_get(void)
|
2007-07-09 18:56:42 +00:00
|
|
|
{
|
2009-01-06 18:38:14 +00:00
|
|
|
struct dma_device *device, *_d;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
int err;
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 18:38:14 +00:00
|
|
|
dmaengine_ref_count++;
|
|
|
|
|
|
|
|
/* try to grab channels */
|
2009-01-06 18:38:15 +00:00
|
|
|
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 18:38:14 +00:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
err = dma_chan_get(chan);
|
|
|
|
if (err == -ENODEV) {
|
|
|
|
/* module removed before we could use it */
|
2009-01-06 18:38:14 +00:00
|
|
|
list_del_rcu(&device->global_node);
|
2009-01-06 18:38:14 +00:00
|
|
|
break;
|
|
|
|
} else if (err)
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_dbg(chan->device->dev,
|
|
|
|
"%s: failed to get %s: (%d)\n",
|
|
|
|
__func__, dma_chan_name(chan), err);
|
2009-01-06 18:38:14 +00:00
|
|
|
}
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
2009-01-06 18:38:14 +00:00
|
|
|
|
2009-01-06 18:38:14 +00:00
|
|
|
/* if this is the first reference and there were channels
|
|
|
|
* waiting we need to rebalance to get those channels
|
|
|
|
* incorporated into the channel table
|
|
|
|
*/
|
|
|
|
if (dmaengine_ref_count == 1)
|
|
|
|
dma_channel_rebalance();
|
2006-05-24 00:18:44 +00:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
2009-01-06 18:38:17 +00:00
|
|
|
EXPORT_SYMBOL(dmaengine_get);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
|
|
|
/**
|
2020-04-29 12:21:51 +00:00
|
|
|
* dmaengine_put - let DMA drivers be removed when ref_count == 0
|
2006-05-24 00:18:44 +00:00
|
|
|
*/
|
2009-01-06 18:38:17 +00:00
|
|
|
void dmaengine_put(void)
|
2006-05-24 00:18:44 +00:00
|
|
|
{
|
2019-12-16 19:01:19 +00:00
|
|
|
struct dma_device *device, *_d;
|
2006-05-24 00:18:44 +00:00
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 18:38:14 +00:00
|
|
|
dmaengine_ref_count--;
|
|
|
|
BUG_ON(dmaengine_ref_count < 0);
|
|
|
|
/* drop channel references */
|
2019-12-16 19:01:19 +00:00
|
|
|
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
|
2009-01-06 18:38:15 +00:00
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 18:38:14 +00:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
dma_chan_put(chan);
|
2009-01-06 18:38:15 +00:00
|
|
|
}
|
2006-05-24 00:18:44 +00:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
2009-01-06 18:38:17 +00:00
|
|
|
EXPORT_SYMBOL(dmaengine_put);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2009-09-09 00:42:51 +00:00
|
|
|
static bool device_has_all_tx_types(struct dma_device *device)
|
|
|
|
{
|
|
|
|
/* A device that satisfies this test has channels that will never cause
|
|
|
|
* an async_tx channel switch event as all possible operation types can
|
|
|
|
* be handled.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_ASYNC_TX_DMA
|
|
|
|
if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
|
2016-05-11 17:39:27 +00:00
|
|
|
#if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
|
2009-09-09 00:42:51 +00:00
|
|
|
if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
|
|
|
|
return false;
|
|
|
|
#endif
|
|
|
|
|
2016-05-11 17:39:27 +00:00
|
|
|
#if IS_ENABLED(CONFIG_ASYNC_XOR)
|
2009-09-09 00:42:51 +00:00
|
|
|
if (!dma_has_cap(DMA_XOR, device->cap_mask))
|
|
|
|
return false;
|
2009-11-20 00:10:37 +00:00
|
|
|
|
|
|
|
#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
|
2009-11-20 00:10:25 +00:00
|
|
|
if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
|
|
|
|
return false;
|
2009-09-09 00:42:51 +00:00
|
|
|
#endif
|
2009-11-20 00:10:37 +00:00
|
|
|
#endif
|
2009-09-09 00:42:51 +00:00
|
|
|
|
2016-05-11 17:39:27 +00:00
|
|
|
#if IS_ENABLED(CONFIG_ASYNC_PQ)
|
2009-09-09 00:42:51 +00:00
|
|
|
if (!dma_has_cap(DMA_PQ, device->cap_mask))
|
|
|
|
return false;
|
2009-11-20 00:10:37 +00:00
|
|
|
|
|
|
|
#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
|
2009-11-20 00:10:25 +00:00
|
|
|
if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
|
|
|
|
return false;
|
2009-09-09 00:42:51 +00:00
|
|
|
#endif
|
2009-11-20 00:10:37 +00:00
|
|
|
#endif
|
2009-09-09 00:42:51 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2009-03-25 16:13:23 +00:00
|
|
|
static int get_dma_id(struct dma_device *device)
|
|
|
|
{
|
2018-06-18 19:41:48 +00:00
|
|
|
int rc = ida_alloc(&dma_ida, GFP_KERNEL);
|
2013-02-28 01:04:03 +00:00
|
|
|
|
2018-06-18 19:41:48 +00:00
|
|
|
if (rc < 0)
|
|
|
|
return rc;
|
|
|
|
device->dev_id = rc;
|
|
|
|
return 0;
|
2009-03-25 16:13:23 +00:00
|
|
|
}
|
|
|
|
|
2020-01-21 23:43:47 +00:00
|
|
|
static int __dma_async_device_channel_register(struct dma_device *device,
|
2020-04-13 17:40:12 +00:00
|
|
|
struct dma_chan *chan)
|
2020-01-21 23:43:47 +00:00
|
|
|
{
|
|
|
|
int rc = 0;
|
|
|
|
|
|
|
|
chan->local = alloc_percpu(typeof(*chan->local));
|
|
|
|
if (!chan->local)
|
|
|
|
goto err_out;
|
|
|
|
chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
|
|
|
|
if (!chan->dev) {
|
|
|
|
free_percpu(chan->local);
|
|
|
|
chan->local = NULL;
|
|
|
|
goto err_out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* When the chan_id is a negative value, we are dynamically adding
|
|
|
|
* the channel. Otherwise we are static enumerating.
|
|
|
|
*/
|
2020-04-13 17:40:12 +00:00
|
|
|
mutex_lock(&device->chan_mutex);
|
|
|
|
chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
|
|
|
|
mutex_unlock(&device->chan_mutex);
|
|
|
|
if (chan->chan_id < 0) {
|
|
|
|
pr_err("%s: unable to alloc ida for chan: %d\n",
|
|
|
|
__func__, chan->chan_id);
|
|
|
|
goto err_out;
|
|
|
|
}
|
|
|
|
|
2020-01-21 23:43:47 +00:00
|
|
|
chan->dev->device.class = &dma_devclass;
|
|
|
|
chan->dev->device.parent = device->dev;
|
|
|
|
chan->dev->chan = chan;
|
|
|
|
chan->dev->dev_id = device->dev_id;
|
|
|
|
dev_set_name(&chan->dev->device, "dma%dchan%d",
|
|
|
|
device->dev_id, chan->chan_id);
|
|
|
|
rc = device_register(&chan->dev->device);
|
|
|
|
if (rc)
|
2020-04-13 17:40:12 +00:00
|
|
|
goto err_out_ida;
|
2020-01-21 23:43:47 +00:00
|
|
|
chan->client_count = 0;
|
2020-04-13 17:40:12 +00:00
|
|
|
device->chancnt++;
|
2020-01-21 23:43:47 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
2020-04-13 17:40:12 +00:00
|
|
|
err_out_ida:
|
|
|
|
mutex_lock(&device->chan_mutex);
|
|
|
|
ida_free(&device->chan_ida, chan->chan_id);
|
|
|
|
mutex_unlock(&device->chan_mutex);
|
2020-01-21 23:43:47 +00:00
|
|
|
err_out:
|
|
|
|
free_percpu(chan->local);
|
|
|
|
kfree(chan->dev);
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2020-01-21 23:43:53 +00:00
|
|
|
int dma_async_device_channel_register(struct dma_device *device,
|
|
|
|
struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
2020-04-13 17:40:12 +00:00
|
|
|
rc = __dma_async_device_channel_register(device, chan);
|
2020-01-21 23:43:53 +00:00
|
|
|
if (rc < 0)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
dma_channel_rebalance();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_async_device_channel_register);
|
|
|
|
|
2020-01-21 23:43:47 +00:00
|
|
|
static void __dma_async_device_channel_unregister(struct dma_device *device,
|
|
|
|
struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
WARN_ONCE(!device->device_release && chan->client_count,
|
|
|
|
"%s called while %d clients hold a reference\n",
|
|
|
|
__func__, chan->client_count);
|
|
|
|
mutex_lock(&dma_list_mutex);
|
2020-01-21 23:43:53 +00:00
|
|
|
list_del(&chan->device_node);
|
|
|
|
device->chancnt--;
|
2020-01-21 23:43:47 +00:00
|
|
|
chan->dev->chan = NULL;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2020-04-13 17:40:12 +00:00
|
|
|
mutex_lock(&device->chan_mutex);
|
|
|
|
ida_free(&device->chan_ida, chan->chan_id);
|
|
|
|
mutex_unlock(&device->chan_mutex);
|
2020-01-21 23:43:47 +00:00
|
|
|
device_unregister(&chan->dev->device);
|
|
|
|
free_percpu(chan->local);
|
|
|
|
}
|
|
|
|
|
2020-01-21 23:43:53 +00:00
|
|
|
void dma_async_device_channel_unregister(struct dma_device *device,
|
|
|
|
struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
__dma_async_device_channel_unregister(device, chan);
|
|
|
|
dma_channel_rebalance();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
/**
|
2006-07-04 02:45:31 +00:00
|
|
|
* dma_async_device_register - registers DMA devices found
|
2020-04-29 12:21:51 +00:00
|
|
|
* @device: pointer to &struct dma_device
|
2019-12-16 19:01:19 +00:00
|
|
|
*
|
|
|
|
* After calling this routine the structure should not be freed except in the
|
|
|
|
* device_release() callback which will be called after
|
|
|
|
* dma_async_device_unregister() is called and no further references are taken.
|
2006-05-24 00:18:44 +00:00
|
|
|
*/
|
|
|
|
int dma_async_device_register(struct dma_device *device)
|
|
|
|
{
|
2020-04-13 17:40:12 +00:00
|
|
|
int rc;
|
2006-05-24 00:18:44 +00:00
|
|
|
struct dma_chan* chan;
|
|
|
|
|
|
|
|
if (!device)
|
|
|
|
return -ENODEV;
|
|
|
|
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
/* validate device routines */
|
2017-08-27 11:25:32 +00:00
|
|
|
if (!device->dev) {
|
|
|
|
pr_err("DMAdevice must have dev\n");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
2019-12-16 19:01:16 +00:00
|
|
|
device->owner = device->dev->driver->owner;
|
|
|
|
|
2017-08-27 11:25:32 +00:00
|
|
|
if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_MEMCPY");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_XOR");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_XOR_VAL");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_PQ");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_PQ_VAL");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_MEMSET");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_INTERRUPT");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_CYCLIC");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) {
|
|
|
|
dev_err(device->dev,
|
|
|
|
"Device claims capability %s, but op is not defined\n",
|
|
|
|
"DMA_INTERLEAVE");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (!device->device_tx_status) {
|
|
|
|
dev_err(device->dev, "Device tx_status is not defined\n");
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if (!device->device_issue_pending) {
|
|
|
|
dev_err(device->dev, "Device issue_pending is not defined\n");
|
|
|
|
return -EIO;
|
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2019-12-16 19:01:19 +00:00
|
|
|
if (!device->device_release)
|
2020-03-06 13:50:18 +00:00
|
|
|
dev_dbg(device->dev,
|
2019-12-16 19:01:19 +00:00
|
|
|
"WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");
|
|
|
|
|
|
|
|
kref_init(&device->ref);
|
|
|
|
|
2009-09-09 00:42:51 +00:00
|
|
|
/* note: this only matters in the
|
2010-10-07 23:44:50 +00:00
|
|
|
* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
|
2009-09-09 00:42:51 +00:00
|
|
|
*/
|
|
|
|
if (device_has_all_tx_types(device))
|
|
|
|
dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
|
|
|
|
|
2009-03-25 16:13:23 +00:00
|
|
|
rc = get_dma_id(device);
|
2020-01-21 23:43:47 +00:00
|
|
|
if (rc != 0)
|
2009-01-06 18:38:21 +00:00
|
|
|
return rc;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2020-04-13 17:40:12 +00:00
|
|
|
mutex_init(&device->chan_mutex);
|
|
|
|
ida_init(&device->chan_ida);
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
/* represent channels in sysfs. Probably want devs too */
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
2020-04-13 17:40:12 +00:00
|
|
|
rc = __dma_async_device_channel_register(device, chan);
|
2020-01-21 23:43:47 +00:00
|
|
|
if (rc < 0)
|
2007-03-08 17:57:34 +00:00
|
|
|
goto err_out;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2016-07-27 21:32:58 +00:00
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 18:38:15 +00:00
|
|
|
/* take references on public channels */
|
|
|
|
if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
2009-01-06 18:38:14 +00:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
/* if clients are already waiting for channels we need
|
|
|
|
* to take references on their behalf
|
|
|
|
*/
|
|
|
|
if (dma_chan_get(chan) == -ENODEV) {
|
|
|
|
/* note we can only get here for the first
|
|
|
|
* channel as the remaining channels are
|
|
|
|
* guaranteed to get a reference
|
|
|
|
*/
|
|
|
|
rc = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
goto err_out;
|
|
|
|
}
|
|
|
|
}
|
2009-01-06 18:38:14 +00:00
|
|
|
list_add_tail_rcu(&device->global_node, &dma_device_list);
|
2009-03-06 11:07:14 +00:00
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
device->privatecnt++; /* Always private */
|
2009-01-06 18:38:14 +00:00
|
|
|
dma_channel_rebalance();
|
2006-05-24 00:18:44 +00:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
2020-03-06 14:28:39 +00:00
|
|
|
dmaengine_debug_register(device);
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
return 0;
|
2007-03-08 17:57:34 +00:00
|
|
|
|
|
|
|
err_out:
|
2009-03-25 16:13:23 +00:00
|
|
|
/* if we never registered a channel just release the idr */
|
2020-01-21 23:43:47 +00:00
|
|
|
if (!device->chancnt) {
|
2018-06-18 19:41:48 +00:00
|
|
|
ida_free(&dma_ida, device->dev_id);
|
2009-03-25 16:13:23 +00:00
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2007-03-08 17:57:34 +00:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
if (chan->local == NULL)
|
|
|
|
continue;
|
2009-01-06 18:38:21 +00:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan->dev->chan = NULL;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
device_unregister(&chan->dev->device);
|
2007-03-08 17:57:34 +00:00
|
|
|
free_percpu(chan->local);
|
|
|
|
}
|
|
|
|
return rc;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2007-03-16 21:38:05 +00:00
|
|
|
EXPORT_SYMBOL(dma_async_device_register);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2006-07-04 02:45:31 +00:00
|
|
|
/**
|
2009-01-06 18:38:14 +00:00
|
|
|
* dma_async_device_unregister - unregister a DMA device
|
2020-04-29 12:21:51 +00:00
|
|
|
* @device: pointer to &struct dma_device
|
2009-01-06 18:38:18 +00:00
|
|
|
*
|
|
|
|
* This routine is called by dma driver exit routines, dmaengine holds module
|
|
|
|
* references to prevent it being called while channels are in use.
|
2006-07-04 02:45:31 +00:00
|
|
|
*/
|
|
|
|
void dma_async_device_unregister(struct dma_device *device)
|
2006-05-24 00:18:44 +00:00
|
|
|
{
|
2020-01-21 23:43:53 +00:00
|
|
|
struct dma_chan *chan, *n;
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2020-03-06 14:28:39 +00:00
|
|
|
dmaengine_debug_unregister(device);
|
|
|
|
|
2020-01-21 23:43:53 +00:00
|
|
|
list_for_each_entry_safe(chan, n, &device->channels, device_node)
|
2020-01-21 23:43:47 +00:00
|
|
|
__dma_async_device_channel_unregister(device, chan);
|
2019-12-16 19:01:19 +00:00
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
/*
|
|
|
|
* setting DMA_PRIVATE ensures the device being torn down will not
|
|
|
|
* be used in the channel_table
|
|
|
|
*/
|
|
|
|
dma_cap_set(DMA_PRIVATE, device->cap_mask);
|
|
|
|
dma_channel_rebalance();
|
2020-04-13 17:40:12 +00:00
|
|
|
ida_free(&dma_ida, device->dev_id);
|
2019-12-16 19:01:19 +00:00
|
|
|
dma_device_put(device);
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2007-03-16 21:38:05 +00:00
|
|
|
EXPORT_SYMBOL(dma_async_device_unregister);
|
2006-05-24 00:18:44 +00:00
|
|
|
|
2018-07-26 06:45:53 +00:00
|
|
|
static void dmam_device_release(struct device *dev, void *res)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
|
|
|
|
device = *(struct dma_device **)res;
|
|
|
|
dma_async_device_unregister(device);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dmaenginem_async_device_register - registers DMA devices found
|
2020-04-29 12:21:51 +00:00
|
|
|
* @device: pointer to &struct dma_device
|
2018-07-26 06:45:53 +00:00
|
|
|
*
|
|
|
|
* The operation is managed and will be undone on driver detach.
|
|
|
|
*/
|
|
|
|
int dmaenginem_async_device_register(struct dma_device *device)
|
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
p = devres_alloc(dmam_device_release, sizeof(void *), GFP_KERNEL);
|
|
|
|
if (!p)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
ret = dma_async_device_register(device);
|
|
|
|
if (!ret) {
|
|
|
|
*(struct dma_device **)p = device;
|
|
|
|
devres_add(device->dev, p);
|
|
|
|
} else {
|
|
|
|
devres_free(p);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dmaenginem_async_device_register);
|
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
struct dmaengine_unmap_pool {
|
|
|
|
struct kmem_cache *cache;
|
|
|
|
const char *name;
|
|
|
|
mempool_t *pool;
|
|
|
|
size_t size;
|
|
|
|
};
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
|
|
|
|
static struct dmaengine_unmap_pool unmap_pool[] = {
|
|
|
|
__UNMAP_POOL(2),
|
2013-12-09 18:33:16 +00:00
|
|
|
#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
|
2013-10-18 17:35:24 +00:00
|
|
|
__UNMAP_POOL(16),
|
|
|
|
__UNMAP_POOL(128),
|
|
|
|
__UNMAP_POOL(256),
|
|
|
|
#endif
|
|
|
|
};
|
2008-02-03 02:49:57 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
|
|
|
|
{
|
|
|
|
int order = get_count_order(nr);
|
|
|
|
|
|
|
|
switch (order) {
|
|
|
|
case 0 ... 1:
|
|
|
|
return &unmap_pool[0];
|
2017-03-13 21:30:29 +00:00
|
|
|
#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
|
2013-10-18 17:35:24 +00:00
|
|
|
case 2 ... 4:
|
|
|
|
return &unmap_pool[1];
|
|
|
|
case 5 ... 7:
|
|
|
|
return &unmap_pool[2];
|
|
|
|
case 8:
|
|
|
|
return &unmap_pool[3];
|
2017-03-13 21:30:29 +00:00
|
|
|
#endif
|
2013-10-18 17:35:24 +00:00
|
|
|
default:
|
|
|
|
BUG();
|
|
|
|
return NULL;
|
2008-02-03 02:49:57 +00:00
|
|
|
}
|
2013-10-18 17:35:24 +00:00
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
static void dmaengine_unmap(struct kref *kref)
|
|
|
|
{
|
|
|
|
struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
|
|
|
|
struct device *dev = unmap->dev;
|
|
|
|
int cnt, i;
|
|
|
|
|
|
|
|
cnt = unmap->to_cnt;
|
|
|
|
for (i = 0; i < cnt; i++)
|
|
|
|
dma_unmap_page(dev, unmap->addr[i], unmap->len,
|
|
|
|
DMA_TO_DEVICE);
|
|
|
|
cnt += unmap->from_cnt;
|
|
|
|
for (; i < cnt; i++)
|
|
|
|
dma_unmap_page(dev, unmap->addr[i], unmap->len,
|
|
|
|
DMA_FROM_DEVICE);
|
|
|
|
cnt += unmap->bidi_cnt;
|
2013-10-18 17:35:29 +00:00
|
|
|
for (; i < cnt; i++) {
|
|
|
|
if (unmap->addr[i] == 0)
|
|
|
|
continue;
|
2013-10-18 17:35:24 +00:00
|
|
|
dma_unmap_page(dev, unmap->addr[i], unmap->len,
|
|
|
|
DMA_BIDIRECTIONAL);
|
2013-10-18 17:35:29 +00:00
|
|
|
}
|
2014-05-21 21:02:37 +00:00
|
|
|
cnt = unmap->map_cnt;
|
2013-10-18 17:35:24 +00:00
|
|
|
mempool_free(unmap, __get_unmap_pool(cnt)->pool);
|
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
|
|
|
|
{
|
|
|
|
if (unmap)
|
|
|
|
kref_put(&unmap->kref, dmaengine_unmap);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
static void dmaengine_destroy_unmap_pool(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
|
|
|
|
struct dmaengine_unmap_pool *p = &unmap_pool[i];
|
|
|
|
|
2015-09-13 12:15:19 +00:00
|
|
|
mempool_destroy(p->pool);
|
2013-10-18 17:35:24 +00:00
|
|
|
p->pool = NULL;
|
2015-09-13 12:15:19 +00:00
|
|
|
kmem_cache_destroy(p->cache);
|
2013-10-18 17:35:24 +00:00
|
|
|
p->cache = NULL;
|
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
}
|
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
static int __init dmaengine_init_unmap_pool(void)
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
{
|
2013-10-18 17:35:24 +00:00
|
|
|
int i;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
|
|
|
|
struct dmaengine_unmap_pool *p = &unmap_pool[i];
|
|
|
|
size_t size;
|
2008-02-03 02:49:57 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
size = sizeof(struct dmaengine_unmap_data) +
|
|
|
|
sizeof(dma_addr_t) * p->size;
|
|
|
|
|
|
|
|
p->cache = kmem_cache_create(p->name, size, 0,
|
|
|
|
SLAB_HWCACHE_ALIGN, NULL);
|
|
|
|
if (!p->cache)
|
|
|
|
break;
|
|
|
|
p->pool = mempool_create_slab_pool(1, p->cache);
|
|
|
|
if (!p->pool)
|
|
|
|
break;
|
2008-02-03 02:49:57 +00:00
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
if (i == ARRAY_SIZE(unmap_pool))
|
|
|
|
return 0;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:24 +00:00
|
|
|
dmaengine_destroy_unmap_pool();
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
2013-10-18 17:35:25 +00:00
|
|
|
struct dmaengine_unmap_data *
|
2013-10-18 17:35:24 +00:00
|
|
|
dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
|
|
|
|
{
|
|
|
|
struct dmaengine_unmap_data *unmap;
|
|
|
|
|
|
|
|
unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
|
|
|
|
if (!unmap)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
memset(unmap, 0, sizeof(*unmap));
|
|
|
|
kref_init(&unmap->kref);
|
|
|
|
unmap->dev = dev;
|
2014-05-21 21:02:37 +00:00
|
|
|
unmap->map_cnt = nr;
|
2013-10-18 17:35:24 +00:00
|
|
|
|
|
|
|
return unmap;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
}
|
2013-10-18 17:35:25 +00:00
|
|
|
EXPORT_SYMBOL(dmaengine_get_unmap_data);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
|
|
|
|
void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
|
|
|
|
struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
tx->chan = chan;
|
2010-10-07 23:44:50 +00:00
|
|
|
#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
spin_lock_init(&tx->lock);
|
2010-05-17 23:24:16 +00:00
|
|
|
#endif
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 18:10:43 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
|
|
|
|
|
2019-12-23 11:04:44 +00:00
|
|
|
static inline int desc_check_and_set_metadata_mode(
|
|
|
|
struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
|
|
|
|
{
|
|
|
|
/* Make sure that the metadata mode is not mixed */
|
|
|
|
if (!desc->desc_metadata_mode) {
|
|
|
|
if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
|
|
|
|
desc->desc_metadata_mode = mode;
|
|
|
|
else
|
|
|
|
return -ENOTSUPP;
|
|
|
|
} else if (desc->desc_metadata_mode != mode) {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
|
|
|
|
void *data, size_t len)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!desc)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (!desc->metadata_ops || !desc->metadata_ops->attach)
|
|
|
|
return -ENOTSUPP;
|
|
|
|
|
|
|
|
return desc->metadata_ops->attach(desc, data, len);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
|
|
|
|
|
|
|
|
void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
|
|
|
|
size_t *payload_len, size_t *max_len)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!desc)
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
|
|
|
|
if (ret)
|
|
|
|
return ERR_PTR(ret);
|
|
|
|
|
|
|
|
if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
|
|
|
|
return ERR_PTR(-ENOTSUPP);
|
|
|
|
|
|
|
|
return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
|
|
|
|
|
|
|
|
int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
|
|
|
|
size_t payload_len)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!desc)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (!desc->metadata_ops || !desc->metadata_ops->set_len)
|
|
|
|
return -ENOTSUPP;
|
|
|
|
|
|
|
|
return desc->metadata_ops->set_len(desc, payload_len);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
|
|
|
|
|
2020-04-29 12:21:51 +00:00
|
|
|
/**
|
|
|
|
* dma_wait_for_async_tx - spin wait for a transaction to complete
|
|
|
|
* @tx: in-flight transaction to wait on
|
2009-01-06 00:14:31 +00:00
|
|
|
*/
|
|
|
|
enum dma_status
|
|
|
|
dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
|
|
|
|
{
|
2009-07-14 19:19:02 +00:00
|
|
|
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
|
2009-01-06 00:14:31 +00:00
|
|
|
|
|
|
|
if (!tx)
|
2013-10-16 07:59:02 +00:00
|
|
|
return DMA_COMPLETE;
|
2009-01-06 00:14:31 +00:00
|
|
|
|
2009-07-14 19:19:02 +00:00
|
|
|
while (tx->cookie == -EBUSY) {
|
|
|
|
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
|
2016-03-14 14:51:09 +00:00
|
|
|
dev_err(tx->chan->device->dev,
|
|
|
|
"%s timeout waiting for descriptor submission\n",
|
|
|
|
__func__);
|
2009-07-14 19:19:02 +00:00
|
|
|
return DMA_ERROR;
|
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
return dma_sync_wait(tx->chan, tx->cookie);
|
2009-01-06 00:14:31 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
|
|
|
|
|
2020-04-29 12:21:51 +00:00
|
|
|
/**
|
|
|
|
* dma_run_dependencies - process dependent operations on the target channel
|
|
|
|
* @tx: transaction with dependencies
|
|
|
|
*
|
|
|
|
* Helper routine for DMA drivers to process (start) dependent operations
|
|
|
|
* on their target channel.
|
2009-01-06 00:14:31 +00:00
|
|
|
*/
|
|
|
|
void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
|
|
|
|
{
|
2010-05-17 23:24:16 +00:00
|
|
|
struct dma_async_tx_descriptor *dep = txd_next(tx);
|
2009-01-06 00:14:31 +00:00
|
|
|
struct dma_async_tx_descriptor *dep_next;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
if (!dep)
|
|
|
|
return;
|
|
|
|
|
2009-01-12 22:17:20 +00:00
|
|
|
/* we'll submit tx->next now, so clear the link */
|
2010-05-17 23:24:16 +00:00
|
|
|
txd_clear_next(tx);
|
2009-01-06 00:14:31 +00:00
|
|
|
chan = dep->chan;
|
|
|
|
|
|
|
|
/* keep submitting up until a channel switch is detected
|
|
|
|
* in that case we will be called again as a result of
|
|
|
|
* processing the interrupt from async_tx_channel_switch
|
|
|
|
*/
|
|
|
|
for (; dep; dep = dep_next) {
|
2010-05-17 23:24:16 +00:00
|
|
|
txd_lock(dep);
|
|
|
|
txd_clear_parent(dep);
|
|
|
|
dep_next = txd_next(dep);
|
2009-01-06 00:14:31 +00:00
|
|
|
if (dep_next && dep_next->chan == chan)
|
2010-05-17 23:24:16 +00:00
|
|
|
txd_clear_next(dep); /* ->next will be submitted */
|
2009-01-06 00:14:31 +00:00
|
|
|
else
|
|
|
|
dep_next = NULL; /* submit current dep and terminate */
|
2010-05-17 23:24:16 +00:00
|
|
|
txd_unlock(dep);
|
2009-01-06 00:14:31 +00:00
|
|
|
|
|
|
|
dep->tx_submit(dep);
|
|
|
|
}
|
|
|
|
|
|
|
|
chan->device->device_issue_pending(chan);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_run_dependencies);
|
|
|
|
|
2006-05-24 00:18:44 +00:00
|
|
|
static int __init dma_bus_init(void)
|
|
|
|
{
|
2013-10-18 17:35:24 +00:00
|
|
|
int err = dmaengine_init_unmap_pool();
|
|
|
|
|
|
|
|
if (err)
|
|
|
|
return err;
|
2020-03-06 14:28:37 +00:00
|
|
|
|
|
|
|
err = class_register(&dma_devclass);
|
|
|
|
if (!err)
|
|
|
|
dmaengine_debugfs_init();
|
|
|
|
|
|
|
|
return err;
|
2006-05-24 00:18:44 +00:00
|
|
|
}
|
2009-01-06 18:38:22 +00:00
|
|
|
arch_initcall(dma_bus_init);
|