mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

dmaengine updates for v5.15-rc1 

New drivers/devices
  - Support for Renesas RZ/G2L dma controller
  - New driver for AMD PTDMA controller
 
 Updates:
  - Big pile of idxd updates
  - Updates for Altera driver, stm32-dma, dw etc
 
 Also contains, bus_remove_return_void-5.15 to resolve dependencies
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+vs47OPLdNbVcHzyfBQHDyUjg0cFAmE4PBwACgkQfBQHDyUj
 g0euABAArP/f4o6yxtlPj5hwk2ZLw4QRTZEFevn0qULuwHazxGSKVhJEZVz2asYM
 S6I6jSvfKYwdO8/s3EVV0jkz4Uxdl4JUzakeMbEsISNF+hacgIhTxuXkgQkvAre9
 N3/WQgHLRShe+P3mbX/uN4JyXSMQoWCPUy3yk5xxQvuyBy9zgiW8c5rMiwDNsG3c
 wF+kX8520Py1QlcK+q5wF+giklAcraPV+buAvJysOukQwxMQjSd2SIMG63Xa+cNx
 ssvj39au9VInfKYyVioWIUdNQcTRa8+3Ctv6eI44F77x9LfvjBsOLT/dy+BbOCCQ
 7zHAlrBJ6UhpGi7WHk+Tnb4RispjdWNAdEvqWU/EHZNk2II/Lb8IJjDnu3wSuXKy
 AU1uiQ8b6uEY5rKj1lc7XxKw0xGArJEUt7r24z6KNQ7kiYOD4z7G759syGC5atml
 q5m0rY8I7zI7OGhPJIpaAOh+urdWLsdVvgywRoHrKS0NiUXVAAkfbmvHgm5WboLu
 INDbm/HWdqvxo2LqnBj/+NSArhvFfrQyUt/po6lYkPddbG0xARAWsjqra+X8XTvR
 n4P/qlydzCl9QkJGnfM6JrsKGikegNnFvXMUR9kO6Go6IGM9Ea8JD4K6GYk84+yy
 jrSFJCQsS54I97UIRAGrpGW6qVQUYsFiPUtSM2cCuBOwTG03Wz4=
 =RYbR
 -----END PGP SIGNATURE-----

Merge tag 'dmaengine-5.15-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

Pull dmaengine updates from Vinod Koul:
 "New drivers/devices
   - Support for Renesas RZ/G2L dma controller
   - New driver for AMD PTDMA controller

  Updates:
   - Big pile of idxd updates
   - Updates for Altera driver, stm32-dma, dw etc"

* tag 'dmaengine-5.15-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (83 commits)
  dmaengine: sh: fix some NULL dereferences
  dmaengine: sh: Fix unused initialization of pointer lmdesc
  MAINTAINERS: Fix AMD PTDMA DRIVER entry
  dmaengine: ptdma: remove PT_OFFSET to avoid redefnition
  dmaengine: ptdma: Add debugfs entries for PTDMA
  dmaengine: ptdma: register PTDMA controller as a DMA resource
  dmaengine: ptdma: Initial driver for the AMD PTDMA
  dmaengine: fsl-dpaa2-qdma: Fix spelling mistake "faile" -> "failed"
  dmaengine: idxd: remove interrupt disable for dev_lock
  dmaengine: idxd: remove interrupt disable for cmd_lock
  dmaengine: idxd: fix setting up priv mode for dwq
  dmaengine: xilinx_dma: Set DMA mask for coherent APIs
  dmaengine: ti: k3-psil-j721e: Add entry for CSI2RX
  dmaengine: sh: Add DMAC driver for RZ/G2L SoC
  dmaengine: Extend the dma_slave_width for 128 bytes
  dt-bindings: dma: Document RZ/G2L bindings
  dmaengine: ioat: depends on !UML
  dmaengine: idxd: set descriptor allocation size to threshold for swq
  dmaengine: idxd: make submit failure path consistent on desc freeing
  dmaengine: idxd: remove interrupt flag for completion list spinlock
  ...
This commit is contained in:
Linus Torvalds 2021-09-09 11:07:47 -07:00
parent a3fa7a101d 11a427be2c
commit 0aa2516017
54 changed files with 4081 additions and 929 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Documentation/ABI/stable/sysfs-driver-dma-idxd
View File

@ -128,6 +128,8 @@ Date: Aug 28, 2020
KernelVersion: 5.10.0
Contact: dmaengine@vger.kernel.org
Description: The last executed device administrative command's status/error.
Also last configuration error overloaded.
Writing to it will clear the status.
What: /sys/bus/dsa/devices/wq<m>.<n>/block_on_fault
Date: Oct 27, 2020
@ -211,6 +213,13 @@ Contact: dmaengine@vger.kernel.org
Description: Indicate whether ATS disable is turned on for the workqueue.
0 indicates ATS is on, and 1 indicates ATS is off for the workqueue.
What: /sys/bus/dsa/devices/wq<m>.<n>/occupancy
Date May 25, 2021
KernelVersion: 5.14.0
Contact: dmaengine@vger.kernel.org
Description: Show the current number of entries in this WQ if WQ Occupancy
Support bit WQ capabilities is 1.
What: /sys/bus/dsa/devices/engine<m>.<n>/group_id
Date: Oct 25, 2019
KernelVersion: 5.6.0

5
Documentation/admin-guide/kernel-parameters.txt
View File

@ -1758,6 +1758,11 @@
support for the idxd driver. By default it is set to
true (1).
idxd.tc_override= [HW]
Format: <bool>
Allow override of default traffic class configuration
for the device. By default it is set to false (0).
ieee754= [MIPS] Select IEEE Std 754 conformance mode
Format: { strict | legacy | 2008 | relaxed }
Default: strict

4
Documentation/devicetree/bindings/dma/altr,msgdma.yaml
View File

@ -24,13 +24,15 @@ properties:
items:
- description: Control and Status Register Slave Port
- description: Descriptor Slave Port
- description: Response Slave Port
- description: Response Slave Port (Optional)
minItems: 2
reg-names:
items:
- const: csr
- const: desc
- const: resp
minItems: 2
interrupts:
maxItems: 1

130
Documentation/devicetree/bindings/dma/renesas,rz-dmac.yaml Normal file
View File

@ -0,0 +1,130 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/dma/renesas,rz-dmac.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Renesas RZ/G2L DMA Controller
maintainers:
- Biju Das <biju.das.jz@bp.renesas.com>
allOf:
- $ref: "dma-controller.yaml#"
properties:
compatible:
items:
- enum:
- renesas,r9a07g044-dmac # RZ/G2{L,LC}
- const: renesas,rz-dmac
reg:
items:
- description: Control and channel register block
- description: DMA extended resource selector block
interrupts:
maxItems: 17
interrupt-names:
items:
- const: error
- const: ch0
- const: ch1
- const: ch2
- const: ch3
- const: ch4
- const: ch5
- const: ch6
- const: ch7
- const: ch8
- const: ch9
- const: ch10
- const: ch11
- const: ch12
- const: ch13
- const: ch14
- const: ch15
clocks:
items:
- description: DMA main clock
- description: DMA register access clock
'#dma-cells':
const: 1
description:
The cell specifies the encoded MID/RID values of the DMAC port
connected to the DMA client and the slave channel configuration
parameters.
bits[0:9] - Specifies MID/RID value
bit[10] - Specifies DMA request high enable (HIEN)
bit[11] - Specifies DMA request detection type (LVL)
bits[12:14] - Specifies DMAACK output mode (AM)
bit[15] - Specifies Transfer Mode (TM)
dma-channels:
const: 16
power-domains:
maxItems: 1
resets:
items:
- description: Reset for DMA ARESETN reset terminal
- description: Reset for DMA RST_ASYNC reset terminal
required:
- compatible
- reg
- interrupts
- interrupt-names
- clocks
- '#dma-cells'
- dma-channels
- power-domains
- resets
additionalProperties: false
examples:
- |
#include <dt-bindings/interrupt-controller/arm-gic.h>
#include <dt-bindings/clock/r9a07g044-cpg.h>
dmac: dma-controller@11820000 {
compatible = "renesas,r9a07g044-dmac",
"renesas,rz-dmac";
reg = <0x11820000 0x10000>,
<0x11830000 0x10000>;
interrupts = <GIC_SPI 141 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 125 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 126 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 127 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 128 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 129 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 130 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 131 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 132 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 133 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 134 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 135 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 136 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 137 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 138 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 139 IRQ_TYPE_EDGE_RISING>,
<GIC_SPI 140 IRQ_TYPE_EDGE_RISING>;
interrupt-names = "error",
"ch0", "ch1", "ch2", "ch3",
"ch4", "ch5", "ch6", "ch7",
"ch8", "ch9", "ch10", "ch11",
"ch12", "ch13", "ch14", "ch15";
clocks = <&cpg CPG_MOD R9A07G044_DMAC_ACLK>,
<&cpg CPG_MOD R9A07G044_DMAC_PCLK>;
power-domains = <&cpg>;
resets = <&cpg R9A07G044_DMAC_ARESETN>,
<&cpg R9A07G044_DMAC_RST_ASYNC>;
#dma-cells = <1>;
dma-channels = <16>;
};

7
Documentation/devicetree/bindings/dma/st,stm32-dma.yaml
View File

@ -40,6 +40,13 @@ description: |
0x0: FIFO mode with threshold selectable with bit 0-1
0x1: Direct mode: each DMA request immediately initiates a transfer
from/to the memory, FIFO is bypassed.
-bit 4: alternative DMA request/acknowledge protocol
0x0: Use standard DMA ACK management, where ACK signal is maintained
up to the removal of request and transfer completion
0x1: Use alternative DMA ACK management, where ACK de-assertion does
not wait for the de-assertion of the REQuest, ACK is only managed
by transfer completion. This must only be used on channels
managing transfers for STM32 USART/UART.
maintainers:

6
MAINTAINERS
View File

@ -985,6 +985,12 @@ S: Supported
T: git https://gitlab.freedesktop.org/agd5f/linux.git
F: drivers/gpu/drm/amd/pm/powerplay/
AMD PTDMA DRIVER
M: Sanjay R Mehta <sanju.mehta@amd.com>
L: dmaengine@vger.kernel.org
S: Maintained
F: drivers/dma/ptdma/
AMD SEATTLE DEVICE TREE SUPPORT
M: Brijesh Singh <brijeshkumar.singh@amd.com>
M: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>

28
drivers/dma/Kconfig
View File

@ -277,10 +277,15 @@ config INTEL_IDMA64
Enable DMA support for Intel Low Power Subsystem such as found on
Intel Skylake PCH.
config INTEL_IDXD_BUS
tristate
default INTEL_IDXD
config INTEL_IDXD
tristate "Intel Data Accelerators support"
depends on PCI && X86_64
depends on PCI && X86_64 && !UML
depends on PCI_MSI
depends on PCI_PASID
depends on SBITMAP
select DMA_ENGINE
help
@ -291,6 +296,23 @@ config INTEL_IDXD
If unsure, say N.
config INTEL_IDXD_COMPAT
bool "Legacy behavior for idxd driver"
depends on PCI && X86_64
select INTEL_IDXD_BUS
help
Compatible driver to support old /sys/bus/dsa/drivers/dsa behavior.
The old behavior performed driver bind/unbind for device and wq
devices all under the dsa driver. The compat driver will emulate
the legacy behavior in order to allow existing support apps (i.e.
accel-config) to continue function. It is expected that accel-config
v3.2 and earlier will need the compat mode. A distro with later
accel-config version can disable this compat config.
Say Y if you have old applications that require such behavior.
If unsure, say N.
# Config symbol that collects all the dependencies that's necessary to
# support shared virtual memory for the devices supported by idxd.
config INTEL_IDXD_SVM
@ -315,7 +337,7 @@ config INTEL_IDXD_PERFMON
config INTEL_IOATDMA
tristate "Intel I/OAT DMA support"
depends on PCI && X86_64
depends on PCI && X86_64 && !UML
select DMA_ENGINE
select DMA_ENGINE_RAID
select DCA
@ -716,6 +738,8 @@ source "drivers/dma/bestcomm/Kconfig"
source "drivers/dma/mediatek/Kconfig"
source "drivers/dma/ptdma/Kconfig"
source "drivers/dma/qcom/Kconfig"
source "drivers/dma/dw/Kconfig"

3
drivers/dma/Makefile
View File

@ -16,6 +16,7 @@ obj-$(CONFIG_DMATEST) += dmatest.o
obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o
obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
obj-$(CONFIG_AMD_PTDMA) += ptdma/
obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
obj-$(CONFIG_AT_XDMAC) += at_xdmac.o
obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o
@ -41,7 +42,7 @@ obj-$(CONFIG_IMX_DMA) += imx-dma.o
obj-$(CONFIG_IMX_SDMA) += imx-sdma.o
obj-$(CONFIG_INTEL_IDMA64) += idma64.o
obj-$(CONFIG_INTEL_IOATDMA) += ioat/
obj-$(CONFIG_INTEL_IDXD) += idxd/
obj-y += idxd/
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_K3_DMA) += k3dma.o
obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o

18
drivers/dma/acpi-dma.c
View File

@ -70,10 +70,22 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp,
si = (const struct acpi_csrt_shared_info *)&grp[1];
/* Match device by MMIO and IRQ */
/* Match device by MMIO */
if (si->mmio_base_low != lower_32_bits(mem) ||
si->mmio_base_high != upper_32_bits(mem) ||
si->gsi_interrupt != irq)
si->mmio_base_high != upper_32_bits(mem))
return 0;
/*
* acpi_gsi_to_irq() can't be used because some platforms do not save
* registered IRQs in the MP table. Instead we just try to register
* the GSI, which is the core part of the above mentioned function.
*/
ret = acpi_register_gsi(NULL, si->gsi_interrupt, si->interrupt_mode, si->interrupt_polarity);
if (ret < 0)
return 0;
/* Match device by Linux vIRQ */
if (ret != irq)
return 0;
dev_dbg(&adev->dev, "matches with %.4s%04X (rev %u)\n",

37
drivers/dma/altera-msgdma.c
View File

@ -691,10 +691,14 @@ static void msgdma_tasklet(struct tasklet_struct *t)
spin_lock_irqsave(&mdev->lock, flags);
/* Read number of responses that are available */
count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
__func__, __LINE__, count);
if (mdev->resp) {
/* Read number of responses that are available */
count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
__func__, __LINE__, count);
} else {
count = 1;
}
while (count--) {
/*
@ -703,8 +707,12 @@ static void msgdma_tasklet(struct tasklet_struct *t)
* have any real values, like transferred bytes or error
* bits. So we need to just drop these values.
*/
size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED);
status = ioread32(mdev->resp + MSGDMA_RESP_STATUS);
if (mdev->resp) {
size = ioread32(mdev->resp +
MSGDMA_RESP_BYTES_TRANSFERRED);
status = ioread32(mdev->resp +
MSGDMA_RESP_STATUS);
}
msgdma_complete_descriptor(mdev);
msgdma_chan_desc_cleanup(mdev);
@ -757,14 +765,21 @@ static void msgdma_dev_remove(struct msgdma_device *mdev)
}
static int request_and_map(struct platform_device *pdev, const char *name,
struct resource **res, void __iomem **ptr)
struct resource **res, void __iomem **ptr,
bool optional)
{
struct resource *region;
struct device *device = &pdev->dev;
*res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
if (*res == NULL) {
dev_err(device, "resource %s not defined\n", name);
if (optional) {
*ptr = NULL;
dev_info(device, "optional resource %s not defined\n",
name);
return 0;
}
dev_err(device, "mandatory resource %s not defined\n", name);
return -ENODEV;
}
@ -805,17 +820,17 @@ static int msgdma_probe(struct platform_device *pdev)
mdev->dev = &pdev->dev;
/* Map CSR space */
ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr);
ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr, false);
if (ret)
return ret;
/* Map (extended) descriptor space */
ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc);
ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc, false);
if (ret)
return ret;
/* Map response space */
ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp);
ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp, true);
if (ret)
return ret;

8
drivers/dma/at_xdmac.c
View File

@ -2240,10 +2240,16 @@ static struct platform_driver at_xdmac_driver = {
static int __init at_xdmac_init(void)
{
return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe);
return platform_driver_register(&at_xdmac_driver);
}
subsys_initcall(at_xdmac_init);
static void __exit at_xdmac_exit(void)
{
platform_driver_unregister(&at_xdmac_driver);
}
module_exit(at_xdmac_exit);
MODULE_DESCRIPTION("Atmel Extended DMA Controller driver");
MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>");
MODULE_LICENSE("GPL");

56
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
View File

@ -363,12 +363,16 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan,
DWAXIDMAC_TT_FC_MEM_TO_PER_DST :
DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC)
<< CH_CFG_H_TT_FC_POS;
if (chan->chip->apb_regs)
reg |= (chan->id << CH_CFG_H_DST_PER_POS);
break;
case DMA_DEV_TO_MEM:
reg |= (chan->config.device_fc ?
DWAXIDMAC_TT_FC_PER_TO_MEM_SRC :
DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC)
<< CH_CFG_H_TT_FC_POS;
if (chan->chip->apb_regs)
reg |= (chan->id << CH_CFG_H_SRC_PER_POS);
break;
default:
break;
@ -470,18 +474,13 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan)
pm_runtime_put(chan->chip->dev);
}
static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip,
u32 handshake_num, bool set)
static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set)
{
unsigned long start = 0;
unsigned long reg_value;
unsigned long reg_mask;
unsigned long reg_set;
unsigned long mask;
unsigned long val;
struct axi_dma_chip *chip = chan->chip;
unsigned long reg_value, val;
if (!chip->apb_regs) {
dev_dbg(chip->dev, "apb_regs not initialized\n");
dev_err(chip->dev, "apb_regs not initialized\n");
return;
}
@ -490,26 +489,22 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip,
* Lock the DMA channel by assign a handshake number to the channel.
* Unlock the DMA channel by assign 0x3F to the channel.
*/
if (set) {
reg_set = UNUSED_CHANNEL;
val = handshake_num;
} else {
reg_set = handshake_num;
if (set)
val = chan->hw_handshake_num;
else
val = UNUSED_CHANNEL;
}
reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0);
for_each_set_clump8(start, reg_mask, &reg_value, 64) {
if (reg_mask == reg_set) {
mask = GENMASK_ULL(start + 7, start);
reg_value &= ~mask;
reg_value |= rol64(val, start);
lo_hi_writeq(reg_value,
chip->apb_regs + DMAC_APB_HW_HS_SEL_0);
break;
}
}
/* Channel is already allocated, set handshake as per channel ID */
/* 64 bit write should handle for 8 channels */
reg_value &= ~(DMA_APB_HS_SEL_MASK <<
(chan->id * DMA_APB_HS_SEL_BIT_SIZE));
reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE));
lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0);
return;
}
/*
@ -742,7 +737,7 @@ dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr,
llp = hw_desc->llp;
} while (total_segments);
dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true);
dw_axi_dma_set_hw_channel(chan, true);
return vchan_tx_prep(&chan->vc, &desc->vd, flags);
@ -822,7 +817,7 @@ dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
llp = hw_desc->llp;
} while (num_sgs);
dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true);
dw_axi_dma_set_hw_channel(chan, true);
return vchan_tx_prep(&chan->vc, &desc->vd, flags);
@ -1098,8 +1093,7 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
"%s failed to stop\n", axi_chan_name(chan));
if (chan->direction != DMA_MEM_TO_MEM)
dw_axi_dma_set_hw_channel(chan->chip,
chan->hw_handshake_num, false);
dw_axi_dma_set_hw_channel(chan, false);
if (chan->direction == DMA_MEM_TO_DEV)
dw_axi_dma_set_byte_halfword(chan, false);
@ -1296,7 +1290,7 @@ static int parse_device_properties(struct axi_dma_chip *chip)
return -EINVAL;
chip->dw->hdata->restrict_axi_burst_len = true;
chip->dw->hdata->axi_rw_burst_len = tmp - 1;
chip->dw->hdata->axi_rw_burst_len = tmp;
}
return 0;
@ -1365,7 +1359,6 @@ static int dw_probe(struct platform_device *pdev)
if (ret)
return ret;
INIT_LIST_HEAD(&dw->dma.channels);
for (i = 0; i < hdata->nr_channels; i++) {
struct axi_dma_chan *chan = &dw->chan[i];
@ -1386,6 +1379,7 @@ static int dw_probe(struct platform_device *pdev)
/* DMA capabilities */
dw->dma.chancnt = hdata->nr_channels;
dw->dma.max_burst = hdata->axi_rw_burst_len;
dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS;
dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS;
dw->dma.directions = BIT(DMA_MEM_TO_MEM);

4
drivers/dma/dw-axi-dmac/dw-axi-dmac.h
View File

@ -184,6 +184,8 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
#define DMAC_APB_HALFWORD_WR_CH_EN 0x020 /* DMAC Halfword write enables */
#define UNUSED_CHANNEL 0x3F /* Set unused DMA channel to 0x3F */
#define DMA_APB_HS_SEL_BIT_SIZE 0x08 /* HW handshake bits per channel */
#define DMA_APB_HS_SEL_MASK 0xFF /* HW handshake select masks */
#define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */
/* DMAC_CFG */
@ -256,6 +258,8 @@ enum {
/* CH_CFG_H */
#define CH_CFG_H_PRIORITY_POS 17
#define CH_CFG_H_DST_PER_POS 12
#define CH_CFG_H_SRC_PER_POS 7
#define CH_CFG_H_HS_SEL_DST_POS 4
#define CH_CFG_H_HS_SEL_SRC_POS 3
enum {

138
drivers/dma/dw/idma32.c
View File

@ -1,15 +1,144 @@
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2013,2018 Intel Corporation
// Copyright (C) 2013,2018,2020-2021 Intel Corporation
#include <linux/bitops.h>
#include <linux/dmaengine.h>
#include <linux/errno.h>
#include <linux/io.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
#include "internal.h"
static void idma32_initialize_chan(struct dw_dma_chan *dwc)
#define DMA_CTL_CH(x) (0x1000 + (x) * 4)
#define DMA_SRC_ADDR_FILLIN(x) (0x1100 + (x) * 4)
#define DMA_DST_ADDR_FILLIN(x) (0x1200 + (x) * 4)
#define DMA_XBAR_SEL(x) (0x1300 + (x) * 4)
#define DMA_REGACCESS_CHID_CFG (0x1400)
#define CTL_CH_TRANSFER_MODE_MASK GENMASK(1, 0)
#define CTL_CH_TRANSFER_MODE_S2S 0
#define CTL_CH_TRANSFER_MODE_S2D 1
#define CTL_CH_TRANSFER_MODE_D2S 2
#define CTL_CH_TRANSFER_MODE_D2D 3
#define CTL_CH_RD_RS_MASK GENMASK(4, 3)
#define CTL_CH_WR_RS_MASK GENMASK(6, 5)
#define CTL_CH_RD_NON_SNOOP_BIT BIT(8)
#define CTL_CH_WR_NON_SNOOP_BIT BIT(9)
#define XBAR_SEL_DEVID_MASK GENMASK(15, 0)
#define XBAR_SEL_RX_TX_BIT BIT(16)
#define XBAR_SEL_RX_TX_SHIFT 16
#define REGACCESS_CHID_MASK GENMASK(2, 0)
static unsigned int idma32_get_slave_devfn(struct dw_dma_chan *dwc)
{
struct device *slave = dwc->chan.slave;
if (!slave || !dev_is_pci(slave))
return 0;
return to_pci_dev(slave)->devfn;
}
static void idma32_initialize_chan_xbar(struct dw_dma_chan *dwc)
{
struct dw_dma *dw = to_dw_dma(dwc->chan.device);
void __iomem *misc = __dw_regs(dw);
u32 cfghi = 0, cfglo = 0;
u8 dst_id, src_id;
u32 value;
/* DMA Channel ID Configuration register must be programmed first */
value = readl(misc + DMA_REGACCESS_CHID_CFG);
value &= ~REGACCESS_CHID_MASK;
value |= dwc->chan.chan_id;
writel(value, misc + DMA_REGACCESS_CHID_CFG);
/* Configure channel attributes */
value = readl(misc + DMA_CTL_CH(dwc->chan.chan_id));
value &= ~(CTL_CH_RD_NON_SNOOP_BIT | CTL_CH_WR_NON_SNOOP_BIT);
value &= ~(CTL_CH_RD_RS_MASK | CTL_CH_WR_RS_MASK);
value &= ~CTL_CH_TRANSFER_MODE_MASK;
switch (dwc->direction) {
case DMA_MEM_TO_DEV:
value |= CTL_CH_TRANSFER_MODE_D2S;
value |= CTL_CH_WR_NON_SNOOP_BIT;
break;
case DMA_DEV_TO_MEM:
value |= CTL_CH_TRANSFER_MODE_S2D;
value |= CTL_CH_RD_NON_SNOOP_BIT;
break;
default:
/*
* Memory-to-Memory and Device-to-Device are ignored for now.
*
* For Memory-to-Memory transfers we would need to set mode
* and disable snooping on both sides.
*/
return;
}
writel(value, misc + DMA_CTL_CH(dwc->chan.chan_id));
/* Configure crossbar selection */
value = readl(misc + DMA_XBAR_SEL(dwc->chan.chan_id));
/* DEVFN selection */
value &= ~XBAR_SEL_DEVID_MASK;
value |= idma32_get_slave_devfn(dwc);
switch (dwc->direction) {
case DMA_MEM_TO_DEV:
value |= XBAR_SEL_RX_TX_BIT;
break;
case DMA_DEV_TO_MEM:
value &= ~XBAR_SEL_RX_TX_BIT;
break;
default:
/* Memory-to-Memory and Device-to-Device are ignored for now */
return;
}
writel(value, misc + DMA_XBAR_SEL(dwc->chan.chan_id));
/* Configure DMA channel low and high registers */
switch (dwc->direction) {
case DMA_MEM_TO_DEV:
dst_id = dwc->chan.chan_id;
src_id = dwc->dws.src_id;
break;
case DMA_DEV_TO_MEM:
dst_id = dwc->dws.dst_id;
src_id = dwc->chan.chan_id;
break;
default:
/* Memory-to-Memory and Device-to-Device are ignored for now */
return;
}
/* Set default burst alignment */
cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN;
/* Low 4 bits of the request lines */
cfghi |= IDMA32C_CFGH_DST_PER(dst_id & 0xf);
cfghi |= IDMA32C_CFGH_SRC_PER(src_id & 0xf);
/* Request line extension (2 bits) */
cfghi |= IDMA32C_CFGH_DST_PER_EXT(dst_id >> 4 & 0x3);
cfghi |= IDMA32C_CFGH_SRC_PER_EXT(src_id >> 4 & 0x3);
channel_writel(dwc, CFG_LO, cfglo);
channel_writel(dwc, CFG_HI, cfghi);
}
static void idma32_initialize_chan_generic(struct dw_dma_chan *dwc)
{
u32 cfghi = 0;
u32 cfglo = 0;
@ -134,7 +263,10 @@ int idma32_dma_probe(struct dw_dma_chip *chip)
return -ENOMEM;
/* Channel operations */
dw->initialize_chan = idma32_initialize_chan;
if (chip->pdata->quirks & DW_DMA_QUIRK_XBAR_PRESENT)
dw->initialize_chan = idma32_initialize_chan_xbar;
else
dw->initialize_chan = idma32_initialize_chan_generic;
dw->suspend_chan = idma32_suspend_chan;
dw->resume_chan = idma32_resume_chan;
dw->prepare_ctllo = idma32_prepare_ctllo;

16
drivers/dma/dw/internal.h
View File

@ -74,4 +74,20 @@ static __maybe_unused const struct dw_dma_chip_pdata idma32_chip_pdata = {
.remove = idma32_dma_remove,
};
static const struct dw_dma_platform_data xbar_pdata = {
.nr_channels = 8,
.chan_allocation_order = CHAN_ALLOCATION_ASCENDING,
.chan_priority = CHAN_PRIORITY_ASCENDING,
.block_size = 131071,
.nr_masters = 1,
.data_width = {4},
.quirks = DW_DMA_QUIRK_XBAR_PRESENT,
};
static __maybe_unused const struct dw_dma_chip_pdata xbar_chip_pdata = {
.pdata = &xbar_pdata,
.probe = idma32_dma_probe,
.remove = idma32_dma_remove,
};
#endif /* _DMA_DW_INTERNAL_H */

49
drivers/dma/dw/of.c
View File

@ -50,15 +50,10 @@ struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
struct dw_dma_platform_data *pdata;
u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS];
u32 tmp, arr[DW_DMA_MAX_NR_MASTERS];
u32 nr_masters;
u32 nr_channels;
if (!np) {
dev_err(&pdev->dev, "Missing DT data\n");
return NULL;
}
if (of_property_read_u32(np, "dma-masters", &nr_masters))
return NULL;
if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS)
@ -76,41 +71,29 @@ struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev)
pdata->nr_masters = nr_masters;
pdata->nr_channels = nr_channels;
if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
pdata->chan_allocation_order = (unsigned char)tmp;
of_property_read_u32(np, "chan_allocation_order", &pdata->chan_allocation_order);
of_property_read_u32(np, "chan_priority", &pdata->chan_priority);
if (!of_property_read_u32(np, "chan_priority", &tmp))
pdata->chan_priority = tmp;
of_property_read_u32(np, "block_size", &pdata->block_size);
if (!of_property_read_u32(np, "block_size", &tmp))
pdata->block_size = tmp;
if (!of_property_read_u32_array(np, "data-width", arr, nr_masters)) {
for (tmp = 0; tmp < nr_masters; tmp++)
pdata->data_width[tmp] = arr[tmp];
} else if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) {
/* Try deprecated property first */
if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) {
for (tmp = 0; tmp < nr_masters; tmp++)
pdata->data_width[tmp] = BIT(arr[tmp] & 0x07);
}
if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) {
for (tmp = 0; tmp < nr_channels; tmp++)
pdata->multi_block[tmp] = mb[tmp];
} else {
for (tmp = 0; tmp < nr_channels; tmp++)
pdata->multi_block[tmp] = 1;
}
/* If "data_width" and "data-width" both provided use the latter one */
of_property_read_u32_array(np, "data-width", pdata->data_width, nr_masters);
if (of_property_read_u32_array(np, "snps,max-burst-len", pdata->max_burst,
nr_channels)) {
memset32(pdata->max_burst, DW_DMA_MAX_BURST, nr_channels);
}
memset32(pdata->multi_block, 1, nr_channels);
of_property_read_u32_array(np, "multi-block", pdata->multi_block, nr_channels);
if (!of_property_read_u32(np, "snps,dma-protection-control", &tmp)) {
if (tmp > CHAN_PROTCTL_MASK)
return NULL;
pdata->protctl = tmp;
}
memset32(pdata->max_burst, DW_DMA_MAX_BURST, nr_channels);
of_property_read_u32_array(np, "snps,max-burst-len", pdata->max_burst, nr_channels);
of_property_read_u32(np, "snps,dma-protection-control", &pdata->protctl);
if (pdata->protctl > CHAN_PROTCTL_MASK)
return NULL;
return pdata;
}

6
drivers/dma/dw/pci.c
View File

@ -120,9 +120,9 @@ static const struct pci_device_id dw_pci_id_table[] = {
{ PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_dma_chip_pdata },
/* Elkhart Lake iDMA 32-bit (PSE DMA) */
{ PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&idma32_chip_pdata },
{ PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&idma32_chip_pdata },
{ PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&idma32_chip_pdata },
{ PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&xbar_chip_pdata },
{ PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&xbar_chip_pdata },
{ PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&xbar_chip_pdata },
/* Haswell */
{ PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_dma_chip_pdata },

6
drivers/dma/dw/platform.c
View File

@ -149,9 +149,9 @@ static const struct acpi_device_id dw_dma_acpi_id_table[] = {
{ "808622C0", (kernel_ulong_t)&dw_dma_chip_pdata },
/* Elkhart Lake iDMA 32-bit (PSE DMA) */
{ "80864BB4", (kernel_ulong_t)&idma32_chip_pdata },
{ "80864BB5", (kernel_ulong_t)&idma32_chip_pdata },
{ "80864BB6", (kernel_ulong_t)&idma32_chip_pdata },
{ "80864BB4", (kernel_ulong_t)&xbar_chip_pdata },
{ "80864BB5", (kernel_ulong_t)&xbar_chip_pdata },
{ "80864BB6", (kernel_ulong_t)&xbar_chip_pdata },
{ }
};

6
drivers/dma/ep93xx_dma.c
View File

@ -897,7 +897,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
if (data && data->name)
name = data->name;
ret = clk_enable(edmac->clk);
ret = clk_prepare_enable(edmac->clk);
if (ret)
return ret;
@ -936,7 +936,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
fail_free_irq:
free_irq(edmac->irq, edmac);
fail_clk_disable:
clk_disable(edmac->clk);
clk_disable_unprepare(edmac->clk);
return ret;
}
@ -969,7 +969,7 @@ static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
list_for_each_entry_safe(desc, d, &list, node)
kfree(desc);
clk_disable(edmac->clk);
clk_disable_unprepare(edmac->clk);
free_irq(edmac->irq, edmac);
}

10
drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c
View File

@ -291,9 +291,8 @@ static void dpaa2_qdma_issue_pending(struct dma_chan *chan)
err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd);
if (err) {
list_del(&dpaa2_comp->list);
list_add_tail(&dpaa2_comp->list,
&dpaa2_chan->comp_free);
list_move_tail(&dpaa2_comp->list,
&dpaa2_chan->comp_free);
}
}
err_enqueue:
@ -626,8 +625,7 @@ static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc)
dpaa2_comp = to_fsl_qdma_comp(vdesc);
qchan = dpaa2_comp->qchan;
spin_lock_irqsave(&qchan->queue_lock, flags);
list_del(&dpaa2_comp->list);
list_add_tail(&dpaa2_comp->list, &qchan->comp_free);
list_move_tail(&dpaa2_comp->list, &qchan->comp_free);
spin_unlock_irqrestore(&qchan->queue_lock, flags);
}
@ -703,7 +701,7 @@ static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev)
/* DPDMAI enable */
err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle);
if (err) {
dev_err(dev, "dpdmai_enable() faile\n");
dev_err(dev, "dpdmai_enable() failed\n");
goto err_enable;
}

10
drivers/dma/hisi_dma.c
View File

@ -133,11 +133,6 @@ static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
writel_relaxed(tmp, addr);
}
static void hisi_dma_free_irq_vectors(void *data)
{
pci_free_irq_vectors(data);
}
static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
bool pause)
{
@ -544,6 +539,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
pci_set_drvdata(pdev, hdma_dev);
pci_set_master(pdev);
/* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
PCI_IRQ_MSI);
if (ret < 0) {
@ -551,10 +547,6 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return ret;
}
ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev);
if (ret)
return ret;
dma_dev = &hdma_dev->dma_dev;
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;

8
drivers/dma/idxd/Makefile
View File

@ -1,4 +1,12 @@
ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=IDXD
obj-$(CONFIG_INTEL_IDXD) += idxd.o
idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o
idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o
obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o
idxd_bus-y := bus.o
obj-$(CONFIG_INTEL_IDXD_COMPAT) += idxd_compat.o
idxd_compat-y := compat.o

91
drivers/dma/idxd/bus.c Normal file
View File

@ -0,0 +1,91 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include "idxd.h"
int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *owner,
const char *mod_name)
{
struct device_driver *drv = &idxd_drv->drv;
if (!idxd_drv->type) {
pr_debug("driver type not set (%ps)\n", __builtin_return_address(0));
return -EINVAL;
}
drv->name = idxd_drv->name;
drv->bus = &dsa_bus_type;
drv->owner = owner;
drv->mod_name = mod_name;
return driver_register(drv);
}
EXPORT_SYMBOL_GPL(__idxd_driver_register);
void idxd_driver_unregister(struct idxd_device_driver *idxd_drv)
{
driver_unregister(&idxd_drv->drv);
}
EXPORT_SYMBOL_GPL(idxd_driver_unregister);
static int idxd_config_bus_match(struct device *dev,
struct device_driver *drv)
{
struct idxd_device_driver *idxd_drv =
container_of(drv, struct idxd_device_driver, drv);
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
int i = 0;
while (idxd_drv->type[i] != IDXD_DEV_NONE) {
if (idxd_dev->type == idxd_drv->type[i])
return 1;
i++;
}
return 0;
}
static int idxd_config_bus_probe(struct device *dev)
{
struct idxd_device_driver *idxd_drv =
container_of(dev->driver, struct idxd_device_driver, drv);
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return idxd_drv->probe(idxd_dev);
}
static void idxd_config_bus_remove(struct device *dev)
{
struct idxd_device_driver *idxd_drv =
container_of(dev->driver, struct idxd_device_driver, drv);
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
idxd_drv->remove(idxd_dev);
}
struct bus_type dsa_bus_type = {
.name = "dsa",
.match = idxd_config_bus_match,
.probe = idxd_config_bus_probe,
.remove = idxd_config_bus_remove,
};
EXPORT_SYMBOL_GPL(dsa_bus_type);
static int __init dsa_bus_init(void)
{
return bus_register(&dsa_bus_type);
}
module_init(dsa_bus_init);
static void __exit dsa_bus_exit(void)
{
bus_unregister(&dsa_bus_type);
}
module_exit(dsa_bus_exit);
MODULE_DESCRIPTION("IDXD driver dsa_bus_type driver");
MODULE_LICENSE("GPL v2");

73
drivers/dma/idxd/cdev.c
View File

@ -41,7 +41,7 @@ struct idxd_user_context {
static void idxd_cdev_dev_release(struct device *dev)
{
struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev);
struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
struct idxd_cdev_context *cdev_ctx;
struct idxd_wq *wq = idxd_cdev->wq;
@ -218,14 +218,13 @@ static __poll_t idxd_cdev_poll(struct file *filp,
struct idxd_user_context *ctx = filp->private_data;
struct idxd_wq *wq = ctx->wq;
struct idxd_device *idxd = wq->idxd;
unsigned long flags;
__poll_t out = 0;
poll_wait(filp, &wq->err_queue, wait);
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock(&idxd->dev_lock);
if (idxd->sw_err.valid)
out = EPOLLIN | EPOLLRDNORM;
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock(&idxd->dev_lock);
return out;
}
@ -256,9 +255,10 @@ int idxd_wq_add_cdev(struct idxd_wq *wq)
if (!idxd_cdev)
return -ENOMEM;
idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
idxd_cdev->wq = wq;
cdev = &idxd_cdev->cdev;
dev = &idxd_cdev->dev;
dev = cdev_dev(idxd_cdev);
cdev_ctx = &ictx[wq->idxd->data->type];
minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
if (minor < 0) {
@ -268,7 +268,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq)
idxd_cdev->minor = minor;
device_initialize(dev);
dev->parent = &wq->conf_dev;
dev->parent = wq_confdev(wq);
dev->bus = &dsa_bus_type;
dev->type = &idxd_cdev_device_type;
dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
@ -299,10 +299,67 @@ void idxd_wq_del_cdev(struct idxd_wq *wq)
idxd_cdev = wq->idxd_cdev;
wq->idxd_cdev = NULL;
cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev);
put_device(&idxd_cdev->dev);
cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
put_device(cdev_dev(idxd_cdev));
}
static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
{
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
struct idxd_device *idxd = wq->idxd;
int rc;
if (idxd->state != IDXD_DEV_ENABLED)
return -ENXIO;
mutex_lock(&wq->wq_lock);
wq->type = IDXD_WQT_USER;
rc = __drv_enable_wq(wq);
if (rc < 0)
goto err;
rc = idxd_wq_add_cdev(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
goto err_cdev;
}
idxd->cmd_status = 0;
mutex_unlock(&wq->wq_lock);
return 0;
err_cdev:
__drv_disable_wq(wq);
err:
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
return rc;
}
static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
{
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
mutex_lock(&wq->wq_lock);
idxd_wq_del_cdev(wq);
__drv_disable_wq(wq);
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
}
static enum idxd_dev_type dev_types[] = {
IDXD_DEV_WQ,
IDXD_DEV_NONE,
};
struct idxd_device_driver idxd_user_drv = {
.probe = idxd_user_drv_probe,
.remove = idxd_user_drv_remove,
.name = "user",
.type = dev_types,
};
EXPORT_SYMBOL_GPL(idxd_user_drv);
int idxd_cdev_register(void)
{
int rc, i;

107
drivers/dma/idxd/compat.c Normal file
View File

@ -0,0 +1,107 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/device/bus.h>
#include "idxd.h"
extern int device_driver_attach(struct device_driver *drv, struct device *dev);
extern void device_driver_detach(struct device *dev);
#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \
struct driver_attribute driver_attr_##_name = \
__ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store)
static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t count)
{
struct bus_type *bus = drv->bus;
struct device *dev;
int rc = -ENODEV;
dev = bus_find_device_by_name(bus, NULL, buf);
if (dev && dev->driver) {
device_driver_detach(dev);
rc = count;
}
return rc;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store);
static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t count)
{
struct bus_type *bus = drv->bus;
struct device *dev;
struct device_driver *alt_drv = NULL;
int rc = -ENODEV;
struct idxd_dev *idxd_dev;
dev = bus_find_device_by_name(bus, NULL, buf);
if (!dev || dev->driver || drv != &dsa_drv.drv)
return -ENODEV;
idxd_dev = confdev_to_idxd_dev(dev);
if (is_idxd_dev(idxd_dev)) {
alt_drv = driver_find("idxd", bus);
} else if (is_idxd_wq_dev(idxd_dev)) {
struct idxd_wq *wq = confdev_to_wq(dev);
if (is_idxd_wq_kernel(wq))
alt_drv = driver_find("dmaengine", bus);
else if (is_idxd_wq_user(wq))
alt_drv = driver_find("user", bus);
}
if (!alt_drv)
return -ENODEV;
rc = device_driver_attach(alt_drv, dev);
if (rc < 0)
return rc;
return count;
}
static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store);
static struct attribute *dsa_drv_compat_attrs[] = {
&driver_attr_bind.attr,
&driver_attr_unbind.attr,
NULL,
};
static const struct attribute_group dsa_drv_compat_attr_group = {
.attrs = dsa_drv_compat_attrs,
};
static const struct attribute_group *dsa_drv_compat_groups[] = {
&dsa_drv_compat_attr_group,
NULL,
};
static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev)
{
return -ENODEV;
}
static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev)
{
}
static enum idxd_dev_type dev_types[] = {
IDXD_DEV_NONE,
};
struct idxd_device_driver dsa_drv = {
.name = "dsa",
.probe = idxd_dsa_drv_probe,
.remove = idxd_dsa_drv_remove,
.type = dev_types,
.drv = {
.suppress_bind_attrs = true,
.groups = dsa_drv_compat_groups,
},
};
module_idxd_driver(dsa_drv);
MODULE_IMPORT_NS(IDXD);

382
drivers/dma/idxd/device.c
View File

@ -15,6 +15,8 @@
static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
u32 *status);
static void idxd_device_wqs_clear_state(struct idxd_device *idxd);
static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
/* Interrupt control bits */
void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id)
@ -139,8 +141,8 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq)
if (wq->type != IDXD_WQT_KERNEL)
return 0;
wq->num_descs = wq->size;
num_descs = wq->size;
num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;
wq->num_descs = num_descs;
rc = alloc_hw_descs(wq, num_descs);
if (rc < 0)
@ -234,7 +236,7 @@ int idxd_wq_enable(struct idxd_wq *wq)
return 0;
}
int idxd_wq_disable(struct idxd_wq *wq)
int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
@ -255,6 +257,8 @@ int idxd_wq_disable(struct idxd_wq *wq)
return -ENXIO;
}
if (reset_config)
idxd_wq_disable_cleanup(wq);
wq->state = IDXD_WQ_DISABLED;
dev_dbg(dev, "WQ %d disabled\n", wq->id);
return 0;
@ -289,6 +293,7 @@ void idxd_wq_reset(struct idxd_wq *wq)
operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);
idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL);
idxd_wq_disable_cleanup(wq);
wq->state = IDXD_WQ_DISABLED;
}
@ -315,6 +320,7 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq)
devm_iounmap(dev, wq->portal);
wq->portal = NULL;
wq->portal_offset = 0;
}
void idxd_wqs_unmap_portal(struct idxd_device *idxd)
@ -335,19 +341,18 @@ int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
int rc;
union wqcfg wqcfg;
unsigned int offset;
unsigned long flags;
rc = idxd_wq_disable(wq);
rc = idxd_wq_disable(wq, false);
if (rc < 0)
return rc;
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock(&idxd->dev_lock);
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
wqcfg.pasid_en = 1;
wqcfg.pasid = pasid;
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock(&idxd->dev_lock);
rc = idxd_wq_enable(wq);
if (rc < 0)
@ -362,19 +367,18 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq)
int rc;
union wqcfg wqcfg;
unsigned int offset;
unsigned long flags;
rc = idxd_wq_disable(wq);
rc = idxd_wq_disable(wq, false);
if (rc < 0)
return rc;
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
spin_lock_irqsave(&idxd->dev_lock, flags);
spin_lock(&idxd->dev_lock);
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
wqcfg.pasid_en = 0;
wqcfg.pasid = 0;
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_unlock(&idxd->dev_lock);
rc = idxd_wq_enable(wq);
if (rc < 0)
@ -383,11 +387,11 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq)
return 0;
}
void idxd_wq_disable_cleanup(struct idxd_wq *wq)
static void idxd_wq_disable_cleanup(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
lockdep_assert_held(&idxd->dev_lock);
lockdep_assert_held(&wq->wq_lock);
memset(wq->wqcfg, 0, idxd->wqcfg_size);
wq->type = IDXD_WQT_NONE;
wq->size = 0;
@ -396,6 +400,7 @@ void idxd_wq_disable_cleanup(struct idxd_wq *wq)
wq->priority = 0;
wq->ats_dis = 0;
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
memset(wq->name, 0, WQ_NAME_SIZE);
}
@ -455,7 +460,6 @@ int idxd_device_init_reset(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
union idxd_command_reg cmd;
unsigned long flags;
if (idxd_device_is_halted(idxd)) {
dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
@ -465,13 +469,13 @@ int idxd_device_init_reset(struct idxd_device *idxd)
memset(&cmd, 0, sizeof(cmd));
cmd.cmd = IDXD_CMD_RESET_DEVICE;
dev_dbg(dev, "%s: sending reset for init.\n", __func__);
spin_lock_irqsave(&idxd->cmd_lock, flags);
spin_lock(&idxd->cmd_lock);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) &
IDXD_CMDSTS_ACTIVE)
cpu_relax();
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
spin_unlock(&idxd->cmd_lock);
return 0;
}
@ -480,7 +484,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
{
union idxd_command_reg cmd;
DECLARE_COMPLETION_ONSTACK(done);
unsigned long flags;
u32 stat;
if (idxd_device_is_halted(idxd)) {
dev_warn(&idxd->pdev->dev, "Device is HALTED!\n");
@ -494,7 +498,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
cmd.operand = operand;
cmd.int_req = 1;
spin_lock_irqsave(&idxd->cmd_lock, flags);
spin_lock(&idxd->cmd_lock);
wait_event_lock_irq(idxd->cmd_waitq,
!test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags),
idxd->cmd_lock);
@ -511,18 +515,18 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand,
* After command submitted, release lock and go to sleep until
* the command completes via interrupt.
*/
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
spin_unlock(&idxd->cmd_lock);
wait_for_completion(&done);
spin_lock_irqsave(&idxd->cmd_lock, flags);
if (status) {
*status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
idxd->cmd_status = *status & GENMASK(7, 0);
}
stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
spin_lock(&idxd->cmd_lock);
if (status)
*status = stat;
idxd->cmd_status = stat & GENMASK(7, 0);
__clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags);
/* Wake up other pending commands */
wake_up(&idxd->cmd_waitq);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
spin_unlock(&idxd->cmd_lock);
}
int idxd_device_enable(struct idxd_device *idxd)
@ -548,27 +552,10 @@ int idxd_device_enable(struct idxd_device *idxd)
return 0;
}
void idxd_device_wqs_clear_state(struct idxd_device *idxd)
{
int i;
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED) {
idxd_wq_disable_cleanup(wq);
wq->state = IDXD_WQ_DISABLED;
}
}
}
int idxd_device_disable(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
u32 status;
unsigned long flags;
if (!idxd_is_enabled(idxd)) {
dev_dbg(dev, "Device is not enabled\n");
@ -584,22 +571,20 @@ int idxd_device_disable(struct idxd_device *idxd)
return -ENXIO;
}
spin_lock_irqsave(&idxd->dev_lock, flags);
idxd_device_wqs_clear_state(idxd);
idxd->state = IDXD_DEV_CONF_READY;
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_lock(&idxd->dev_lock);
idxd_device_clear_state(idxd);
idxd->state = IDXD_DEV_DISABLED;
spin_unlock(&idxd->dev_lock);
return 0;
}
void idxd_device_reset(struct idxd_device *idxd)
{
unsigned long flags;
idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
spin_lock_irqsave(&idxd->dev_lock, flags);
idxd_device_wqs_clear_state(idxd);
idxd->state = IDXD_DEV_CONF_READY;
spin_unlock_irqrestore(&idxd->dev_lock, flags);
spin_lock(&idxd->dev_lock);
idxd_device_clear_state(idxd);
idxd->state = IDXD_DEV_DISABLED;
spin_unlock(&idxd->dev_lock);
}
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid)
@ -649,7 +634,6 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
struct device *dev = &idxd->pdev->dev;
u32 operand, status;
union idxd_command_reg cmd;
unsigned long flags;
if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)))
return -EOPNOTSUPP;
@ -667,13 +651,13 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand);
spin_lock_irqsave(&idxd->cmd_lock, flags);
spin_lock(&idxd->cmd_lock);
iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET);
while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE)
cpu_relax();
status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET);
spin_unlock_irqrestore(&idxd->cmd_lock, flags);
spin_unlock(&idxd->cmd_lock);
if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) {
dev_dbg(dev, "release int handle failed: %#x\n", status);
@ -685,6 +669,59 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
}
/* Device configuration bits */
static void idxd_engines_clear_state(struct idxd_device *idxd)
{
struct idxd_engine *engine;
int i;
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_engines; i++) {
engine = idxd->engines[i];
engine->group = NULL;
}
}
static void idxd_groups_clear_state(struct idxd_device *idxd)
{
struct idxd_group *group;
int i;
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_groups; i++) {
group = idxd->groups[i];
memset(&group->grpcfg, 0, sizeof(group->grpcfg));
group->num_engines = 0;
group->num_wqs = 0;
group->use_token_limit = false;
group->tokens_allowed = 0;
group->tokens_reserved = 0;
group->tc_a = -1;
group->tc_b = -1;
}
}
static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
{
int i;
lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
if (wq->state == IDXD_WQ_ENABLED) {
idxd_wq_disable_cleanup(wq);
wq->state = IDXD_WQ_DISABLED;
}
}
}
void idxd_device_clear_state(struct idxd_device *idxd)
{
idxd_groups_clear_state(idxd);
idxd_engines_clear_state(idxd);
idxd_device_wqs_clear_state(idxd);
}
void idxd_msix_perm_setup(struct idxd_device *idxd)
{
union msix_perm mperm;
@ -773,6 +810,15 @@ static int idxd_groups_config_write(struct idxd_device *idxd)
return 0;
}
static bool idxd_device_pasid_priv_enabled(struct idxd_device *idxd)
{
struct pci_dev *pdev = idxd->pdev;
if (pdev->pasid_enabled && (pdev->pasid_features & PCI_PASID_CAP_PRIV))
return true;
return false;
}
static int idxd_wq_config_write(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
@ -796,6 +842,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
wq->wqcfg->wq_size = wq->size;
if (wq->size == 0) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_SIZE;
dev_warn(dev, "Incorrect work queue size: 0\n");
return -EINVAL;
}
@ -804,7 +851,6 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
wq->wqcfg->wq_thresh = wq->threshold;
/* byte 8-11 */
wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
if (wq_dedicated(wq))
wq->wqcfg->mode = 1;
@ -814,6 +860,25 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
wq->wqcfg->pasid = idxd->pasid;
}
/*
* Here the priv bit is set depending on the WQ type. priv = 1 if the
* WQ type is kernel to indicate privileged access. This setting only
* matters for dedicated WQ. According to the DSA spec:
* If the WQ is in dedicated mode, WQ PASID Enable is 1, and the
* Privileged Mode Enable field of the PCI Express PASID capability
* is 0, this field must be 0.
*
* In the case of a dedicated kernel WQ that is not able to support
* the PASID cap, then the configuration will be rejected.
*/
wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
!idxd_device_pasid_priv_enabled(idxd) &&
wq->type == IDXD_WQT_KERNEL) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_PRIV;
return -EOPNOTSUPP;
}
wq->wqcfg->priority = wq->priority;
if (idxd->hw.gen_cap.block_on_fault &&
@ -931,6 +996,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
continue;
if (wq_shared(wq) && !device_swq_supported(idxd)) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT;
dev_warn(dev, "No shared wq support but configured.\n");
return -EINVAL;
}
@ -939,8 +1005,10 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
configured++;
}
if (configured == 0)
if (configured == 0) {
idxd->cmd_status = IDXD_SCMD_WQ_NONE_CONFIGURED;
return -EINVAL;
}
return 0;
}
@ -1086,3 +1154,203 @@ int idxd_device_load_config(struct idxd_device *idxd)
return 0;
}
int __drv_enable_wq(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
int rc = -ENXIO;
lockdep_assert_held(&wq->wq_lock);
if (idxd->state != IDXD_DEV_ENABLED) {
idxd->cmd_status = IDXD_SCMD_DEV_NOT_ENABLED;
goto err;
}
if (wq->state != IDXD_WQ_DISABLED) {
dev_dbg(dev, "wq %d already enabled.\n", wq->id);
idxd->cmd_status = IDXD_SCMD_WQ_ENABLED;
rc = -EBUSY;
goto err;
}
if (!wq->group) {
dev_dbg(dev, "wq %d not attached to group.\n", wq->id);
idxd->cmd_status = IDXD_SCMD_WQ_NO_GRP;
goto err;
}
if (strlen(wq->name) == 0) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_NAME;
dev_dbg(dev, "wq %d name not set.\n", wq->id);
goto err;
}
/* Shared WQ checks */
if (wq_shared(wq)) {
if (!device_swq_supported(idxd)) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM;
dev_dbg(dev, "PASID not enabled and shared wq.\n");
goto err;
}
/*
* Shared wq with the threshold set to 0 means the user
* did not set the threshold or transitioned from a
* dedicated wq but did not set threshold. A value
* of 0 would effectively disable the shared wq. The
* driver does not allow a value of 0 to be set for
* threshold via sysfs.
*/
if (wq->threshold == 0) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_THRESH;
dev_dbg(dev, "Shared wq and threshold 0.\n");
goto err;
}
}
rc = 0;
spin_lock(&idxd->dev_lock);
if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
rc = idxd_device_config(idxd);
spin_unlock(&idxd->dev_lock);
if (rc < 0) {
dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);
goto err;
}
rc = idxd_wq_enable(wq);
if (rc < 0) {
dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);
goto err;
}
rc = idxd_wq_map_portal(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_WQ_PORTAL_ERR;
dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);
goto err_map_portal;
}
wq->client_count = 0;
return 0;
err_map_portal:
rc = idxd_wq_disable(wq, false);
if (rc < 0)
dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));
err:
return rc;
}
int drv_enable_wq(struct idxd_wq *wq)
{
int rc;
mutex_lock(&wq->wq_lock);
rc = __drv_enable_wq(wq);
mutex_unlock(&wq->wq_lock);
return rc;
}
void __drv_disable_wq(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
lockdep_assert_held(&wq->wq_lock);
if (idxd_wq_refcount(wq))
dev_warn(dev, "Clients has claim on wq %d: %d\n",
wq->id, idxd_wq_refcount(wq));
idxd_wq_unmap_portal(wq);
idxd_wq_drain(wq);
idxd_wq_reset(wq);
wq->client_count = 0;
}
void drv_disable_wq(struct idxd_wq *wq)
{
mutex_lock(&wq->wq_lock);
__drv_disable_wq(wq);
mutex_unlock(&wq->wq_lock);
}
int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
{
struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
int rc = 0;
/*
* Device should be in disabled state for the idxd_drv to load. If it's in
* enabled state, then the device was altered outside of driver's control.
* If the state is in halted state, then we don't want to proceed.
*/
if (idxd->state != IDXD_DEV_DISABLED) {
idxd->cmd_status = IDXD_SCMD_DEV_ENABLED;
return -ENXIO;
}
/* Device configuration */
spin_lock(&idxd->dev_lock);
if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
rc = idxd_device_config(idxd);
spin_unlock(&idxd->dev_lock);
if (rc < 0)
return -ENXIO;
/* Start device */
rc = idxd_device_enable(idxd);
if (rc < 0)
return rc;
/* Setup DMA device without channels */
rc = idxd_register_dma_device(idxd);
if (rc < 0) {
idxd_device_disable(idxd);
idxd->cmd_status = IDXD_SCMD_DEV_DMA_ERR;
return rc;
}
idxd->cmd_status = 0;
return 0;
}
void idxd_device_drv_remove(struct idxd_dev *idxd_dev)
{
struct device *dev = &idxd_dev->conf_dev;
struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
int i;
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
struct device *wq_dev = wq_confdev(wq);
if (wq->state == IDXD_WQ_DISABLED)
continue;
dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev));
device_release_driver(wq_dev);
}
idxd_unregister_dma_device(idxd);
idxd_device_disable(idxd);
if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
idxd_device_reset(idxd);
}
static enum idxd_dev_type dev_types[] = {
IDXD_DEV_DSA,
IDXD_DEV_IAX,
IDXD_DEV_NONE,
};
struct idxd_device_driver idxd_drv = {
.type = dev_types,
.probe = idxd_device_drv_probe,
.remove = idxd_device_drv_remove,
.name = "idxd",
};
EXPORT_SYMBOL_GPL(idxd_drv);

96
drivers/dma/idxd/dma.c
View File

@ -69,7 +69,11 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
hw->src_addr = addr_f1;
hw->dst_addr = addr_f2;
hw->xfer_size = len;
hw->priv = !!(wq->type == IDXD_WQT_KERNEL);
/*
* For dedicated WQ, this field is ignored and HW will use the WQCFG.priv
* field instead. This field should be set to 1 for kernel descriptors.
*/
hw->priv = 1;
hw->completion_addr = compl;
}
@ -149,10 +153,8 @@ static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
cookie = dma_cookie_assign(tx);
rc = idxd_submit_desc(wq, desc);
if (rc < 0) {
idxd_free_desc(wq, desc);
if (rc < 0)
return rc;
}
return cookie;
}
@ -245,7 +247,7 @@ int idxd_register_dma_channel(struct idxd_wq *wq)
wq->idxd_chan = idxd_chan;
idxd_chan->wq = wq;
get_device(&wq->conf_dev);
get_device(wq_confdev(wq));
return 0;
}
@ -260,5 +262,87 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq)
list_del(&chan->device_node);
kfree(wq->idxd_chan);
wq->idxd_chan = NULL;
put_device(&wq->conf_dev);
put_device(wq_confdev(wq));
}
static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
{
struct device *dev = &idxd_dev->conf_dev;
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
struct idxd_device *idxd = wq->idxd;
int rc;
if (idxd->state != IDXD_DEV_ENABLED)
return -ENXIO;
mutex_lock(&wq->wq_lock);
wq->type = IDXD_WQT_KERNEL;
rc = __drv_enable_wq(wq);
if (rc < 0) {
dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
rc = -ENXIO;
goto err;
}
rc = idxd_wq_alloc_resources(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
dev_dbg(dev, "WQ resource alloc failed\n");
goto err_res_alloc;
}
rc = idxd_wq_init_percpu_ref(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
dev_dbg(dev, "percpu_ref setup failed\n");
goto err_ref;
}
rc = idxd_register_dma_channel(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
dev_dbg(dev, "Failed to register dma channel\n");
goto err_dma;
}
idxd->cmd_status = 0;
mutex_unlock(&wq->wq_lock);
return 0;
err_dma:
idxd_wq_quiesce(wq);
err_ref:
idxd_wq_free_resources(wq);
err_res_alloc:
__drv_disable_wq(wq);
err:
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
return rc;
}
static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
{
struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
mutex_lock(&wq->wq_lock);
idxd_wq_quiesce(wq);
idxd_unregister_dma_channel(wq);
__drv_disable_wq(wq);
idxd_wq_free_resources(wq);
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
}
static enum idxd_dev_type dev_types[] = {
IDXD_DEV_WQ,
IDXD_DEV_NONE,
};
struct idxd_device_driver idxd_dmaengine_drv = {
.probe = idxd_dmaengine_drv_probe,
.remove = idxd_dmaengine_drv_remove,
.name = "dmaengine",
.type = dev_types,
};
EXPORT_SYMBOL_GPL(idxd_dmaengine_drv);

167
drivers/dma/idxd/idxd.h
View File

@ -11,14 +11,32 @@
#include <linux/idr.h>
#include <linux/pci.h>
#include <linux/perf_event.h>
#include <uapi/linux/idxd.h>
#include "registers.h"
#define IDXD_DRIVER_VERSION "1.00"
extern struct kmem_cache *idxd_desc_pool;
extern bool tc_override;
struct idxd_device;
struct idxd_wq;
struct idxd_dev;
enum idxd_dev_type {
IDXD_DEV_NONE = -1,
IDXD_DEV_DSA = 0,
IDXD_DEV_IAX,
IDXD_DEV_WQ,
IDXD_DEV_GROUP,
IDXD_DEV_ENGINE,
IDXD_DEV_CDEV,
IDXD_DEV_MAX_TYPE,
};
struct idxd_dev {
struct device conf_dev;
enum idxd_dev_type type;
};
#define IDXD_REG_TIMEOUT 50
#define IDXD_DRAIN_TIMEOUT 5000
@ -34,9 +52,18 @@ enum idxd_type {
#define IDXD_PMU_EVENT_MAX 64
struct idxd_device_driver {
const char *name;
enum idxd_dev_type *type;
int (*probe)(struct idxd_dev *idxd_dev);
void (*remove)(struct idxd_dev *idxd_dev);
struct device_driver drv;
};
extern struct idxd_device_driver dsa_drv;
extern struct idxd_device_driver idxd_drv;
extern struct idxd_device_driver idxd_dmaengine_drv;
extern struct idxd_device_driver idxd_user_drv;
struct idxd_irq_entry {
struct idxd_device *idxd;
int id;
@ -51,7 +78,7 @@ struct idxd_irq_entry {
};
struct idxd_group {
struct device conf_dev;
struct idxd_dev idxd_dev;
struct idxd_device *idxd;
struct grpcfg grpcfg;
int id;
@ -110,7 +137,7 @@ enum idxd_wq_type {
struct idxd_cdev {
struct idxd_wq *wq;
struct cdev cdev;
struct device dev;
struct idxd_dev idxd_dev;
int minor;
};
@ -136,9 +163,10 @@ struct idxd_dma_chan {
struct idxd_wq {
void __iomem *portal;
u32 portal_offset;
struct percpu_ref wq_active;
struct completion wq_dead;
struct device conf_dev;
struct idxd_dev idxd_dev;
struct idxd_cdev *idxd_cdev;
struct wait_queue_head err_queue;
struct idxd_device *idxd;
@ -153,7 +181,6 @@ struct idxd_wq {
enum idxd_wq_state state;
unsigned long flags;
union wqcfg *wqcfg;
u32 vec_ptr; /* interrupt steering */
struct dsa_hw_desc **hw_descs;
int num_descs;
union {
@ -174,7 +201,7 @@ struct idxd_wq {
};
struct idxd_engine {
struct device conf_dev;
struct idxd_dev idxd_dev;
int id;
struct idxd_group *group;
struct idxd_device *idxd;
@ -194,7 +221,6 @@ struct idxd_hw {
enum idxd_device_state {
IDXD_DEV_HALTED = -1,
IDXD_DEV_DISABLED = 0,
IDXD_DEV_CONF_READY,
IDXD_DEV_ENABLED,
};
@ -218,7 +244,7 @@ struct idxd_driver_data {
};
struct idxd_device {
struct device conf_dev;
struct idxd_dev idxd_dev;
struct idxd_driver_data *data;
struct list_head list;
struct idxd_hw hw;
@ -226,7 +252,7 @@ struct idxd_device {
unsigned long flags;
int id;
int major;
u8 cmd_status;
u32 cmd_status;
struct pci_dev *pdev;
void __iomem *reg_base;
@ -290,7 +316,6 @@ struct idxd_desc {
struct list_head list;
int id;
int cpu;
unsigned int vector;
struct idxd_wq *wq;
};
@ -302,11 +327,62 @@ enum idxd_completion_status {
IDXD_COMP_DESC_ABORT = 0xff,
};
#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
#define wq_confdev(wq) &wq->idxd_dev.conf_dev
#define engine_confdev(engine) &engine->idxd_dev.conf_dev
#define group_confdev(group) &group->idxd_dev.conf_dev
#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
static inline struct idxd_device *confdev_to_idxd(struct device *dev)
{
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return idxd_dev_to_idxd(idxd_dev);
}
static inline struct idxd_wq *confdev_to_wq(struct device *dev)
{
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return idxd_dev_to_wq(idxd_dev);
}
static inline struct idxd_engine *confdev_to_engine(struct device *dev)
{
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return container_of(idxd_dev, struct idxd_engine, idxd_dev);
}
static inline struct idxd_group *confdev_to_group(struct device *dev)
{
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return container_of(idxd_dev, struct idxd_group, idxd_dev);
}
static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
{
struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
}
static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
{
if (type >= IDXD_DEV_MAX_TYPE) {
idev->type = IDXD_DEV_NONE;
return;
}
idev->type = type;
}
extern struct bus_type dsa_bus_type;
extern struct bus_type iax_bus_type;
extern bool support_enqcmd;
extern struct ida idxd_ida;
@ -316,24 +392,24 @@ extern struct device_type idxd_wq_device_type;
extern struct device_type idxd_engine_device_type;
extern struct device_type idxd_group_device_type;
static inline bool is_dsa_dev(struct device *dev)
static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
{
return dev->type == &dsa_device_type;
return idxd_dev->type == IDXD_DEV_DSA;
}
static inline bool is_iax_dev(struct device *dev)
static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
{
return dev->type == &iax_device_type;
return idxd_dev->type == IDXD_DEV_IAX;
}
static inline bool is_idxd_dev(struct device *dev)
static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
{
return is_dsa_dev(dev) || is_iax_dev(dev);
return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
}
static inline bool is_idxd_wq_dev(struct device *dev)
static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
{
return dev->type == &idxd_wq_device_type;
return idxd_dev->type == IDXD_DEV_WQ;
}
static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
@ -343,11 +419,16 @@ static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
return false;
}
static inline bool is_idxd_wq_cdev(struct idxd_wq *wq)
static inline bool is_idxd_wq_user(struct idxd_wq *wq)
{
return wq->type == IDXD_WQT_USER;
}
static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
{
return wq->type == IDXD_WQT_KERNEL;
}
static inline bool wq_dedicated(struct idxd_wq *wq)
{
return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
@ -389,6 +470,24 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id,
return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
}
#define IDXD_PORTAL_MASK (PAGE_SIZE - 1)
/*
* Even though this function can be accessed by multiple threads, it is safe to use.
* At worst the address gets used more than once before it gets incremented. We don't
* hit a threshold until iops becomes many million times a second. So the occasional
* reuse of the same address is tolerable compare to using an atomic variable. This is
* safe on a system that has atomic load/store for 32bit integers. Given that this is an
* Intel iEP device, that should not be a problem.
*/
static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
{
int ofs = wq->portal_offset;
wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
return wq->portal + ofs;
}
static inline void idxd_wq_get(struct idxd_wq *wq)
{
wq->client_count++;
@ -404,6 +503,16 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq)
return wq->client_count;
};
int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
struct module *module, const char *mod_name);
#define idxd_driver_register(driver) \
__idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
#define module_idxd_driver(__idxd_driver) \
module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
int idxd_register_bus_type(void);
void idxd_unregister_bus_type(void);
int idxd_register_devices(struct idxd_device *idxd);
@ -424,13 +533,20 @@ void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
/* device control */
int idxd_register_idxd_drv(void);
void idxd_unregister_idxd_drv(void);
int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
int drv_enable_wq(struct idxd_wq *wq);
int __drv_enable_wq(struct idxd_wq *wq);
void drv_disable_wq(struct idxd_wq *wq);
void __drv_disable_wq(struct idxd_wq *wq);
int idxd_device_init_reset(struct idxd_device *idxd);
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
void idxd_device_reset(struct idxd_device *idxd);
void idxd_device_cleanup(struct idxd_device *idxd);
void idxd_device_clear_state(struct idxd_device *idxd);
int idxd_device_config(struct idxd_device *idxd);
void idxd_device_wqs_clear_state(struct idxd_device *idxd);
void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
int idxd_device_load_config(struct idxd_device *idxd);
int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
@ -443,12 +559,11 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd);
int idxd_wq_alloc_resources(struct idxd_wq *wq);
void idxd_wq_free_resources(struct idxd_wq *wq);
int idxd_wq_enable(struct idxd_wq *wq);
int idxd_wq_disable(struct idxd_wq *wq);
int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
void idxd_wq_drain(struct idxd_wq *wq);
void idxd_wq_reset(struct idxd_wq *wq);
int idxd_wq_map_portal(struct idxd_wq *wq);
void idxd_wq_unmap_portal(struct idxd_wq *wq);
void idxd_wq_disable_cleanup(struct idxd_wq *wq);
int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
int idxd_wq_disable_pasid(struct idxd_wq *wq);
void idxd_wq_quiesce(struct idxd_wq *wq);

150
drivers/dma/idxd/init.c
View File

@ -26,11 +26,16 @@
MODULE_VERSION(IDXD_DRIVER_VERSION);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Intel Corporation");
MODULE_IMPORT_NS(IDXD);
static bool sva = true;
module_param(sva, bool, 0644);
MODULE_PARM_DESC(sva, "Toggle SVA support on/off");
bool tc_override;
module_param(tc_override, bool, 0644);
MODULE_PARM_DESC(tc_override, "Override traffic class defaults");
#define DRV_NAME "idxd"
bool support_enqcmd;
@ -200,6 +205,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
struct idxd_wq *wq;
struct device *conf_dev;
int i, rc;
idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *),
@ -214,15 +220,17 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
goto err;
}
idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ);
conf_dev = wq_confdev(wq);
wq->id = i;
wq->idxd = idxd;
device_initialize(&wq->conf_dev);
wq->conf_dev.parent = &idxd->conf_dev;
wq->conf_dev.bus = &dsa_bus_type;
wq->conf_dev.type = &idxd_wq_device_type;
rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id);
device_initialize(wq_confdev(wq));
conf_dev->parent = idxd_confdev(idxd);
conf_dev->bus = &dsa_bus_type;
conf_dev->type = &idxd_wq_device_type;
rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id);
if (rc < 0) {
put_device(&wq->conf_dev);
put_device(conf_dev);
goto err;
}
@ -233,7 +241,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
wq->max_batch_size = idxd->max_batch_size;
wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));
if (!wq->wqcfg) {
put_device(&wq->conf_dev);
put_device(conf_dev);
rc = -ENOMEM;
goto err;
}
@ -243,8 +251,11 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
return 0;
err:
while (--i >= 0)
put_device(&idxd->wqs[i]->conf_dev);
while (--i >= 0) {
wq = idxd->wqs[i];
conf_dev = wq_confdev(wq);
put_device(conf_dev);
}
return rc;
}
@ -252,6 +263,7 @@ static int idxd_setup_engines(struct idxd_device *idxd)
{
struct idxd_engine *engine;
struct device *dev = &idxd->pdev->dev;
struct device *conf_dev;
int i, rc;
idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *),
@ -266,15 +278,17 @@ static int idxd_setup_engines(struct idxd_device *idxd)
goto err;
}
idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE);
conf_dev = engine_confdev(engine);
engine->id = i;
engine->idxd = idxd;
device_initialize(&engine->conf_dev);
engine->conf_dev.parent = &idxd->conf_dev;
engine->conf_dev.bus = &dsa_bus_type;
engine->conf_dev.type = &idxd_engine_device_type;
rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id);
device_initialize(conf_dev);
conf_dev->parent = idxd_confdev(idxd);
conf_dev->bus = &dsa_bus_type;
conf_dev->type = &idxd_engine_device_type;
rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id);
if (rc < 0) {
put_device(&engine->conf_dev);
put_device(conf_dev);
goto err;
}
@ -284,14 +298,18 @@ static int idxd_setup_engines(struct idxd_device *idxd)
return 0;
err:
while (--i >= 0)
put_device(&idxd->engines[i]->conf_dev);
while (--i >= 0) {
engine = idxd->engines[i];
conf_dev = engine_confdev(engine);
put_device(conf_dev);
}
return rc;
}
static int idxd_setup_groups(struct idxd_device *idxd)
{
struct device *dev = &idxd->pdev->dev;
struct device *conf_dev;
struct idxd_group *group;
int i, rc;
@ -307,28 +325,37 @@ static int idxd_setup_groups(struct idxd_device *idxd)
goto err;
}
idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP);
conf_dev = group_confdev(group);
group->id = i;
group->idxd = idxd;
device_initialize(&group->conf_dev);
group->conf_dev.parent = &idxd->conf_dev;
group->conf_dev.bus = &dsa_bus_type;
group->conf_dev.type = &idxd_group_device_type;
rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id);
device_initialize(conf_dev);
conf_dev->parent = idxd_confdev(idxd);
conf_dev->bus = &dsa_bus_type;
conf_dev->type = &idxd_group_device_type;
rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id);
if (rc < 0) {
put_device(&group->conf_dev);
put_device(conf_dev);
goto err;
}
idxd->groups[i] = group;
group->tc_a = -1;
group->tc_b = -1;
if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) {
group->tc_a = 1;
group->tc_b = 1;
} else {
group->tc_a = -1;
group->tc_b = -1;
}
}
return 0;
err:
while (--i >= 0)
put_device(&idxd->groups[i]->conf_dev);
while (--i >= 0) {
group = idxd->groups[i];
put_device(group_confdev(group));
}
return rc;
}
@ -337,11 +364,11 @@ static void idxd_cleanup_internals(struct idxd_device *idxd)
int i;
for (i = 0; i < idxd->max_groups; i++)
put_device(&idxd->groups[i]->conf_dev);
put_device(group_confdev(idxd->groups[i]));
for (i = 0; i < idxd->max_engines; i++)
put_device(&idxd->engines[i]->conf_dev);
put_device(engine_confdev(idxd->engines[i]));
for (i = 0; i < idxd->max_wqs; i++)
put_device(&idxd->wqs[i]->conf_dev);
put_device(wq_confdev(idxd->wqs[i]));
destroy_workqueue(idxd->wq);
}
@ -381,13 +408,13 @@ static int idxd_setup_internals(struct idxd_device *idxd)
err_wkq_create:
for (i = 0; i < idxd->max_groups; i++)
put_device(&idxd->groups[i]->conf_dev);
put_device(group_confdev(idxd->groups[i]));
err_group:
for (i = 0; i < idxd->max_engines; i++)
put_device(&idxd->engines[i]->conf_dev);
put_device(engine_confdev(idxd->engines[i]));
err_engine:
for (i = 0; i < idxd->max_wqs; i++)
put_device(&idxd->wqs[i]->conf_dev);
put_device(wq_confdev(idxd->wqs[i]));
err_wqs:
kfree(idxd->int_handles);
return rc;
@ -469,6 +496,7 @@ static void idxd_read_caps(struct idxd_device *idxd)
static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data)
{
struct device *dev = &pdev->dev;
struct device *conf_dev;
struct idxd_device *idxd;
int rc;
@ -476,19 +504,21 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d
if (!idxd)
return NULL;
conf_dev = idxd_confdev(idxd);
idxd->pdev = pdev;
idxd->data = data;
idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type);
idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL);
if (idxd->id < 0)
return NULL;
device_initialize(&idxd->conf_dev);
idxd->conf_dev.parent = dev;
idxd->conf_dev.bus = &dsa_bus_type;
idxd->conf_dev.type = idxd->data->dev_type;
rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
device_initialize(conf_dev);
conf_dev->parent = dev;
conf_dev->bus = &dsa_bus_type;
conf_dev->type = idxd->data->dev_type;
rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id);
if (rc < 0) {
put_device(&idxd->conf_dev);
put_device(conf_dev);
return NULL;
}
@ -639,15 +669,9 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
}
dev_dbg(dev, "Set DMA masks\n");
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (rc)
goto err;
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
if (rc)
rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (rc)
goto err;
@ -668,8 +692,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
goto err_dev_register;
}
idxd->state = IDXD_DEV_CONF_READY;
dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n",
idxd->hw.version);
@ -680,7 +702,7 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
err:
pci_iounmap(pdev, idxd->reg_base);
err_iomap:
put_device(&idxd->conf_dev);
put_device(idxd_confdev(idxd));
err_idxd_alloc:
pci_disable_device(pdev);
return rc;
@ -793,7 +815,7 @@ static void idxd_remove(struct pci_dev *pdev)
pci_disable_device(pdev);
destroy_workqueue(idxd->wq);
perfmon_pmu_remove(idxd);
device_unregister(&idxd->conf_dev);
device_unregister(idxd_confdev(idxd));
}
static struct pci_driver idxd_pci_driver = {
@ -824,14 +846,18 @@ static int __init idxd_init_module(void)
perfmon_init();
err = idxd_register_bus_type();
if (err < 0)
return err;
err = idxd_register_driver();
err = idxd_driver_register(&idxd_drv);
if (err < 0)
goto err_idxd_driver_register;
err = idxd_driver_register(&idxd_dmaengine_drv);
if (err < 0)
goto err_idxd_dmaengine_driver_register;
err = idxd_driver_register(&idxd_user_drv);
if (err < 0)
goto err_idxd_user_driver_register;
err = idxd_cdev_register();
if (err)
goto err_cdev_register;
@ -845,19 +871,23 @@ static int __init idxd_init_module(void)
err_pci_register:
idxd_cdev_remove();
err_cdev_register:
idxd_unregister_driver();
idxd_driver_unregister(&idxd_user_drv);
err_idxd_user_driver_register:
idxd_driver_unregister(&idxd_dmaengine_drv);
err_idxd_dmaengine_driver_register:
idxd_driver_unregister(&idxd_drv);
err_idxd_driver_register:
idxd_unregister_bus_type();
return err;
}
module_init(idxd_init_module);
static void __exit idxd_exit_module(void)
{
idxd_unregister_driver();
idxd_driver_unregister(&idxd_user_drv);
idxd_driver_unregister(&idxd_dmaengine_drv);
idxd_driver_unregister(&idxd_drv);
pci_unregister_driver(&idxd_pci_driver);
idxd_cdev_remove();
idxd_unregister_bus_type();
perfmon_exit();
}
module_exit(idxd_exit_module);

190
drivers/dma/idxd/irq.c
View File

@ -22,13 +22,6 @@ struct idxd_fault {
struct idxd_device *idxd;
};
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data);
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data);
static void idxd_device_reinit(struct work_struct *work)
{
struct idxd_device *idxd = container_of(work, struct idxd_device, work);
@ -51,7 +44,7 @@ static void idxd_device_reinit(struct work_struct *work)
rc = idxd_wq_enable(wq);
if (rc < 0) {
dev_warn(dev, "Unable to re-enable wq %s\n",
dev_name(&wq->conf_dev));
dev_name(wq_confdev(wq)));
}
}
}
@ -59,47 +52,7 @@ static void idxd_device_reinit(struct work_struct *work)
return;
out:
idxd_device_wqs_clear_state(idxd);
}
static void idxd_device_fault_work(struct work_struct *work)
{
struct idxd_fault *fault = container_of(work, struct idxd_fault, work);
struct idxd_irq_entry *ie;
int i;
int processed;
int irqcnt = fault->idxd->num_wq_irqs + 1;
for (i = 1; i < irqcnt; i++) {
ie = &fault->idxd->irq_entries[i];
irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT,
&processed, fault->addr);
if (processed)
break;
irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT,
&processed, fault->addr);
if (processed)
break;
}
kfree(fault);
}
static int idxd_device_schedule_fault_process(struct idxd_device *idxd,
u64 fault_addr)
{
struct idxd_fault *fault;
fault = kmalloc(sizeof(*fault), GFP_ATOMIC);
if (!fault)
return -ENOMEM;
fault->addr = fault_addr;
fault->idxd = idxd;
INIT_WORK(&fault->work, idxd_device_fault_work);
queue_work(idxd->wq, &fault->work);
return 0;
idxd_device_clear_state(idxd);
}
static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
@ -111,7 +64,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
bool err = false;
if (cause & IDXD_INTC_ERR) {
spin_lock_bh(&idxd->dev_lock);
spin_lock(&idxd->dev_lock);
for (i = 0; i < 4; i++)
idxd->sw_err.bits[i] = ioread64(idxd->reg_base +
IDXD_SWERR_OFFSET + i * sizeof(u64));
@ -136,7 +89,7 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
}
}
spin_unlock_bh(&idxd->dev_lock);
spin_unlock(&idxd->dev_lock);
val |= IDXD_INTC_ERR;
for (i = 0; i < 4; i++)
@ -168,15 +121,6 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
if (!err)
return 0;
/*
* This case should rarely happen and typically is due to software
* programming error by the driver.
*/
if (idxd->sw_err.valid &&
idxd->sw_err.desc_valid &&
idxd->sw_err.fault_addr)
idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr);
gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);
if (gensts.state == IDXD_DEVICE_STATE_HALT) {
idxd->state = IDXD_DEV_HALTED;
@ -189,15 +133,15 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause)
INIT_WORK(&idxd->work, idxd_device_reinit);
queue_work(idxd->wq, &idxd->work);
} else {
spin_lock_bh(&idxd->dev_lock);
spin_lock(&idxd->dev_lock);
idxd_wqs_quiesce(idxd);
idxd_wqs_unmap_portal(idxd);
idxd_device_wqs_clear_state(idxd);
idxd_device_clear_state(idxd);
dev_err(&idxd->pdev->dev,
"idxd halted, need %s.\n",
gensts.reset_type == IDXD_DEVICE_RESET_FLR ?
"FLR" : "system reset");
spin_unlock_bh(&idxd->dev_lock);
spin_unlock(&idxd->dev_lock);
return -ENXIO;
}
}
@ -228,127 +172,79 @@ irqreturn_t idxd_misc_thread(int vec, void *data)
return IRQ_HANDLED;
}
static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr)
{
/*
* Completion address can be bad as well. Check fault address match for descriptor
* and completion address.
*/
if ((u64)desc->hw == fault_addr || (u64)desc->completion == fault_addr) {
struct idxd_device *idxd = desc->wq->idxd;
struct device *dev = &idxd->pdev->dev;
dev_warn(dev, "desc with fault address: %#llx\n", fault_addr);
return true;
}
return false;
}
static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data)
static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry)
{
struct idxd_desc *desc, *t;
struct llist_node *head;
int queued = 0;
unsigned long flags;
enum idxd_complete_type reason;
*processed = 0;
head = llist_del_all(&irq_entry->pending_llist);
if (!head)
goto out;
if (wtype == IRQ_WORK_NORMAL)
reason = IDXD_COMPLETE_NORMAL;
else
reason = IDXD_COMPLETE_DEV_FAIL;
return;
llist_for_each_entry_safe(desc, t, head, llnode) {
u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;
if (status) {
if (unlikely(status == IDXD_COMP_DESC_ABORT)) {
/*
* Check against the original status as ABORT is software defined
* and 0xff, which DSA_COMP_STATUS_MASK can mask out.
*/
if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {
complete_desc(desc, IDXD_COMPLETE_ABORT);
(*processed)++;
continue;
}
if (unlikely(status != DSA_COMP_SUCCESS))
match_fault(desc, data);
complete_desc(desc, reason);
(*processed)++;
complete_desc(desc, IDXD_COMPLETE_NORMAL);
} else {
spin_lock_irqsave(&irq_entry->list_lock, flags);
spin_lock(&irq_entry->list_lock);
list_add_tail(&desc->list,
&irq_entry->work_list);
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
queued++;
spin_unlock(&irq_entry->list_lock);
}
}
out:
return queued;
}
static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
enum irq_work_type wtype,
int *processed, u64 data)
static void irq_process_work_list(struct idxd_irq_entry *irq_entry)
{
int queued = 0;
unsigned long flags;
LIST_HEAD(flist);
struct idxd_desc *desc, *n;
enum idxd_complete_type reason;
*processed = 0;
if (wtype == IRQ_WORK_NORMAL)
reason = IDXD_COMPLETE_NORMAL;
else
reason = IDXD_COMPLETE_DEV_FAIL;
/*
* This lock protects list corruption from access of list outside of the irq handler
* thread.
*/
spin_lock_irqsave(&irq_entry->list_lock, flags);
spin_lock(&irq_entry->list_lock);
if (list_empty(&irq_entry->work_list)) {
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
return 0;
spin_unlock(&irq_entry->list_lock);
return;
}
list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) {
if (desc->completion->status) {
list_del(&desc->list);
(*processed)++;
list_add_tail(&desc->list, &flist);
} else {
queued++;
}
}
spin_unlock_irqrestore(&irq_entry->list_lock, flags);
spin_unlock(&irq_entry->list_lock);
list_for_each_entry(desc, &flist, list) {
u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;
if (unlikely(status == IDXD_COMP_DESC_ABORT)) {
/*
* Check against the original status as ABORT is software defined
* and 0xff, which DSA_COMP_STATUS_MASK can mask out.
*/
if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {
complete_desc(desc, IDXD_COMPLETE_ABORT);
continue;
}
if (unlikely(status != DSA_COMP_SUCCESS))
match_fault(desc, data);
complete_desc(desc, reason);
complete_desc(desc, IDXD_COMPLETE_NORMAL);
}
return queued;
}
static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
irqreturn_t idxd_wq_thread(int irq, void *data)
{
int rc, processed, total = 0;
struct idxd_irq_entry *irq_entry = data;
/*
* There are two lists we are processing. The pending_llist is where
@ -367,31 +263,9 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
* and process the completed entries.
* 4. If the entry is still waiting on hardware, list_add_tail() to
* the work_list.
* 5. Repeat until no more descriptors.
*/
do {
rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL,
&processed, 0);
total += processed;
if (rc != 0)
continue;
rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL,
&processed, 0);
total += processed;
} while (rc != 0);
return total;
}
irqreturn_t idxd_wq_thread(int irq, void *data)
{
struct idxd_irq_entry *irq_entry = data;
int processed;
processed = idxd_desc_process(irq_entry);
if (processed == 0)
return IRQ_NONE;
irq_process_work_list(irq_entry);
irq_process_pending_llist(irq_entry);
return IRQ_HANDLED;
}

6
drivers/dma/idxd/registers.h
View File

@ -7,6 +7,9 @@
#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25
#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe
#define DEVICE_VERSION_1 0x100
#define DEVICE_VERSION_2 0x200
#define IDXD_MMIO_BAR 0
#define IDXD_WQ_BAR 2
#define IDXD_PORTAL_SIZE PAGE_SIZE
@ -349,6 +352,9 @@ union wqcfg {
} __packed;
#define WQCFG_PASID_IDX 2
#define WQCFG_OCCUP_IDX 6
#define WQCFG_OCCUP_MASK 0xffff
/*
* This macro calculates the offset into the WQCFG register

43
drivers/dma/idxd/submit.c
View File

@ -22,21 +22,13 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu)
desc->hw->pasid = idxd->pasid;
/*
* Descriptor completion vectors are 1...N for MSIX. We will round
* robin through the N vectors.
* On host, MSIX vecotr 0 is used for misc interrupt. Therefore when we match
* vector 1:1 to the WQ id, we need to add 1
*/
wq->vec_ptr = desc->vector = (wq->vec_ptr % idxd->num_wq_irqs) + 1;
if (!idxd->int_handles) {
desc->hw->int_handle = wq->vec_ptr;
} else {
/*
* int_handles are only for descriptor completion. However for device
* MSIX enumeration, vec 0 is used for misc interrupts. Therefore even
* though we are rotating through 1...N for descriptor interrupts, we
* need to acqurie the int_handles from 0..N-1.
*/
desc->hw->int_handle = idxd->int_handles[desc->vector - 1];
}
if (!idxd->int_handles)
desc->hw->int_handle = wq->id + 1;
else
desc->hw->int_handle = idxd->int_handles[wq->id];
return desc;
}
@ -67,7 +59,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype)
if (signal_pending_state(TASK_INTERRUPTIBLE, current))
break;
idx = sbitmap_queue_get(sbq, &cpu);
if (idx > 0)
if (idx >= 0)
break;
schedule();
}
@ -114,14 +106,13 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
{
struct idxd_desc *d, *t, *found = NULL;
struct llist_node *head;
unsigned long flags;
desc->completion->status = IDXD_COMP_DESC_ABORT;
/*
* Grab the list lock so it will block the irq thread handler. This allows the
* abort code to locate the descriptor need to be aborted.
*/
spin_lock_irqsave(&ie->list_lock, flags);
spin_lock(&ie->list_lock);
head = llist_del_all(&ie->pending_llist);
if (head) {
llist_for_each_entry_safe(d, t, head, llnode) {
@ -135,7 +126,7 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie,
if (!found)
found = list_abort_desc(wq, ie, desc);
spin_unlock_irqrestore(&ie->list_lock, flags);
spin_unlock(&ie->list_lock);
if (found)
complete_desc(found, IDXD_COMPLETE_ABORT);
@ -148,13 +139,17 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
void __iomem *portal;
int rc;
if (idxd->state != IDXD_DEV_ENABLED)
if (idxd->state != IDXD_DEV_ENABLED) {
idxd_free_desc(wq, desc);
return -EIO;
}
if (!percpu_ref_tryget_live(&wq->wq_active))
if (!percpu_ref_tryget_live(&wq->wq_active)) {
idxd_free_desc(wq, desc);
return -ENXIO;
}
portal = wq->portal;
portal = idxd_wq_portal_addr(wq);
/*
* The wmb() flushes writes to coherent DMA data before
@ -168,7 +163,7 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
* that we designated the descriptor to.
*/
if (desc->hw->flags & IDXD_OP_FLAG_RCI) {
ie = &idxd->irq_entries[desc->vector];
ie = &idxd->irq_entries[wq->id + 1];
llist_add(&desc->llnode, &ie->pending_llist);
}
@ -183,8 +178,12 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc)
*/
rc = enqcmds(portal, desc->hw);
if (rc < 0) {
percpu_ref_put(&wq->wq_active);
/* abort operation frees the descriptor */
if (ie)
llist_abort_desc(wq, ie, desc);
else
idxd_free_desc(wq, desc);
return rc;
}
}

601
drivers/dma/idxd/sysfs.c
View File

File diff suppressed because it is too large Load Diff

12
drivers/dma/ppc4xx/adma.c
View File

@ -4319,6 +4319,7 @@ static ssize_t enable_store(struct device_driver *dev, const char *buf,
size_t count)
{
unsigned long val;
int err;
if (!count || count > 11)
return -EINVAL;
@ -4327,7 +4328,10 @@ static ssize_t enable_store(struct device_driver *dev, const char *buf,
return -EFAULT;
/* Write a key */
sscanf(buf, "%lx", &val);
err = kstrtoul(buf, 16, &val);
if (err)
return err;
dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
isync();
@ -4368,7 +4372,7 @@ static ssize_t poly_store(struct device_driver *dev, const char *buf,
size_t count)
{
unsigned long reg, val;
int err;
#ifdef CONFIG_440SP
/* 440SP uses default 0x14D polynomial only */
return -EINVAL;
@ -4378,7 +4382,9 @@ static ssize_t poly_store(struct device_driver *dev, const char *buf,
return -EINVAL;
/* e.g., 0x14D or 0x11D */
sscanf(buf, "%lx", &val);
err = kstrtoul(buf, 16, &val);
if (err)
return err;
if (val & ~0x1FF)
return -EINVAL;

13
drivers/dma/ptdma/Kconfig Normal file
View File

@ -0,0 +1,13 @@
# SPDX-License-Identifier: GPL-2.0-only
config AMD_PTDMA
tristate "AMD PassThru DMA Engine"
depends on X86_64 && PCI
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
Enable support for the AMD PTDMA controller. This controller
provides DMA capabilities to perform high bandwidth memory to
memory and IO copy operations. It performs DMA transfer through
queue-based descriptor management. This DMA controller is intended
to be used with AMD Non-Transparent Bridge devices and not for
general purpose peripheral DMA.

10
drivers/dma/ptdma/Makefile Normal file
View File

@ -0,0 +1,10 @@
# SPDX-License-Identifier: GPL-2.0-only
#
# AMD Passthru DMA driver
#
obj-$(CONFIG_AMD_PTDMA) += ptdma.o
ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o
ptdma-$(CONFIG_PCI) += ptdma-pci.o

106
drivers/dma/ptdma/ptdma-debugfs.c Normal file
View File

@ -0,0 +1,106 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* AMD Passthrough DMA device driver
* -- Based on the CCP driver
*
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
*
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include "ptdma.h"
/* DebugFS helpers */
#define RI_VERSION_NUM 0x0000003F
#define RI_NUM_VQM 0x00078000
#define RI_NVQM_SHIFT 15
static int pt_debugfs_info_show(struct seq_file *s, void *p)
{
struct pt_device *pt = s->private;
unsigned int regval;
seq_printf(s, "Device name: %s\n", dev_name(pt->dev));
seq_printf(s, " # Queues: %d\n", 1);
seq_printf(s, " # Cmds: %d\n", pt->cmd_count);
regval = ioread32(pt->io_regs + CMD_PT_VERSION);
seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM);
seq_puts(s, " Engines:");
seq_puts(s, "\n");
seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT);
return 0;
}
/*
* Return a formatted buffer containing the current
* statistics of queue for PTDMA
*/
static int pt_debugfs_stats_show(struct seq_file *s, void *p)
{
struct pt_device *pt = s->private;
seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts);
return 0;
}
static int pt_debugfs_queue_show(struct seq_file *s, void *p)
{
struct pt_cmd_queue *cmd_q = s->private;
unsigned int regval;
if (!cmd_q)
return 0;
seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops);
regval = ioread32(cmd_q->reg_control + 0x000C);
seq_puts(s, " Enabled Interrupts:");
if (regval & INT_EMPTY_QUEUE)
seq_puts(s, " EMPTY");
if (regval & INT_QUEUE_STOPPED)
seq_puts(s, " STOPPED");
if (regval & INT_ERROR)
seq_puts(s, " ERROR");
if (regval & INT_COMPLETION)
seq_puts(s, " COMPLETION");
seq_puts(s, "\n");
return 0;
}
DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info);
DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue);
DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats);
void ptdma_debugfs_setup(struct pt_device *pt)
{
struct pt_cmd_queue *cmd_q;
struct dentry *debugfs_q_instance;
if (!debugfs_initialized())
return;
debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt,
&pt_debugfs_info_fops);
debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt,
&pt_debugfs_stats_fops);
cmd_q = &pt->cmd_q;
debugfs_q_instance =
debugfs_create_dir("q", pt->dma_dev.dbg_dev_root);
debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q,
&pt_debugfs_queue_fops);
}

305
drivers/dma/ptdma/ptdma-dev.c Normal file
View File

@ -0,0 +1,305 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* AMD Passthru DMA device driver
* -- Based on the CCP driver
*
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
*
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/bitfield.h>
#include <linux/dma-mapping.h>
#include <linux/debugfs.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>
#include "ptdma.h"
/* Human-readable error strings */
static char *pt_error_codes[] = {
"",
"ERR 01: ILLEGAL_ENGINE",
"ERR 03: ILLEGAL_FUNCTION_TYPE",
"ERR 04: ILLEGAL_FUNCTION_MODE",
"ERR 06: ILLEGAL_FUNCTION_SIZE",
"ERR 08: ILLEGAL_FUNCTION_RSVD",
"ERR 09: ILLEGAL_BUFFER_LENGTH",
"ERR 10: VLSB_FAULT",
"ERR 11: ILLEGAL_MEM_ADDR",
"ERR 12: ILLEGAL_MEM_SEL",
"ERR 13: ILLEGAL_CONTEXT_ID",
"ERR 15: 0xF Reserved",
"ERR 18: CMD_TIMEOUT",
"ERR 19: IDMA0_AXI_SLVERR",
"ERR 20: IDMA0_AXI_DECERR",
"ERR 21: 0x15 Reserved",
"ERR 22: IDMA1_AXI_SLAVE_FAULT",
"ERR 23: IDMA1_AIXI_DECERR",
"ERR 24: 0x18 Reserved",
"ERR 27: 0x1B Reserved",
"ERR 38: ODMA0_AXI_SLVERR",
"ERR 39: ODMA0_AXI_DECERR",
"ERR 40: 0x28 Reserved",
"ERR 41: ODMA1_AXI_SLVERR",
"ERR 42: ODMA1_AXI_DECERR",
"ERR 43: LSB_PARITY_ERR",
};
static void pt_log_error(struct pt_device *d, int e)
{
dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
}
void pt_start_queue(struct pt_cmd_queue *cmd_q)
{
/* Turn on the run bit */
iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
}
void pt_stop_queue(struct pt_cmd_queue *cmd_q)
{
/* Turn off the run bit */
iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
}
static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
{
bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
u32 tail;
if (soc) {
desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
desc->dw0 &= ~DWORD0_SOC;
}
mutex_lock(&cmd_q->q_mutex);
/* Copy 32-byte command descriptor to hw queue. */
memcpy(q_desc, desc, 32);
cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
/* The data used by this command must be flushed to memory */
wmb();
/* Write the new tail address back to the queue register */
tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
iowrite32(tail, cmd_q->reg_control + 0x0004);
/* Turn the queue back on using our cached control register */
pt_start_queue(cmd_q);
mutex_unlock(&cmd_q->q_mutex);
return 0;
}
int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
struct pt_passthru_engine *pt_engine)
{
struct ptdma_desc desc;
cmd_q->cmd_error = 0;
cmd_q->total_pt_ops++;
memset(&desc, 0, sizeof(desc));
desc.dw0 = CMD_DESC_DW0_VAL;
desc.length = pt_engine->src_len;
desc.src_lo = lower_32_bits(pt_engine->src_dma);
desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
return pt_core_execute_cmd(&desc, cmd_q);
}
static inline void pt_core_disable_queue_interrupts(struct pt_device *pt)
{
iowrite32(0, pt->cmd_q.reg_control + 0x000C);
}
static inline void pt_core_enable_queue_interrupts(struct pt_device *pt)
{
iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C);
}
static void pt_do_cmd_complete(unsigned long data)
{
struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;
struct pt_cmd *cmd = tdata->cmd;
struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;
u32 tail;
if (cmd_q->cmd_error) {
/*
* Log the error and flush the queue by
* moving the head pointer
*/
tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
pt_log_error(cmd_q->pt, cmd_q->cmd_error);
iowrite32(tail, cmd_q->reg_control + 0x0008);
}
cmd->pt_cmd_callback(cmd->data, cmd->ret);
}
static irqreturn_t pt_core_irq_handler(int irq, void *data)
{
struct pt_device *pt = data;
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
u32 status;
pt_core_disable_queue_interrupts(pt);
pt->total_interrupts++;
status = ioread32(cmd_q->reg_control + 0x0010);
if (status) {
cmd_q->int_status = status;
cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
/* On error, only save the first error value */
if ((status & INT_ERROR) && !cmd_q->cmd_error)
cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
/* Acknowledge the interrupt */
iowrite32(status, cmd_q->reg_control + 0x0010);
pt_core_enable_queue_interrupts(pt);
pt_do_cmd_complete((ulong)&pt->tdata);
}
return IRQ_HANDLED;
}
int pt_core_init(struct pt_device *pt)
{
char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
u32 dma_addr_lo, dma_addr_hi;
struct device *dev = pt->dev;
struct dma_pool *dma_pool;
int ret;
/* Allocate a dma pool for the queue */
snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
dma_pool = dma_pool_create(dma_pool_name, dev,
PT_DMAPOOL_MAX_SIZE,
PT_DMAPOOL_ALIGN, 0);
if (!dma_pool)
return -ENOMEM;
/* ptdma core initialisation */
iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
cmd_q->pt = pt;
cmd_q->dma_pool = dma_pool;
mutex_init(&cmd_q->q_mutex);
/* Page alignment satisfies our needs for N <= 128 */
cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
&cmd_q->qbase_dma,
GFP_KERNEL);
if (!cmd_q->qbase) {
dev_err(dev, "unable to allocate command queue\n");
ret = -ENOMEM;
goto e_dma_alloc;
}
cmd_q->qidx = 0;
/* Preset some register values */
cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
/* Turn off the queues and disable interrupts until ready */
pt_core_disable_queue_interrupts(pt);
cmd_q->qcontrol = 0; /* Start with nothing */
iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
ioread32(cmd_q->reg_control + 0x0104);
ioread32(cmd_q->reg_control + 0x0100);
/* Clear the interrupt status */
iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
/* Request an irq */
ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
if (ret)
goto e_pool;
/* Update the device registers with queue information. */
cmd_q->qcontrol &= ~CMD_Q_SIZE;
cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
cmd_q->qdma_tail = cmd_q->qbase_dma;
dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
cmd_q->qcontrol |= (dma_addr_hi << 16);
iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
pt_core_enable_queue_interrupts(pt);
/* Register the DMA engine support */
ret = pt_dmaengine_register(pt);
if (ret)
goto e_dmaengine;
/* Set up debugfs entries */
ptdma_debugfs_setup(pt);
return 0;
e_dmaengine:
free_irq(pt->pt_irq, pt);
e_dma_alloc:
dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
e_pool:
dev_err(dev, "unable to allocate an IRQ\n");
dma_pool_destroy(pt->cmd_q.dma_pool);
return ret;
}
void pt_core_destroy(struct pt_device *pt)
{
struct device *dev = pt->dev;
struct pt_cmd_queue *cmd_q = &pt->cmd_q;
struct pt_cmd *cmd;
/* Unregister the DMA engine */
pt_dmaengine_unregister(pt);
/* Disable and clear interrupts */
pt_core_disable_queue_interrupts(pt);
/* Turn off the run bit */
pt_stop_queue(cmd_q);
/* Clear the interrupt status */
iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
ioread32(cmd_q->reg_control + 0x0104);
ioread32(cmd_q->reg_control + 0x0100);
free_irq(pt->pt_irq, pt);
dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
cmd_q->qbase_dma);
/* Flush the cmd queue */
while (!list_empty(&pt->cmd)) {
/* Invoke the callback directly with an error code */
cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
list_del(&cmd->entry);
cmd->pt_cmd_callback(cmd->data, -ENODEV);
}
}

389
drivers/dma/ptdma/ptdma-dmaengine.c Normal file
View File

@ -0,0 +1,389 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* AMD Passthrough DMA device driver
* -- Based on the CCP driver
*
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
*
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include "ptdma.h"
#include "../dmaengine.h"
#include "../virt-dma.h"
static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan)
{
return container_of(dma_chan, struct pt_dma_chan, vc.chan);
}
static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd)
{
return container_of(vd, struct pt_dma_desc, vd);
}
static void pt_free_chan_resources(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
vchan_free_chan_resources(&chan->vc);
}
static void pt_synchronize(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
vchan_synchronize(&chan->vc);
}
static void pt_do_cleanup(struct virt_dma_desc *vd)
{
struct pt_dma_desc *desc = to_pt_desc(vd);
struct pt_device *pt = desc->pt;
kmem_cache_free(pt->dma_desc_cache, desc);
}
static int pt_dma_start_desc(struct pt_dma_desc *desc)
{
struct pt_passthru_engine *pt_engine;
struct pt_device *pt;
struct pt_cmd *pt_cmd;
struct pt_cmd_queue *cmd_q;
desc->issued_to_hw = 1;
pt_cmd = &desc->pt_cmd;
pt = pt_cmd->pt;
cmd_q = &pt->cmd_q;
pt_engine = &pt_cmd->passthru;
pt->tdata.cmd = pt_cmd;
/* Execute the command */
pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine);
return 0;
}
static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan)
{
/* Get the next DMA descriptor on the active list */
struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
return vd ? to_pt_desc(vd) : NULL;
}
static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan,
struct pt_dma_desc *desc)
{
struct dma_async_tx_descriptor *tx_desc;
struct virt_dma_desc *vd;
unsigned long flags;
/* Loop over descriptors until one is found with commands */
do {
if (desc) {
if (!desc->issued_to_hw) {
/* No errors, keep going */
if (desc->status != DMA_ERROR)
return desc;
}
tx_desc = &desc->vd.tx;
vd = &desc->vd;
} else {
tx_desc = NULL;
}
spin_lock_irqsave(&chan->vc.lock, flags);
if (desc) {
if (desc->status != DMA_ERROR)
desc->status = DMA_COMPLETE;
dma_cookie_complete(tx_desc);
dma_descriptor_unmap(tx_desc);
list_del(&desc->vd.node);
}
desc = pt_next_dma_desc(chan);
spin_unlock_irqrestore(&chan->vc.lock, flags);
if (tx_desc) {
dmaengine_desc_get_callback_invoke(tx_desc, NULL);
dma_run_dependencies(tx_desc);
vchan_vdesc_fini(vd);
}
} while (desc);
return NULL;
}
static void pt_cmd_callback(void *data, int err)
{
struct pt_dma_desc *desc = data;
struct dma_chan *dma_chan;
struct pt_dma_chan *chan;
int ret;
if (err == -EINPROGRESS)
return;
dma_chan = desc->vd.tx.chan;
chan = to_pt_chan(dma_chan);
if (err)
desc->status = DMA_ERROR;
while (true) {
/* Check for DMA descriptor completion */
desc = pt_handle_active_desc(chan, desc);
/* Don't submit cmd if no descriptor or DMA is paused */
if (!desc)
break;
ret = pt_dma_start_desc(desc);
if (!ret)
break;
desc->status = DMA_ERROR;
}
}
static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan,
unsigned long flags)
{
struct pt_dma_desc *desc;
desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT);
if (!desc)
return NULL;
vchan_tx_prep(&chan->vc, &desc->vd, flags);
desc->pt = chan->pt;
desc->issued_to_hw = 0;
desc->status = DMA_IN_PROGRESS;
return desc;
}
static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan,
dma_addr_t dst,
dma_addr_t src,
unsigned int len,
unsigned long flags)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
struct pt_passthru_engine *pt_engine;
struct pt_dma_desc *desc;
struct pt_cmd *pt_cmd;
desc = pt_alloc_dma_desc(chan, flags);
if (!desc)
return NULL;
pt_cmd = &desc->pt_cmd;
pt_cmd->pt = chan->pt;
pt_engine = &pt_cmd->passthru;
pt_cmd->engine = PT_ENGINE_PASSTHRU;
pt_engine->src_dma = src;
pt_engine->dst_dma = dst;
pt_engine->src_len = len;
pt_cmd->pt_cmd_callback = pt_cmd_callback;
pt_cmd->data = desc;
desc->len = len;
return desc;
}
static struct dma_async_tx_descriptor *
pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst,
dma_addr_t src, size_t len, unsigned long flags)
{
struct pt_dma_desc *desc;
desc = pt_create_desc(dma_chan, dst, src, len, flags);
if (!desc)
return NULL;
return &desc->vd.tx;
}
static struct dma_async_tx_descriptor *
pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
struct pt_dma_desc *desc;
desc = pt_alloc_dma_desc(chan, flags);
if (!desc)
return NULL;
return &desc->vd.tx;
}
static void pt_issue_pending(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
struct pt_dma_desc *desc;
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
vchan_issue_pending(&chan->vc);
desc = pt_next_dma_desc(chan);
spin_unlock_irqrestore(&chan->vc.lock, flags);
/* If there was nothing active, start processing */
if (desc)
pt_cmd_callback(desc, 0);
}
static int pt_pause(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
pt_stop_queue(&chan->pt->cmd_q);
spin_unlock_irqrestore(&chan->vc.lock, flags);
return 0;
}
static int pt_resume(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
struct pt_dma_desc *desc = NULL;
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
pt_start_queue(&chan->pt->cmd_q);
desc = pt_next_dma_desc(chan);
spin_unlock_irqrestore(&chan->vc.lock, flags);
/* If there was something active, re-start */
if (desc)
pt_cmd_callback(desc, 0);
return 0;
}
static int pt_terminate_all(struct dma_chan *dma_chan)
{
struct pt_dma_chan *chan = to_pt_chan(dma_chan);
unsigned long flags;
LIST_HEAD(head);
spin_lock_irqsave(&chan->vc.lock, flags);
vchan_get_all_descriptors(&chan->vc, &head);
spin_unlock_irqrestore(&chan->vc.lock, flags);
vchan_dma_desc_free_list(&chan->vc, &head);
vchan_free_chan_resources(&chan->vc);
return 0;
}
int pt_dmaengine_register(struct pt_device *pt)
{
struct pt_dma_chan *chan;
struct dma_device *dma_dev = &pt->dma_dev;
char *cmd_cache_name;
char *desc_cache_name;
int ret;
pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan),
GFP_KERNEL);
if (!pt->pt_dma_chan)
return -ENOMEM;
cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
"%s-dmaengine-cmd-cache",
dev_name(pt->dev));
if (!cmd_cache_name)
return -ENOMEM;
desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
"%s-dmaengine-desc-cache",
dev_name(pt->dev));
if (!desc_cache_name) {
ret = -ENOMEM;
goto err_cache;
}
pt->dma_desc_cache = kmem_cache_create(desc_cache_name,
sizeof(struct pt_dma_desc), 0,
SLAB_HWCACHE_ALIGN, NULL);
if (!pt->dma_desc_cache) {
ret = -ENOMEM;
goto err_cache;
}
dma_dev->dev = pt->dev;
dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
dma_dev->directions = DMA_MEM_TO_MEM;
dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
/*
* PTDMA is intended to be used with the AMD NTB devices, hence
* marking it as DMA_PRIVATE.
*/
dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
INIT_LIST_HEAD(&dma_dev->channels);
chan = pt->pt_dma_chan;
chan->pt = pt;
/* Set base and prep routines */
dma_dev->device_free_chan_resources = pt_free_chan_resources;
dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy;
dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt;
dma_dev->device_issue_pending = pt_issue_pending;
dma_dev->device_tx_status = dma_cookie_status;
dma_dev->device_pause = pt_pause;
dma_dev->device_resume = pt_resume;
dma_dev->device_terminate_all = pt_terminate_all;
dma_dev->device_synchronize = pt_synchronize;
chan->vc.desc_free = pt_do_cleanup;
vchan_init(&chan->vc, dma_dev);
dma_set_mask_and_coherent(pt->dev, DMA_BIT_MASK(64));
ret = dma_async_device_register(dma_dev);
if (ret)
goto err_reg;
return 0;
err_reg:
kmem_cache_destroy(pt->dma_desc_cache);
err_cache:
kmem_cache_destroy(pt->dma_cmd_cache);
return ret;
}
void pt_dmaengine_unregister(struct pt_device *pt)
{
struct dma_device *dma_dev = &pt->dma_dev;
dma_async_device_unregister(dma_dev);
kmem_cache_destroy(pt->dma_desc_cache);
kmem_cache_destroy(pt->dma_cmd_cache);
}

243
drivers/dma/ptdma/ptdma-pci.c Normal file
View File

@ -0,0 +1,243 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* AMD Passthru DMA device driver
* -- Based on the CCP driver
*
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
*
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
* Author: Tom Lendacky <thomas.lendacky@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/pci_ids.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include "ptdma.h"
struct pt_msix {
int msix_count;
struct msix_entry msix_entry;
};
/*
* pt_alloc_struct - allocate and initialize the pt_device struct
*
* @dev: device struct of the PTDMA
*/
static struct pt_device *pt_alloc_struct(struct device *dev)
{
struct pt_device *pt;
pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL);
if (!pt)
return NULL;
pt->dev = dev;
INIT_LIST_HEAD(&pt->cmd);
return pt;
}
static int pt_get_msix_irqs(struct pt_device *pt)
{
struct pt_msix *pt_msix = pt->pt_msix;
struct device *dev = pt->dev;
struct pci_dev *pdev = to_pci_dev(dev);
int ret;
pt_msix->msix_entry.entry = 0;
ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1);
if (ret < 0)
return ret;
pt_msix->msix_count = ret;
pt->pt_irq = pt_msix->msix_entry.vector;
return 0;
}
static int pt_get_msi_irq(struct pt_device *pt)
{
struct device *dev = pt->dev;
struct pci_dev *pdev = to_pci_dev(dev);
int ret;
ret = pci_enable_msi(pdev);
if (ret)
return ret;
pt->pt_irq = pdev->irq;
return 0;
}
static int pt_get_irqs(struct pt_device *pt)
{
struct device *dev = pt->dev;
int ret;
ret = pt_get_msix_irqs(pt);
if (!ret)
return 0;
/* Couldn't get MSI-X vectors, try MSI */
dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret);
ret = pt_get_msi_irq(pt);
if (!ret)
return 0;
/* Couldn't get MSI interrupt */
dev_err(dev, "could not enable MSI (%d)\n", ret);
return ret;
}
static void pt_free_irqs(struct pt_device *pt)
{
struct pt_msix *pt_msix = pt->pt_msix;
struct device *dev = pt->dev;
struct pci_dev *pdev = to_pci_dev(dev);
if (pt_msix->msix_count)
pci_disable_msix(pdev);
else if (pt->pt_irq)
pci_disable_msi(pdev);
pt->pt_irq = 0;
}
static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct pt_device *pt;
struct pt_msix *pt_msix;
struct device *dev = &pdev->dev;
void __iomem * const *iomap_table;
int bar_mask;
int ret = -ENOMEM;
pt = pt_alloc_struct(dev);
if (!pt)
goto e_err;
pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL);
if (!pt_msix)
goto e_err;
pt->pt_msix = pt_msix;
pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data;
if (!pt->dev_vdata) {
ret = -ENODEV;
dev_err(dev, "missing driver data\n");
goto e_err;
}
ret = pcim_enable_device(pdev);
if (ret) {
dev_err(dev, "pcim_enable_device failed (%d)\n", ret);
goto e_err;
}
bar_mask = pci_select_bars(pdev, IORESOURCE_MEM);
ret = pcim_iomap_regions(pdev, bar_mask, "ptdma");
if (ret) {
dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret);
goto e_err;
}
iomap_table = pcim_iomap_table(pdev);
if (!iomap_table) {
dev_err(dev, "pcim_iomap_table failed\n");
ret = -ENOMEM;
goto e_err;
}
pt->io_regs = iomap_table[pt->dev_vdata->bar];
if (!pt->io_regs) {
dev_err(dev, "ioremap failed\n");
ret = -ENOMEM;
goto e_err;
}
ret = pt_get_irqs(pt);
if (ret)
goto e_err;
pci_set_master(pdev);
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
if (ret) {
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
if (ret) {
dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n",
ret);
goto e_err;
}
}
dev_set_drvdata(dev, pt);
if (pt->dev_vdata)
ret = pt_core_init(pt);
if (ret)
goto e_err;
return 0;
e_err:
dev_err(dev, "initialization failed ret = %d\n", ret);
return ret;
}
static void pt_pci_remove(struct pci_dev *pdev)
{
struct device *dev = &pdev->dev;
struct pt_device *pt = dev_get_drvdata(dev);
if (!pt)
return;
if (pt->dev_vdata)
pt_core_destroy(pt);
pt_free_irqs(pt);
}
static const struct pt_dev_vdata dev_vdata[] = {
{
.bar = 2,
},
};
static const struct pci_device_id pt_pci_table[] = {
{ PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] },
/* Last entry must be zero */
{ 0, }
};
MODULE_DEVICE_TABLE(pci, pt_pci_table);
static struct pci_driver pt_pci_driver = {
.name = "ptdma",
.id_table = pt_pci_table,
.probe = pt_pci_probe,
.remove = pt_pci_remove,
};
module_pci_driver(pt_pci_driver);
MODULE_AUTHOR("Sanjay R Mehta <sanju.mehta@amd.com>");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("AMD PassThru DMA driver");

324
drivers/dma/ptdma/ptdma.h Normal file
View File

@ -0,0 +1,324 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* AMD Passthru DMA device driver
* -- Based on the CCP driver
*
* Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
*
* Author: Sanjay R Mehta <sanju.mehta@amd.com>
* Author: Tom Lendacky <thomas.lendacky@amd.com>
* Author: Gary R Hook <gary.hook@amd.com>
*/
#ifndef __PT_DEV_H__
#define __PT_DEV_H__
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/pci.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/wait.h>
#include <linux/dmapool.h>
#include "../virt-dma.h"
#define MAX_PT_NAME_LEN 16
#define MAX_DMAPOOL_NAME_LEN 32
#define MAX_HW_QUEUES 1
#define MAX_CMD_QLEN 100
#define PT_ENGINE_PASSTHRU 5
/* Register Mappings */
#define IRQ_MASK_REG 0x040
#define IRQ_STATUS_REG 0x200
#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
#define CMD_QUEUE_PRIO_OFFSET 0x00
#define CMD_REQID_CONFIG_OFFSET 0x04
#define CMD_TIMEOUT_OFFSET 0x08
#define CMD_PT_VERSION 0x10
#define CMD_Q_CONTROL_BASE 0x0000
#define CMD_Q_TAIL_LO_BASE 0x0004
#define CMD_Q_HEAD_LO_BASE 0x0008
#define CMD_Q_INT_ENABLE_BASE 0x000C
#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010
#define CMD_Q_STATUS_BASE 0x0100
#define CMD_Q_INT_STATUS_BASE 0x0104
#define CMD_Q_DMA_STATUS_BASE 0x0108
#define CMD_Q_DMA_READ_STATUS_BASE 0x010C
#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110
#define CMD_Q_ABORT_BASE 0x0114
#define CMD_Q_AX_CACHE_BASE 0x0118
#define CMD_CONFIG_OFFSET 0x1120
#define CMD_CLK_GATE_CTL_OFFSET 0x6004
#define CMD_DESC_DW0_VAL 0x500012
/* Address offset for virtual queue registers */
#define CMD_Q_STATUS_INCR 0x1000
/* Bit masks */
#define CMD_CONFIG_REQID 0
#define CMD_TIMEOUT_DISABLE 0
#define CMD_CLK_DYN_GATING_DIS 0
#define CMD_CLK_SW_GATE_MODE 0
#define CMD_CLK_GATE_CTL 0
#define CMD_QUEUE_PRIO GENMASK(2, 1)
#define CMD_CONFIG_VHB_EN BIT(0)
#define CMD_CLK_DYN_GATING_EN BIT(0)
#define CMD_CLK_HW_GATE_MODE BIT(0)
#define CMD_CLK_GATE_ON_DELAY BIT(12)
#define CMD_CLK_GATE_OFF_DELAY BIT(12)
#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \
CMD_CLK_HW_GATE_MODE | \
CMD_CLK_GATE_ON_DELAY | \
CMD_CLK_DYN_GATING_EN | \
CMD_CLK_GATE_OFF_DELAY)
#define CMD_Q_LEN 32
#define CMD_Q_RUN BIT(0)
#define CMD_Q_HALT BIT(1)
#define CMD_Q_MEM_LOCATION BIT(2)
#define CMD_Q_SIZE_MASK GENMASK(4, 0)
#define CMD_Q_SIZE GENMASK(7, 3)
#define CMD_Q_SHIFT GENMASK(1, 0)
#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \
CMD_Q_SIZE_MASK)
#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1)
#define Q_DESC_SIZE sizeof(struct ptdma_desc)
#define Q_SIZE(n) (CMD_Q_LEN * (n))
#define INT_COMPLETION BIT(0)
#define INT_ERROR BIT(1)
#define INT_QUEUE_STOPPED BIT(2)
#define INT_EMPTY_QUEUE BIT(3)
#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR)
/****** Local Storage Block ******/
#define LSB_START 0
#define LSB_END 127
#define LSB_COUNT (LSB_END - LSB_START + 1)
#define PT_DMAPOOL_MAX_SIZE 64
#define PT_DMAPOOL_ALIGN BIT(5)
#define PT_PASSTHRU_BLOCKSIZE 512
struct pt_device;
struct pt_tasklet_data {
struct completion completion;
struct pt_cmd *cmd;
};
/*
* struct pt_passthru_engine - pass-through operation
* without performing DMA mapping
* @mask: mask to be applied to data
* @mask_len: length in bytes of mask
* @src_dma: data to be used for this operation
* @dst_dma: data produced by this operation
* @src_len: length in bytes of data used for this operation
*
* Variables required to be set when calling pt_enqueue_cmd():
* - bit_mod, byte_swap, src, dst, src_len
* - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP
*/
struct pt_passthru_engine {
dma_addr_t mask;
u32 mask_len; /* In bytes */
dma_addr_t src_dma, dst_dma;
u64 src_len; /* In bytes */
};
/*
* struct pt_cmd - PTDMA operation request
* @entry: list element
* @work: work element used for callbacks
* @pt: PT device to be run on
* @ret: operation return code
* @flags: cmd processing flags
* @engine: PTDMA operation to perform (passthru)
* @engine_error: PT engine return code
* @passthru: engine specific structures, refer to specific engine struct below
* @callback: operation completion callback function
* @data: parameter value to be supplied to the callback function
*
* Variables required to be set when calling pt_enqueue_cmd():
* - engine, callback
* - See the operation structures below for what is required for each
* operation.
*/
struct pt_cmd {
struct list_head entry;
struct work_struct work;
struct pt_device *pt;
int ret;
u32 engine;
u32 engine_error;
struct pt_passthru_engine passthru;
/* Completion callback support */
void (*pt_cmd_callback)(void *data, int err);
void *data;
};
struct pt_dma_desc {
struct virt_dma_desc vd;
struct pt_device *pt;
enum dma_status status;
size_t len;
bool issued_to_hw;
struct pt_cmd pt_cmd;
};
struct pt_dma_chan {
struct virt_dma_chan vc;
struct pt_device *pt;
};
struct pt_cmd_queue {
struct pt_device *pt;
/* Queue dma pool */
struct dma_pool *dma_pool;
/* Queue base address (not neccessarily aligned)*/
struct ptdma_desc *qbase;
/* Aligned queue start address (per requirement) */
struct mutex q_mutex ____cacheline_aligned;
unsigned int qidx;
unsigned int qsize;
dma_addr_t qbase_dma;
dma_addr_t qdma_tail;
unsigned int active;
unsigned int suspended;
/* Register addresses for queue */
void __iomem *reg_control;
u32 qcontrol; /* Cached control register */
/* Status values from job */
u32 int_status;
u32 q_status;
u32 q_int_status;
u32 cmd_error;
/* Queue Statistics */
unsigned long total_pt_ops;
} ____cacheline_aligned;
struct pt_device {
struct list_head entry;
unsigned int ord;
char name[MAX_PT_NAME_LEN];
struct device *dev;
/* Bus specific device information */
struct pt_msix *pt_msix;
struct pt_dev_vdata *dev_vdata;
unsigned int pt_irq;
/* I/O area used for device communication */
void __iomem *io_regs;
spinlock_t cmd_lock ____cacheline_aligned;
unsigned int cmd_count;
struct list_head cmd;
/*
* The command queue. This represent the queue available on the
* PTDMA that are available for processing cmds
*/
struct pt_cmd_queue cmd_q;
/* Support for the DMA Engine capabilities */
struct dma_device dma_dev;
struct pt_dma_chan *pt_dma_chan;
struct kmem_cache *dma_cmd_cache;
struct kmem_cache *dma_desc_cache;
wait_queue_head_t lsb_queue;
/* Device Statistics */
unsigned long total_interrupts;
struct pt_tasklet_data tdata;
};
/*
* descriptor for PTDMA commands
* 8 32-bit words:
* word 0: function; engine; control bits
* word 1: length of source data
* word 2: low 32 bits of source pointer
* word 3: upper 16 bits of source pointer; source memory type
* word 4: low 32 bits of destination pointer
* word 5: upper 16 bits of destination pointer; destination memory type
* word 6: reserved 32 bits
* word 7: reserved 32 bits
*/
#define DWORD0_SOC BIT(0)
#define DWORD0_IOC BIT(1)
struct dword3 {
unsigned int src_hi:16;
unsigned int src_mem:2;
unsigned int lsb_cxt_id:8;
unsigned int rsvd1:5;
unsigned int fixed:1;
};
struct dword5 {
unsigned int dst_hi:16;
unsigned int dst_mem:2;
unsigned int rsvd1:13;
unsigned int fixed:1;
};
struct ptdma_desc {
u32 dw0;
u32 length;
u32 src_lo;
struct dword3 dw3;
u32 dst_lo;
struct dword5 dw5;
__le32 rsvd1;
__le32 rsvd2;
};
/* Structure to hold PT device data */
struct pt_dev_vdata {
const unsigned int bar;
};
int pt_dmaengine_register(struct pt_device *pt);
void pt_dmaengine_unregister(struct pt_device *pt);
void ptdma_debugfs_setup(struct pt_device *pt);
int pt_core_init(struct pt_device *pt);
void pt_core_destroy(struct pt_device *pt);
int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
struct pt_passthru_engine *pt_engine);
void pt_start_queue(struct pt_cmd_queue *cmd_q);
void pt_stop_queue(struct pt_cmd_queue *cmd_q);
#endif

9
drivers/dma/sh/Kconfig
View File

@ -47,3 +47,12 @@ config RENESAS_USB_DMAC
help
This driver supports the USB-DMA controller found in the Renesas
SoCs.
config RZ_DMAC
tristate "Renesas RZ/G2L DMA Controller"
depends on ARCH_R9A07G044 || COMPILE_TEST
select RENESAS_DMA
select DMA_VIRTUAL_CHANNELS
help
This driver supports the general purpose DMA controller found in the
Renesas RZ/G2L SoC variants.

1
drivers/dma/sh/Makefile
View File

@ -15,3 +15,4 @@ obj-$(CONFIG_SH_DMAE) += shdma.o
obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o
obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o
obj-$(CONFIG_RZ_DMAC) += rz-dmac.o

969
drivers/dma/sh/rz-dmac.c Normal file
View File

@ -0,0 +1,969 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Renesas RZ/G2L DMA Controller Driver
*
* Based on imx-dma.c
*
* Copyright (C) 2021 Renesas Electronics Corp.
* Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
* Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
*/
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_dma.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "../dmaengine.h"
#include "../virt-dma.h"
enum rz_dmac_prep_type {
RZ_DMAC_DESC_MEMCPY,
RZ_DMAC_DESC_SLAVE_SG,
};
struct rz_lmdesc {
u32 header;
u32 sa;
u32 da;
u32 tb;
u32 chcfg;
u32 chitvl;
u32 chext;
u32 nxla;
};
struct rz_dmac_desc {
struct virt_dma_desc vd;
dma_addr_t src;
dma_addr_t dest;
size_t len;
struct list_head node;
enum dma_transfer_direction direction;
enum rz_dmac_prep_type type;
/* For slave sg */
struct scatterlist *sg;
unsigned int sgcount;
};
#define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd)
struct rz_dmac_chan {
struct virt_dma_chan vc;
void __iomem *ch_base;
void __iomem *ch_cmn_base;
unsigned int index;
int irq;
struct rz_dmac_desc *desc;
int descs_allocated;
enum dma_slave_buswidth src_word_size;
enum dma_slave_buswidth dst_word_size;
dma_addr_t src_per_address;
dma_addr_t dst_per_address;
u32 chcfg;
u32 chctrl;
int mid_rid;
struct list_head ld_free;
struct list_head ld_queue;
struct list_head ld_active;
struct {
struct rz_lmdesc *base;
struct rz_lmdesc *head;
struct rz_lmdesc *tail;
dma_addr_t base_dma;
} lmdesc;
};
#define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan)
struct rz_dmac {
struct dma_device engine;
struct device *dev;
void __iomem *base;
void __iomem *ext_base;
unsigned int n_channels;
struct rz_dmac_chan *channels;
DECLARE_BITMAP(modules, 1024);
};
#define to_rz_dmac(d) container_of(d, struct rz_dmac, engine)
/*
* -----------------------------------------------------------------------------
* Registers
*/
#define CHSTAT 0x0024
#define CHCTRL 0x0028
#define CHCFG 0x002c
#define NXLA 0x0038
#define DCTRL 0x0000
#define EACH_CHANNEL_OFFSET 0x0040
#define CHANNEL_0_7_OFFSET 0x0000
#define CHANNEL_0_7_COMMON_BASE 0x0300
#define CHANNEL_8_15_OFFSET 0x0400
#define CHANNEL_8_15_COMMON_BASE 0x0700
#define CHSTAT_ER BIT(4)
#define CHSTAT_EN BIT(0)
#define CHCTRL_CLRINTMSK BIT(17)
#define CHCTRL_CLRSUS BIT(9)
#define CHCTRL_CLRTC BIT(6)
#define CHCTRL_CLREND BIT(5)
#define CHCTRL_CLRRQ BIT(4)
#define CHCTRL_SWRST BIT(3)
#define CHCTRL_STG BIT(2)
#define CHCTRL_CLREN BIT(1)
#define CHCTRL_SETEN BIT(0)
#define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
CHCTRL_CLRTC | CHCTRL_CLREND | \
CHCTRL_CLRRQ | CHCTRL_SWRST | \
CHCTRL_CLREN)
#define CHCFG_DMS BIT(31)
#define CHCFG_DEM BIT(24)
#define CHCFG_DAD BIT(21)
#define CHCFG_SAD BIT(20)
#define CHCFG_REQD BIT(3)
#define CHCFG_SEL(bits) ((bits) & 0x07)
#define CHCFG_MEM_COPY (0x80400008)
#define CHCFG_FILL_DDS(a) (((a) << 16) & GENMASK(19, 16))
#define CHCFG_FILL_SDS(a) (((a) << 12) & GENMASK(15, 12))
#define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22)
#define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6)
#define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5)
#define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5)
#define MID_RID_MASK GENMASK(9, 0)
#define CHCFG_MASK GENMASK(15, 10)
#define CHCFG_DS_INVALID 0xFF
#define DCTRL_LVINT BIT(1)
#define DCTRL_PR BIT(0)
#define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR)
/* LINK MODE DESCRIPTOR */
#define HEADER_LV BIT(0)
#define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16
#define RZ_DMAC_MAX_CHANNELS 16
#define DMAC_NR_LMDESC 64
/*
* -----------------------------------------------------------------------------
* Device access
*/
static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
unsigned int offset)
{
writel(val, dmac->base + offset);
}
static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
unsigned int offset)
{
writel(val, dmac->ext_base + offset);
}
static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
{
return readl(dmac->ext_base + offset);
}
static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
unsigned int offset, int which)
{
if (which)
writel(val, channel->ch_base + offset);
else
writel(val, channel->ch_cmn_base + offset);
}
static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
unsigned int offset, int which)
{
if (which)
return readl(channel->ch_base + offset);
else
return readl(channel->ch_cmn_base + offset);
}
/*
* -----------------------------------------------------------------------------
* Initialization
*/
static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
struct rz_lmdesc *lmdesc)
{
u32 nxla;
channel->lmdesc.base = lmdesc;
channel->lmdesc.head = lmdesc;
channel->lmdesc.tail = lmdesc;
nxla = channel->lmdesc.base_dma;
while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
lmdesc->header = 0;
nxla += sizeof(*lmdesc);
lmdesc->nxla = nxla;
lmdesc++;
}
lmdesc->header = 0;
lmdesc->nxla = channel->lmdesc.base_dma;
}
/*
* -----------------------------------------------------------------------------
* Descriptors preparation
*/
static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
{
struct rz_lmdesc *lmdesc = channel->lmdesc.head;
while (!(lmdesc->header & HEADER_LV)) {
lmdesc->header = 0;
lmdesc++;
if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
lmdesc = channel->lmdesc.base;
}
channel->lmdesc.head = lmdesc;
}
static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
unsigned long flags;
u32 nxla;
u32 chctrl;
u32 chstat;
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
local_irq_save(flags);
rz_dmac_lmdesc_recycle(channel);
nxla = channel->lmdesc.base_dma +
(sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
channel->lmdesc.base));
chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
if (!(chstat & CHSTAT_EN)) {
chctrl = (channel->chctrl | CHCTRL_SETEN);
rz_dmac_ch_writel(channel, nxla, NXLA, 1);
rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
}
local_irq_restore(flags);
}
static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
unsigned long flags;
dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
local_irq_save(flags);
rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
local_irq_restore(flags);
}
static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
{
u32 dmars_offset = (nr / 2) * 4;
u32 shift = (nr % 2) * 16;
u32 dmars32;
dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
dmars32 &= ~(0xffff << shift);
dmars32 |= dmars << shift;
rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
}
static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
struct rz_dmac_desc *d = channel->desc;
u32 chcfg = CHCFG_MEM_COPY;
/* prepare descriptor */
lmdesc->sa = d->src;
lmdesc->da = d->dest;
lmdesc->tb = d->len;
lmdesc->chcfg = chcfg;
lmdesc->chitvl = 0;
lmdesc->chext = 0;
lmdesc->header = HEADER_LV;
rz_dmac_set_dmars_register(dmac, channel->index, 0);
channel->chcfg = chcfg;
channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
}
static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *d = channel->desc;
struct scatterlist *sg, *sgl = d->sg;
struct rz_lmdesc *lmdesc;
unsigned int i, sg_len = d->sgcount;
channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
if (d->direction == DMA_DEV_TO_MEM) {
channel->chcfg |= CHCFG_SAD;
channel->chcfg &= ~CHCFG_REQD;
} else {
channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
}
lmdesc = channel->lmdesc.tail;
for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
if (d->direction == DMA_DEV_TO_MEM) {
lmdesc->sa = channel->src_per_address;
lmdesc->da = sg_dma_address(sg);
} else {
lmdesc->sa = sg_dma_address(sg);
lmdesc->da = channel->dst_per_address;
}
lmdesc->tb = sg_dma_len(sg);
lmdesc->chitvl = 0;
lmdesc->chext = 0;
if (i == (sg_len - 1)) {
lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
lmdesc->header = HEADER_LV;
} else {
lmdesc->chcfg = channel->chcfg;
lmdesc->header = HEADER_LV;
}
if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
lmdesc = channel->lmdesc.base;
}
channel->lmdesc.tail = lmdesc;
rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
channel->chctrl = CHCTRL_SETEN;
}
static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
{
struct rz_dmac_desc *d = chan->desc;
struct virt_dma_desc *vd;
vd = vchan_next_desc(&chan->vc);
if (!vd)
return 0;
list_del(&vd->node);
switch (d->type) {
case RZ_DMAC_DESC_MEMCPY:
rz_dmac_prepare_desc_for_memcpy(chan);
break;
case RZ_DMAC_DESC_SLAVE_SG:
rz_dmac_prepare_descs_for_slave_sg(chan);
break;
default:
return -EINVAL;
}
rz_dmac_enable_hw(chan);
return 0;
}
/*
* -----------------------------------------------------------------------------
* DMA engine operations
*/
static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
struct rz_dmac_desc *desc;
desc = kzalloc(sizeof(*desc), GFP_KERNEL);
if (!desc)
break;
list_add_tail(&desc->node, &channel->ld_free);
channel->descs_allocated++;
}
if (!channel->descs_allocated)
return -ENOMEM;
return channel->descs_allocated;
}
static void rz_dmac_free_chan_resources(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_lmdesc *lmdesc = channel->lmdesc.base;
struct rz_dmac_desc *desc, *_desc;
unsigned long flags;
unsigned int i;
spin_lock_irqsave(&channel->vc.lock, flags);
for (i = 0; i < DMAC_NR_LMDESC; i++)
lmdesc[i].header = 0;
rz_dmac_disable_hw(channel);
list_splice_tail_init(&channel->ld_active, &channel->ld_free);
list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
if (channel->mid_rid >= 0) {
clear_bit(channel->mid_rid, dmac->modules);
channel->mid_rid = -EINVAL;
}
spin_unlock_irqrestore(&channel->vc.lock, flags);
list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
kfree(desc);
channel->descs_allocated--;
}
INIT_LIST_HEAD(&channel->ld_free);
vchan_free_chan_resources(&channel->vc);
}
static struct dma_async_tx_descriptor *
rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
size_t len, unsigned long flags)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *desc;
dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
__func__, channel->index, &src, &dest, len);
if (list_empty(&channel->ld_free))
return NULL;
desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
desc->type = RZ_DMAC_DESC_MEMCPY;
desc->src = src;
desc->dest = dest;
desc->len = len;
desc->direction = DMA_MEM_TO_MEM;
list_move_tail(channel->ld_free.next, &channel->ld_queue);
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
static struct dma_async_tx_descriptor *
rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len,
enum dma_transfer_direction direction,
unsigned long flags, void *context)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac_desc *desc;
struct scatterlist *sg;
int dma_length = 0;
int i = 0;
if (list_empty(&channel->ld_free))
return NULL;
desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
for_each_sg(sgl, sg, sg_len, i) {
dma_length += sg_dma_len(sg);
}
desc->type = RZ_DMAC_DESC_SLAVE_SG;
desc->sg = sgl;
desc->sgcount = sg_len;
desc->len = dma_length;
desc->direction = direction;
if (direction == DMA_DEV_TO_MEM)
desc->src = channel->src_per_address;
else
desc->dest = channel->dst_per_address;
list_move_tail(channel->ld_free.next, &channel->ld_queue);
return vchan_tx_prep(&channel->vc, &desc->vd, flags);
}
static int rz_dmac_terminate_all(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
unsigned long flags;
LIST_HEAD(head);
rz_dmac_disable_hw(channel);
spin_lock_irqsave(&channel->vc.lock, flags);
list_splice_tail_init(&channel->ld_active, &channel->ld_free);
list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
spin_unlock_irqrestore(&channel->vc.lock, flags);
vchan_get_all_descriptors(&channel->vc, &head);
vchan_dma_desc_free_list(&channel->vc, &head);
return 0;
}
static void rz_dmac_issue_pending(struct dma_chan *chan)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct rz_dmac_desc *desc;
unsigned long flags;
spin_lock_irqsave(&channel->vc.lock, flags);
if (!list_empty(&channel->ld_queue)) {
desc = list_first_entry(&channel->ld_queue,
struct rz_dmac_desc, node);
channel->desc = desc;
if (vchan_issue_pending(&channel->vc)) {
if (rz_dmac_xfer_desc(channel) < 0)
dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
channel->index);
else
list_move_tail(channel->ld_queue.next,
&channel->ld_active);
}
}
spin_unlock_irqrestore(&channel->vc.lock, flags);
}
static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
{
u8 i;
const enum dma_slave_buswidth ds_lut[] = {
DMA_SLAVE_BUSWIDTH_1_BYTE,
DMA_SLAVE_BUSWIDTH_2_BYTES,
DMA_SLAVE_BUSWIDTH_4_BYTES,
DMA_SLAVE_BUSWIDTH_8_BYTES,
DMA_SLAVE_BUSWIDTH_16_BYTES,
DMA_SLAVE_BUSWIDTH_32_BYTES,
DMA_SLAVE_BUSWIDTH_64_BYTES,
DMA_SLAVE_BUSWIDTH_128_BYTES,
};
for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
if (ds_lut[i] == ds)
return i;
}
return CHCFG_DS_INVALID;
}
static int rz_dmac_config(struct dma_chan *chan,
struct dma_slave_config *config)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
u32 val;
channel->src_per_address = config->src_addr;
channel->src_word_size = config->src_addr_width;
channel->dst_per_address = config->dst_addr;
channel->dst_word_size = config->dst_addr_width;
val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
if (val == CHCFG_DS_INVALID)
return -EINVAL;
channel->chcfg |= CHCFG_FILL_DDS(val);
val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
if (val == CHCFG_DS_INVALID)
return -EINVAL;
channel->chcfg |= CHCFG_FILL_SDS(val);
return 0;
}
static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
{
/*
* Place holder
* Descriptor allocation is done during alloc_chan_resources and
* get freed during free_chan_resources.
* list is used to manage the descriptors and avoid any memory
* allocation/free during DMA read/write.
*/
}
/*
* -----------------------------------------------------------------------------
* IRQ handling
*/
static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
{
struct dma_chan *chan = &channel->vc.chan;
struct rz_dmac *dmac = to_rz_dmac(chan->device);
u32 chstat, chctrl;
chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
if (chstat & CHSTAT_ER) {
dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
channel->index, chstat);
rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
goto done;
}
chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
done:
return;
}
static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
if (channel) {
rz_dmac_irq_handle_channel(channel);
return IRQ_WAKE_THREAD;
}
/* handle DMAERR irq */
return IRQ_HANDLED;
}
static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
{
struct rz_dmac_chan *channel = dev_id;
struct rz_dmac_desc *desc = NULL;
unsigned long flags;
spin_lock_irqsave(&channel->vc.lock, flags);
if (list_empty(&channel->ld_active)) {
/* Someone might have called terminate all */
goto out;
}
desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
vchan_cookie_complete(&desc->vd);
list_move_tail(channel->ld_active.next, &channel->ld_free);
if (!list_empty(&channel->ld_queue)) {
desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
node);
channel->desc = desc;
if (rz_dmac_xfer_desc(channel) == 0)
list_move_tail(channel->ld_queue.next, &channel->ld_active);
}
out:
spin_unlock_irqrestore(&channel->vc.lock, flags);
return IRQ_HANDLED;
}
/*
* -----------------------------------------------------------------------------
* OF xlate and channel filter
*/
static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
{
struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
struct rz_dmac *dmac = to_rz_dmac(chan->device);
struct of_phandle_args *dma_spec = arg;
u32 ch_cfg;
channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
return !test_and_set_bit(channel->mid_rid, dmac->modules);
}
static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
struct of_dma *ofdma)
{
dma_cap_mask_t mask;
if (dma_spec->args_count != 1)
return NULL;
/* Only slave DMA channels can be allocated via DT */
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec);
}
/*
* -----------------------------------------------------------------------------
* Probe and remove
*/
static int rz_dmac_chan_probe(struct rz_dmac *dmac,
struct rz_dmac_chan *channel,
unsigned int index)
{
struct platform_device *pdev = to_platform_device(dmac->dev);
struct rz_lmdesc *lmdesc;
char pdev_irqname[5];
char *irqname;
int ret;
channel->index = index;
channel->mid_rid = -EINVAL;
/* Request the channel interrupt. */
sprintf(pdev_irqname, "ch%u", index);
channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
if (channel->irq < 0)
return channel->irq;
irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
dev_name(dmac->dev), index);
if (!irqname)
return -ENOMEM;
ret = devm_request_threaded_irq(dmac->dev, channel->irq,
rz_dmac_irq_handler,
rz_dmac_irq_handler_thread, 0,
irqname, channel);
if (ret) {
dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
channel->irq, ret);
return ret;
}
/* Set io base address for each channel */
if (index < 8) {
channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
EACH_CHANNEL_OFFSET * index;
channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
} else {
channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
EACH_CHANNEL_OFFSET * (index - 8);
channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
}
/* Allocate descriptors */
lmdesc = dma_alloc_coherent(&pdev->dev,
sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
&channel->lmdesc.base_dma, GFP_KERNEL);
if (!lmdesc) {
dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
return -ENOMEM;
}
rz_lmdesc_setup(channel, lmdesc);
/* Initialize register for each channel */
rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
channel->vc.desc_free = rz_dmac_virt_desc_free;
vchan_init(&channel->vc, &dmac->engine);
INIT_LIST_HEAD(&channel->ld_queue);
INIT_LIST_HEAD(&channel->ld_free);
INIT_LIST_HEAD(&channel->ld_active);
return 0;
}
static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
{
struct device_node *np = dev->of_node;
int ret;
ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
if (ret < 0) {
dev_err(dev, "unable to read dma-channels property\n");
return ret;
}
if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
return -EINVAL;
}
return 0;
}
static int rz_dmac_probe(struct platform_device *pdev)
{
const char *irqname = "error";
struct dma_device *engine;
struct rz_dmac *dmac;
int channel_num;
unsigned int i;
int ret;
int irq;
dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
if (!dmac)
return -ENOMEM;
dmac->dev = &pdev->dev;
platform_set_drvdata(pdev, dmac);
ret = rz_dmac_parse_of(&pdev->dev, dmac);
if (ret < 0)
return ret;
dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
sizeof(*dmac->channels), GFP_KERNEL);
if (!dmac->channels)
return -ENOMEM;
/* Request resources */
dmac->base = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(dmac->base))
return PTR_ERR(dmac->base);
dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
if (IS_ERR(dmac->ext_base))
return PTR_ERR(dmac->ext_base);
/* Register interrupt handler for error */
irq = platform_get_irq_byname(pdev, irqname);
if (irq < 0)
return irq;
ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
irqname, NULL);
if (ret) {
dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
irq, ret);
return ret;
}
/* Initialize the channels. */
INIT_LIST_HEAD(&dmac->engine.channels);
for (i = 0; i < dmac->n_channels; i++) {
ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
if (ret < 0)
goto err;
}
/* Register the DMAC as a DMA provider for DT. */
ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
NULL);
if (ret < 0)
goto err;
/* Register the DMA engine device. */
engine = &dmac->engine;
dma_cap_set(DMA_SLAVE, engine->cap_mask);
dma_cap_set(DMA_MEMCPY, engine->cap_mask);
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
engine->dev = &pdev->dev;
engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
engine->device_free_chan_resources = rz_dmac_free_chan_resources;
engine->device_tx_status = dma_cookie_status;
engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
engine->device_config = rz_dmac_config;
engine->device_terminate_all = rz_dmac_terminate_all;
engine->device_issue_pending = rz_dmac_issue_pending;
engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
dma_set_max_seg_size(engine->dev, U32_MAX);
ret = dma_async_device_register(engine);
if (ret < 0) {
dev_err(&pdev->dev, "unable to register\n");
goto dma_register_err;
}
return 0;
dma_register_err:
of_dma_controller_free(pdev->dev.of_node);
err:
channel_num = i ? i - 1 : 0;
for (i = 0; i < channel_num; i++) {
struct rz_dmac_chan *channel = &dmac->channels[i];
dma_free_coherent(&pdev->dev,
sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
channel->lmdesc.base,
channel->lmdesc.base_dma);
}
return ret;
}
static int rz_dmac_remove(struct platform_device *pdev)
{
struct rz_dmac *dmac = platform_get_drvdata(pdev);
unsigned int i;
for (i = 0; i < dmac->n_channels; i++) {
struct rz_dmac_chan *channel = &dmac->channels[i];
dma_free_coherent(&pdev->dev,
sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
channel->lmdesc.base,
channel->lmdesc.base_dma);
}
of_dma_controller_free(pdev->dev.of_node);
dma_async_device_unregister(&dmac->engine);
return 0;
}
static const struct of_device_id of_rz_dmac_match[] = {
{ .compatible = "renesas,rz-dmac", },
{ /* Sentinel */ }
};
MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
static struct platform_driver rz_dmac_driver = {
.driver = {
.name = "rz-dmac",
.of_match_table = of_rz_dmac_match,
},
.probe = rz_dmac_probe,
.remove = rz_dmac_remove,
};
module_platform_driver(rz_dmac_driver);
MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
MODULE_LICENSE("GPL v2");

2
drivers/dma/sh/usb-dmac.c
View File

@ -466,7 +466,7 @@ static int usb_dmac_chan_terminate_all(struct dma_chan *chan)
static unsigned int usb_dmac_get_current_residue(struct usb_dmac_chan *chan,
struct usb_dmac_desc *desc,
int sg_index)
unsigned int sg_index)
{
struct usb_dmac_sg *sg = desc->sg + sg_index;
u32 mem_addr = sg->mem_addr & 0xffffffff;

1
drivers/dma/sprd-dma.c
View File

@ -1265,6 +1265,7 @@ static const struct of_device_id sprd_dma_match[] = {
{ .compatible = "sprd,sc9860-dma", },
{},
};
MODULE_DEVICE_TABLE(of, sprd_dma_match);
static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
{

8
drivers/dma/stm32-dma.c
View File

@ -60,6 +60,7 @@
#define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11)
#define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6)
#define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6)
#define STM32_DMA_SCR_TRBUFF BIT(20) /* Bufferable transfer for USART/UART */
#define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */
#define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */
#define STM32_DMA_SCR_PINCOS BIT(15) /* Peripheral inc offset size */
@ -138,8 +139,9 @@
#define STM32_DMA_THRESHOLD_FTR_MASK GENMASK(1, 0)
#define STM32_DMA_THRESHOLD_FTR_GET(n) ((n) & STM32_DMA_THRESHOLD_FTR_MASK)
#define STM32_DMA_DIRECT_MODE_MASK BIT(2)
#define STM32_DMA_DIRECT_MODE_GET(n) (((n) & STM32_DMA_DIRECT_MODE_MASK) \
>> 2)
#define STM32_DMA_DIRECT_MODE_GET(n) (((n) & STM32_DMA_DIRECT_MODE_MASK) >> 2)
#define STM32_DMA_ALT_ACK_MODE_MASK BIT(4)
#define STM32_DMA_ALT_ACK_MODE_GET(n) (((n) & STM32_DMA_ALT_ACK_MODE_MASK) >> 4)
enum stm32_dma_width {
STM32_DMA_BYTE,
@ -1252,6 +1254,8 @@ static void stm32_dma_set_config(struct stm32_dma_chan *chan,
chan->threshold = STM32_DMA_THRESHOLD_FTR_GET(cfg->features);
if (STM32_DMA_DIRECT_MODE_GET(cfg->features))
chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE;
if (STM32_DMA_ALT_ACK_MODE_GET(cfg->features))
chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF;
}
static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec,

7
drivers/dma/tegra210-adma.c
View File

@ -655,9 +655,8 @@ static int tegra_adma_alloc_chan_resources(struct dma_chan *dc)
return ret;
}
ret = pm_runtime_get_sync(tdc2dev(tdc));
ret = pm_runtime_resume_and_get(tdc2dev(tdc));
if (ret < 0) {
pm_runtime_put_noidle(tdc2dev(tdc));
free_irq(tdc->irq, tdc);
return ret;
}
@ -869,10 +868,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
pm_runtime_enable(&pdev->dev);
ret = pm_runtime_get_sync(&pdev->dev);
if (ret < 0) {
pm_runtime_put_noidle(&pdev->dev);
if (ret < 0)
goto rpm_disable;
}
ret = tegra_adma_init(tdma);
if (ret)

73
drivers/dma/ti/k3-psil-j721e.c
View File

@ -58,6 +58,14 @@
}, \
}
#define PSIL_CSI2RX(x) \
{ \
.thread_id = x, \
.ep_config = { \
.ep_type = PSIL_EP_NATIVE, \
}, \
}
/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */
static struct psil_ep j721e_src_ep_map[] = {
/* SA2UL */
@ -138,6 +146,71 @@ static struct psil_ep j721e_src_ep_map[] = {
PSIL_PDMA_XY_PKT(0x4707),
PSIL_PDMA_XY_PKT(0x4708),
PSIL_PDMA_XY_PKT(0x4709),
/* CSI2RX */
PSIL_CSI2RX(0x4940),
PSIL_CSI2RX(0x4941),
PSIL_CSI2RX(0x4942),
PSIL_CSI2RX(0x4943),
PSIL_CSI2RX(0x4944),
PSIL_CSI2RX(0x4945),
PSIL_CSI2RX(0x4946),
PSIL_CSI2RX(0x4947),
PSIL_CSI2RX(0x4948),
PSIL_CSI2RX(0x4949),
PSIL_CSI2RX(0x494a),
PSIL_CSI2RX(0x494b),
PSIL_CSI2RX(0x494c),
PSIL_CSI2RX(0x494d),
PSIL_CSI2RX(0x494e),
PSIL_CSI2RX(0x494f),
PSIL_CSI2RX(0x4950),
PSIL_CSI2RX(0x4951),
PSIL_CSI2RX(0x4952),
PSIL_CSI2RX(0x4953),
PSIL_CSI2RX(0x4954),
PSIL_CSI2RX(0x4955),
PSIL_CSI2RX(0x4956),
PSIL_CSI2RX(0x4957),
PSIL_CSI2RX(0x4958),
PSIL_CSI2RX(0x4959),
PSIL_CSI2RX(0x495a),
PSIL_CSI2RX(0x495b),
PSIL_CSI2RX(0x495c),
PSIL_CSI2RX(0x495d),
PSIL_CSI2RX(0x495e),
PSIL_CSI2RX(0x495f),
PSIL_CSI2RX(0x4960),
PSIL_CSI2RX(0x4961),
PSIL_CSI2RX(0x4962),
PSIL_CSI2RX(0x4963),
PSIL_CSI2RX(0x4964),
PSIL_CSI2RX(0x4965),
PSIL_CSI2RX(0x4966),
PSIL_CSI2RX(0x4967),
PSIL_CSI2RX(0x4968),
PSIL_CSI2RX(0x4969),
PSIL_CSI2RX(0x496a),
PSIL_CSI2RX(0x496b),
PSIL_CSI2RX(0x496c),
PSIL_CSI2RX(0x496d),
PSIL_CSI2RX(0x496e),
PSIL_CSI2RX(0x496f),
PSIL_CSI2RX(0x4970),
PSIL_CSI2RX(0x4971),
PSIL_CSI2RX(0x4972),
PSIL_CSI2RX(0x4973),
PSIL_CSI2RX(0x4974),
PSIL_CSI2RX(0x4975),
PSIL_CSI2RX(0x4976),
PSIL_CSI2RX(0x4977),
PSIL_CSI2RX(0x4978),
PSIL_CSI2RX(0x4979),
PSIL_CSI2RX(0x497a),
PSIL_CSI2RX(0x497b),
PSIL_CSI2RX(0x497c),
PSIL_CSI2RX(0x497d),
PSIL_CSI2RX(0x497e),
PSIL_CSI2RX(0x497f),
/* CPSW9 */
PSIL_ETHERNET(0x4a00),
/* CPSW0 */

17
drivers/dma/xilinx/xilinx_dma.c
View File

@ -1420,8 +1420,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
chan->desc_submitcount++;
chan->desc_pendingcount--;
list_del(&desc->node);
list_add_tail(&desc->node, &chan->active_list);
list_move_tail(&desc->node, &chan->active_list);
if (chan->desc_submitcount == chan->num_frms)
chan->desc_submitcount = 0;
@ -1658,6 +1657,17 @@ static void xilinx_dma_issue_pending(struct dma_chan *dchan)
spin_unlock_irqrestore(&chan->lock, flags);
}
/**
* xilinx_dma_device_config - Configure the DMA channel
* @dchan: DMA channel
* @config: channel configuration
*/
static int xilinx_dma_device_config(struct dma_chan *dchan,
struct dma_slave_config *config)
{
return 0;
}
/**
* xilinx_dma_complete_descriptor - Mark the active descriptor as complete
* @chan : xilinx DMA channel
@ -3077,7 +3087,7 @@ static int xilinx_dma_probe(struct platform_device *pdev)
xdev->ext_addr = false;
/* Set the dma mask bits */
dma_set_mask(xdev->dev, DMA_BIT_MASK(addr_width));
dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));
/* Initialize the DMA engine */
xdev->common.dev = &pdev->dev;
@ -3096,6 +3106,7 @@ static int xilinx_dma_probe(struct platform_device *pdev)
xdev->common.device_synchronize = xilinx_dma_synchronize;
xdev->common.device_tx_status = xilinx_dma_tx_status;
xdev->common.device_issue_pending = xilinx_dma_issue_pending;
xdev->common.device_config = xilinx_dma_device_config;
if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask);
xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg;

3
drivers/dma/xilinx/zynqmp_dma.c
View File

@ -434,8 +434,7 @@ static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan,
struct zynqmp_dma_desc_sw *child, *next;
chan->desc_free_cnt++;
list_del(&sdesc->node);
list_add_tail(&sdesc->node, &chan->free_list);
list_move_tail(&sdesc->node, &chan->free_list);
list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {
chan->desc_free_cnt++;
list_move_tail(&child->node, &chan->free_list);

3
include/linux/dmaengine.h
View File

@ -380,6 +380,7 @@ enum dma_slave_buswidth {
DMA_SLAVE_BUSWIDTH_16_BYTES = 16,
DMA_SLAVE_BUSWIDTH_32_BYTES = 32,
DMA_SLAVE_BUSWIDTH_64_BYTES = 64,
DMA_SLAVE_BUSWIDTH_128_BYTES = 128,
};
/**
@ -398,7 +399,7 @@ enum dma_slave_buswidth {
* @src_addr_width: this is the width in bytes of the source (RX)
* register where DMA data shall be read. If the source
* is memory this may be ignored depending on architecture.
* Legal values: 1, 2, 3, 4, 8, 16, 32, 64.
* Legal values: 1, 2, 3, 4, 8, 16, 32, 64, 128.
* @dst_addr_width: same as src_addr_width but for destination
* target (TX) mutatis mutandis.
* @src_maxburst: the maximum number of words (note: words, as in

21
include/linux/platform_data/dma-dw.h
View File

@ -41,36 +41,39 @@ struct dw_dma_slave {
/**
* struct dw_dma_platform_data - Controller configuration parameters
* @nr_masters: Number of AHB masters supported by the controller
* @nr_channels: Number of channels supported by hardware (max 8)
* @chan_allocation_order: Allocate channels starting from 0 or 7
* @chan_priority: Set channel priority increasing from 0 to 7 or 7 to 0.
* @block_size: Maximum block size supported by the controller
* @nr_masters: Number of AHB masters supported by the controller
* @data_width: Maximum data width supported by hardware per AHB master
* (in bytes, power of 2)
* @multi_block: Multi block transfers supported by hardware per channel.
* @max_burst: Maximum value of burst transaction size supported by hardware
* per channel (in units of CTL.SRC_TR_WIDTH/CTL.DST_TR_WIDTH).
* @protctl: Protection control signals setting per channel.
* @quirks: Optional platform quirks.
*/
struct dw_dma_platform_data {
unsigned int nr_channels;
u32 nr_masters;
u32 nr_channels;
#define CHAN_ALLOCATION_ASCENDING 0 /* zero to seven */
#define CHAN_ALLOCATION_DESCENDING 1 /* seven to zero */
unsigned char chan_allocation_order;
u32 chan_allocation_order;
#define CHAN_PRIORITY_ASCENDING 0 /* chan0 highest */
#define CHAN_PRIORITY_DESCENDING 1 /* chan7 highest */
unsigned char chan_priority;
unsigned int block_size;
unsigned char nr_masters;
unsigned char data_width[DW_DMA_MAX_NR_MASTERS];
unsigned char multi_block[DW_DMA_MAX_NR_CHANNELS];
u32 chan_priority;
u32 block_size;
u32 data_width[DW_DMA_MAX_NR_MASTERS];
u32 multi_block[DW_DMA_MAX_NR_CHANNELS];
u32 max_burst[DW_DMA_MAX_NR_CHANNELS];
#define CHAN_PROTCTL_PRIVILEGED BIT(0)
#define CHAN_PROTCTL_BUFFERABLE BIT(1)
#define CHAN_PROTCTL_CACHEABLE BIT(2)
#define CHAN_PROTCTL_MASK GENMASK(2, 0)
unsigned char protctl;
u32 protctl;
#define DW_DMA_QUIRK_XBAR_PRESENT BIT(0)
u32 quirks;
};
#endif /* _PLATFORM_DATA_DMA_DW_H */

24
include/uapi/linux/idxd.h
View File

@ -9,6 +9,30 @@
#include <stdint.h>
#endif
/* Driver command error status */
enum idxd_scmd_stat {
IDXD_SCMD_DEV_ENABLED = 0x80000010,
IDXD_SCMD_DEV_NOT_ENABLED = 0x80000020,
IDXD_SCMD_WQ_ENABLED = 0x80000021,
IDXD_SCMD_DEV_DMA_ERR = 0x80020000,
IDXD_SCMD_WQ_NO_GRP = 0x80030000,
IDXD_SCMD_WQ_NO_NAME = 0x80040000,
IDXD_SCMD_WQ_NO_SVM = 0x80050000,
IDXD_SCMD_WQ_NO_THRESH = 0x80060000,
IDXD_SCMD_WQ_PORTAL_ERR = 0x80070000,
IDXD_SCMD_WQ_RES_ALLOC_ERR = 0x80080000,
IDXD_SCMD_PERCPU_ERR = 0x80090000,
IDXD_SCMD_DMA_CHAN_ERR = 0x800a0000,
IDXD_SCMD_CDEV_ERR = 0x800b0000,
IDXD_SCMD_WQ_NO_SWQ_SUPPORT = 0x800c0000,
IDXD_SCMD_WQ_NONE_CONFIGURED = 0x800d0000,
IDXD_SCMD_WQ_NO_SIZE = 0x800e0000,
IDXD_SCMD_WQ_NO_PRIV = 0x800f0000,
};
#define IDXD_SCMD_SOFTERR_MASK 0x80000000
#define IDXD_SCMD_SOFTERR_SHIFT 16
/* Descriptor flags */
#define IDXD_OP_FLAG_FENCE 0x0001
#define IDXD_OP_FLAG_BOF 0x0002