2018-05-22 02:53:30 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0+
|
|
|
|
//
|
|
|
|
// drivers/dma/imx-sdma.c
|
|
|
|
//
|
|
|
|
// This file contains a driver for the Freescale Smart DMA engine
|
|
|
|
//
|
|
|
|
// Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
|
|
|
|
//
|
|
|
|
// Based on code from Freescale:
|
|
|
|
//
|
|
|
|
// Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved.
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
#include <linux/init.h>
|
dmaengine: imx-sdma: ack channel 0 IRQ in the interrupt handler
Currently the handler ignores the channel 0 interrupt and thus doesn't ack
it properly. This is done in order to allow sdma_run_channel0() to poll
on the irq status bit, as this function may be called in atomic context,
but needs to know when the channel has finished.
This works mostly, as the polling happens under a spinlock, disabling IRQs
on the local CPU, leaving only a very slight race window for a spurious
IRQ to happen if the handler is executed on another CPU in an SMP system.
Still this is clearly suboptimal.
This behavior turns into a real problem on an RT system, where the spinlock
doesn't disable IRQs on the local CPU. Not acking the IRQ in the handler
in such a setup is very likely to drown the CPU in an IRQ storm, leaving
it unable to make any progress in the polling loop, leading to the IRQ
never being acked.
Fix this by properly acknowledging the channel 0 IRQ in the handler.
As the IRQ status bit can no longer be used to poll for the channel
completion, switch over to using the SDMA_H_STATSTOP register for this
purpose, where bit 0 is cleared by the hardware when the channel is done.
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2016-07-07 09:35:51 +00:00
|
|
|
#include <linux/iopoll.h>
|
2011-08-30 07:08:24 +00:00
|
|
|
#include <linux/module.h>
|
2010-09-30 13:56:34 +00:00
|
|
|
#include <linux/types.h>
|
2012-01-13 03:10:01 +00:00
|
|
|
#include <linux/bitops.h>
|
2010-09-30 13:56:34 +00:00
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/clk.h>
|
2012-05-11 07:14:27 +00:00
|
|
|
#include <linux/delay.h>
|
2010-09-30 13:56:34 +00:00
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/semaphore.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/dma-mapping.h>
|
|
|
|
#include <linux/firmware.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/platform_device.h>
|
|
|
|
#include <linux/dmaengine.h>
|
2011-07-14 00:35:48 +00:00
|
|
|
#include <linux/of.h>
|
2015-07-10 09:08:16 +00:00
|
|
|
#include <linux/of_address.h>
|
2011-07-14 00:35:48 +00:00
|
|
|
#include <linux/of_device.h>
|
2013-05-30 14:23:32 +00:00
|
|
|
#include <linux/of_dma.h>
|
2018-11-06 03:40:33 +00:00
|
|
|
#include <linux/workqueue.h>
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
#include <asm/irq.h>
|
2012-08-24 13:14:29 +00:00
|
|
|
#include <linux/platform_data/dma-imx-sdma.h>
|
|
|
|
#include <linux/platform_data/dma-imx.h>
|
2015-07-23 03:40:49 +00:00
|
|
|
#include <linux/regmap.h>
|
|
|
|
#include <linux/mfd/syscon.h>
|
|
|
|
#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-03-06 22:34:26 +00:00
|
|
|
#include "dmaengine.h"
|
2018-06-19 16:57:00 +00:00
|
|
|
#include "virt-dma.h"
|
2012-03-06 22:34:26 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/* SDMA registers */
|
|
|
|
#define SDMA_H_C0PTR 0x000
|
|
|
|
#define SDMA_H_INTR 0x004
|
|
|
|
#define SDMA_H_STATSTOP 0x008
|
|
|
|
#define SDMA_H_START 0x00c
|
|
|
|
#define SDMA_H_EVTOVR 0x010
|
|
|
|
#define SDMA_H_DSPOVR 0x014
|
|
|
|
#define SDMA_H_HOSTOVR 0x018
|
|
|
|
#define SDMA_H_EVTPEND 0x01c
|
|
|
|
#define SDMA_H_DSPENBL 0x020
|
|
|
|
#define SDMA_H_RESET 0x024
|
|
|
|
#define SDMA_H_EVTERR 0x028
|
|
|
|
#define SDMA_H_INTRMSK 0x02c
|
|
|
|
#define SDMA_H_PSW 0x030
|
|
|
|
#define SDMA_H_EVTERRDBG 0x034
|
|
|
|
#define SDMA_H_CONFIG 0x038
|
|
|
|
#define SDMA_ONCE_ENB 0x040
|
|
|
|
#define SDMA_ONCE_DATA 0x044
|
|
|
|
#define SDMA_ONCE_INSTR 0x048
|
|
|
|
#define SDMA_ONCE_STAT 0x04c
|
|
|
|
#define SDMA_ONCE_CMD 0x050
|
|
|
|
#define SDMA_EVT_MIRROR 0x054
|
|
|
|
#define SDMA_ILLINSTADDR 0x058
|
|
|
|
#define SDMA_CHN0ADDR 0x05c
|
|
|
|
#define SDMA_ONCE_RTB 0x060
|
|
|
|
#define SDMA_XTRIG_CONF1 0x070
|
|
|
|
#define SDMA_XTRIG_CONF2 0x074
|
2011-07-13 13:33:17 +00:00
|
|
|
#define SDMA_CHNENBL0_IMX35 0x200
|
|
|
|
#define SDMA_CHNENBL0_IMX31 0x080
|
2010-09-30 13:56:34 +00:00
|
|
|
#define SDMA_CHNPRI_0 0x100
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer descriptor status values.
|
|
|
|
*/
|
|
|
|
#define BD_DONE 0x01
|
|
|
|
#define BD_WRAP 0x02
|
|
|
|
#define BD_CONT 0x04
|
|
|
|
#define BD_INTR 0x08
|
|
|
|
#define BD_RROR 0x10
|
|
|
|
#define BD_LAST 0x20
|
|
|
|
#define BD_EXTD 0x80
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Data Node descriptor status values.
|
|
|
|
*/
|
|
|
|
#define DND_END_OF_FRAME 0x80
|
|
|
|
#define DND_END_OF_XFER 0x40
|
|
|
|
#define DND_DONE 0x20
|
|
|
|
#define DND_UNUSED 0x01
|
|
|
|
|
|
|
|
/*
|
|
|
|
* IPCV2 descriptor status values.
|
|
|
|
*/
|
|
|
|
#define BD_IPCV2_END_OF_FRAME 0x40
|
|
|
|
|
|
|
|
#define IPCV2_MAX_NODES 50
|
|
|
|
/*
|
|
|
|
* Error bit set in the CCB status field by the SDMA,
|
|
|
|
* in setbd routine, in case of a transfer error
|
|
|
|
*/
|
|
|
|
#define DATA_ERROR 0x10000000
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer descriptor commands.
|
|
|
|
*/
|
|
|
|
#define C0_ADDR 0x01
|
|
|
|
#define C0_LOAD 0x02
|
|
|
|
#define C0_DUMP 0x03
|
|
|
|
#define C0_SETCTX 0x07
|
|
|
|
#define C0_GETCTX 0x03
|
|
|
|
#define C0_SETDM 0x01
|
|
|
|
#define C0_SETPM 0x04
|
|
|
|
#define C0_GETDM 0x02
|
|
|
|
#define C0_GETPM 0x08
|
|
|
|
/*
|
|
|
|
* Change endianness indicator in the BD command field
|
|
|
|
*/
|
|
|
|
#define CHANGE_ENDIANNESS 0x80
|
|
|
|
|
2015-07-10 09:08:16 +00:00
|
|
|
/*
|
|
|
|
* p_2_p watermark_level description
|
|
|
|
* Bits Name Description
|
|
|
|
* 0-7 Lower WML Lower watermark level
|
|
|
|
* 8 PS 1: Pad Swallowing
|
|
|
|
* 0: No Pad Swallowing
|
|
|
|
* 9 PA 1: Pad Adding
|
|
|
|
* 0: No Pad Adding
|
|
|
|
* 10 SPDIF If this bit is set both source
|
|
|
|
* and destination are on SPBA
|
|
|
|
* 11 Source Bit(SP) 1: Source on SPBA
|
|
|
|
* 0: Source on AIPS
|
|
|
|
* 12 Destination Bit(DP) 1: Destination on SPBA
|
|
|
|
* 0: Destination on AIPS
|
|
|
|
* 13-15 --------- MUST BE 0
|
|
|
|
* 16-23 Higher WML HWML
|
|
|
|
* 24-27 N Total number of samples after
|
|
|
|
* which Pad adding/Swallowing
|
|
|
|
* must be done. It must be odd.
|
|
|
|
* 28 Lower WML Event(LWE) SDMA events reg to check for
|
|
|
|
* LWML event mask
|
|
|
|
* 0: LWE in EVENTS register
|
|
|
|
* 1: LWE in EVENTS2 register
|
|
|
|
* 29 Higher WML Event(HWE) SDMA events reg to check for
|
|
|
|
* HWML event mask
|
|
|
|
* 0: HWE in EVENTS register
|
|
|
|
* 1: HWE in EVENTS2 register
|
|
|
|
* 30 --------- MUST BE 0
|
|
|
|
* 31 CONT 1: Amount of samples to be
|
|
|
|
* transferred is unknown and
|
|
|
|
* script will keep on
|
|
|
|
* transferring samples as long as
|
|
|
|
* both events are detected and
|
|
|
|
* script must be manually stopped
|
|
|
|
* by the application
|
|
|
|
* 0: The amount of samples to be
|
|
|
|
* transferred is equal to the
|
|
|
|
* count field of mode word
|
|
|
|
*/
|
|
|
|
#define SDMA_WATERMARK_LEVEL_LWML 0xFF
|
|
|
|
#define SDMA_WATERMARK_LEVEL_PS BIT(8)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_PA BIT(9)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_SP BIT(11)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_DP BIT(12)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_LWE BIT(28)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_HWE BIT(29)
|
|
|
|
#define SDMA_WATERMARK_LEVEL_CONT BIT(31)
|
|
|
|
|
2017-09-14 18:46:43 +00:00
|
|
|
#define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
|
|
|
|
BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
|
|
|
|
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
|
|
|
|
|
|
|
|
#define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \
|
|
|
|
BIT(DMA_MEM_TO_DEV) | \
|
|
|
|
BIT(DMA_DEV_TO_DEV))
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/*
|
|
|
|
* Mode/Count of data node descriptors - IPCv2
|
|
|
|
*/
|
|
|
|
struct sdma_mode_count {
|
2018-07-23 17:46:10 +00:00
|
|
|
#define SDMA_BD_MAX_CNT 0xffff
|
2010-09-30 13:56:34 +00:00
|
|
|
u32 count : 16; /* size of the buffer pointed by this BD */
|
|
|
|
u32 status : 8; /* E,R,I,C,W,D status bits stored here */
|
2016-08-08 20:45:58 +00:00
|
|
|
u32 command : 8; /* command mostly used for channel 0 */
|
2010-09-30 13:56:34 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Buffer descriptor
|
|
|
|
*/
|
|
|
|
struct sdma_buffer_descriptor {
|
|
|
|
struct sdma_mode_count mode;
|
|
|
|
u32 buffer_addr; /* address of the buffer described */
|
|
|
|
u32 ext_buffer_addr; /* extended buffer address */
|
|
|
|
} __attribute__ ((packed));
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct sdma_channel_control - Channel control Block
|
|
|
|
*
|
2018-07-04 10:06:42 +00:00
|
|
|
* @current_bd_ptr: current buffer descriptor processed
|
|
|
|
* @base_bd_ptr: first element of buffer descriptor array
|
|
|
|
* @unused: padding. The SDMA engine expects an array of 128 byte
|
2010-09-30 13:56:34 +00:00
|
|
|
* control blocks
|
|
|
|
*/
|
|
|
|
struct sdma_channel_control {
|
|
|
|
u32 current_bd_ptr;
|
|
|
|
u32 base_bd_ptr;
|
|
|
|
u32 unused[2];
|
|
|
|
} __attribute__ ((packed));
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct sdma_state_registers - SDMA context for a channel
|
|
|
|
*
|
|
|
|
* @pc: program counter
|
2018-07-04 10:06:42 +00:00
|
|
|
* @unused1: unused
|
2010-09-30 13:56:34 +00:00
|
|
|
* @t: test bit: status of arithmetic & test instruction
|
|
|
|
* @rpc: return program counter
|
2018-07-04 10:06:42 +00:00
|
|
|
* @unused0: unused
|
2010-09-30 13:56:34 +00:00
|
|
|
* @sf: source fault while loading data
|
|
|
|
* @spc: loop start program counter
|
2018-07-04 10:06:42 +00:00
|
|
|
* @unused2: unused
|
2010-09-30 13:56:34 +00:00
|
|
|
* @df: destination fault while storing data
|
|
|
|
* @epc: loop end program counter
|
|
|
|
* @lm: loop mode
|
|
|
|
*/
|
|
|
|
struct sdma_state_registers {
|
|
|
|
u32 pc :14;
|
|
|
|
u32 unused1: 1;
|
|
|
|
u32 t : 1;
|
|
|
|
u32 rpc :14;
|
|
|
|
u32 unused0: 1;
|
|
|
|
u32 sf : 1;
|
|
|
|
u32 spc :14;
|
|
|
|
u32 unused2: 1;
|
|
|
|
u32 df : 1;
|
|
|
|
u32 epc :14;
|
|
|
|
u32 lm : 2;
|
|
|
|
} __attribute__ ((packed));
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct sdma_context_data - sdma context specific to a channel
|
|
|
|
*
|
|
|
|
* @channel_state: channel state bits
|
|
|
|
* @gReg: general registers
|
|
|
|
* @mda: burst dma destination address register
|
|
|
|
* @msa: burst dma source address register
|
|
|
|
* @ms: burst dma status register
|
|
|
|
* @md: burst dma data register
|
|
|
|
* @pda: peripheral dma destination address register
|
|
|
|
* @psa: peripheral dma source address register
|
|
|
|
* @ps: peripheral dma status register
|
|
|
|
* @pd: peripheral dma data register
|
|
|
|
* @ca: CRC polynomial register
|
|
|
|
* @cs: CRC accumulator register
|
|
|
|
* @dda: dedicated core destination address register
|
|
|
|
* @dsa: dedicated core source address register
|
|
|
|
* @ds: dedicated core status register
|
|
|
|
* @dd: dedicated core data register
|
2018-07-04 10:06:42 +00:00
|
|
|
* @scratch0: 1st word of dedicated ram for context switch
|
|
|
|
* @scratch1: 2nd word of dedicated ram for context switch
|
|
|
|
* @scratch2: 3rd word of dedicated ram for context switch
|
|
|
|
* @scratch3: 4th word of dedicated ram for context switch
|
|
|
|
* @scratch4: 5th word of dedicated ram for context switch
|
|
|
|
* @scratch5: 6th word of dedicated ram for context switch
|
|
|
|
* @scratch6: 7th word of dedicated ram for context switch
|
|
|
|
* @scratch7: 8th word of dedicated ram for context switch
|
2010-09-30 13:56:34 +00:00
|
|
|
*/
|
|
|
|
struct sdma_context_data {
|
|
|
|
struct sdma_state_registers channel_state;
|
|
|
|
u32 gReg[8];
|
|
|
|
u32 mda;
|
|
|
|
u32 msa;
|
|
|
|
u32 ms;
|
|
|
|
u32 md;
|
|
|
|
u32 pda;
|
|
|
|
u32 psa;
|
|
|
|
u32 ps;
|
|
|
|
u32 pd;
|
|
|
|
u32 ca;
|
|
|
|
u32 cs;
|
|
|
|
u32 dda;
|
|
|
|
u32 dsa;
|
|
|
|
u32 ds;
|
|
|
|
u32 dd;
|
|
|
|
u32 scratch0;
|
|
|
|
u32 scratch1;
|
|
|
|
u32 scratch2;
|
|
|
|
u32 scratch3;
|
|
|
|
u32 scratch4;
|
|
|
|
u32 scratch5;
|
|
|
|
u32 scratch6;
|
|
|
|
u32 scratch7;
|
|
|
|
} __attribute__ ((packed));
|
|
|
|
|
|
|
|
|
|
|
|
struct sdma_engine;
|
|
|
|
|
2018-06-19 16:56:59 +00:00
|
|
|
/**
|
|
|
|
* struct sdma_desc - descriptor structor for one transfer
|
2018-07-04 10:06:42 +00:00
|
|
|
* @vd: descriptor for virt dma
|
|
|
|
* @num_bd: number of descriptors currently handling
|
|
|
|
* @bd_phys: physical address of bd
|
|
|
|
* @buf_tail: ID of the buffer that was processed
|
|
|
|
* @buf_ptail: ID of the previous buffer that was processed
|
|
|
|
* @period_len: period length, used in cyclic.
|
|
|
|
* @chn_real_count: the real count updated from bd->mode.count
|
|
|
|
* @chn_count: the transfer count set
|
|
|
|
* @sdmac: sdma_channel pointer
|
|
|
|
* @bd: pointer of allocate bd
|
2018-06-19 16:56:59 +00:00
|
|
|
*/
|
|
|
|
struct sdma_desc {
|
2018-06-19 16:57:00 +00:00
|
|
|
struct virt_dma_desc vd;
|
2018-06-19 16:56:59 +00:00
|
|
|
unsigned int num_bd;
|
|
|
|
dma_addr_t bd_phys;
|
|
|
|
unsigned int buf_tail;
|
|
|
|
unsigned int buf_ptail;
|
|
|
|
unsigned int period_len;
|
|
|
|
unsigned int chn_real_count;
|
|
|
|
unsigned int chn_count;
|
|
|
|
struct sdma_channel *sdmac;
|
|
|
|
struct sdma_buffer_descriptor *bd;
|
|
|
|
};
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/**
|
|
|
|
* struct sdma_channel - housekeeping for a SDMA channel
|
|
|
|
*
|
2018-07-04 10:06:42 +00:00
|
|
|
* @vc: virt_dma base structure
|
|
|
|
* @desc: sdma description including vd and other special member
|
|
|
|
* @sdma: pointer to the SDMA engine for this channel
|
|
|
|
* @channel: the channel number, matches dmaengine chan_id + 1
|
|
|
|
* @direction: transfer type. Needed for setting SDMA script
|
2018-10-25 14:15:28 +00:00
|
|
|
* @slave_config Slave configuration
|
2018-07-04 10:06:42 +00:00
|
|
|
* @peripheral_type: Peripheral type. Needed for setting SDMA script
|
|
|
|
* @event_id0: aka dma request line
|
|
|
|
* @event_id1: for channels that use 2 events
|
|
|
|
* @word_size: peripheral access size
|
|
|
|
* @pc_from_device: script address for those device_2_memory
|
|
|
|
* @pc_to_device: script address for those memory_2_device
|
|
|
|
* @device_to_device: script address for those device_2_device
|
2018-07-23 17:46:11 +00:00
|
|
|
* @pc_to_pc: script address for those memory_2_memory
|
2018-07-04 10:06:42 +00:00
|
|
|
* @flags: loop mode or not
|
|
|
|
* @per_address: peripheral source or destination address in common case
|
|
|
|
* destination address in p_2_p case
|
|
|
|
* @per_address2: peripheral source address in p_2_p case
|
|
|
|
* @event_mask: event mask used in p_2_p script
|
|
|
|
* @watermark_level: value for gReg[7], some script will extend it from
|
|
|
|
* basic watermark such as p_2_p
|
|
|
|
* @shp_addr: value for gReg[6]
|
|
|
|
* @per_addr: value for gReg[2]
|
|
|
|
* @status: status of dma channel
|
|
|
|
* @data: specific sdma interface structure
|
|
|
|
* @bd_pool: dma_pool for bd
|
2010-09-30 13:56:34 +00:00
|
|
|
*/
|
|
|
|
struct sdma_channel {
|
2018-06-19 16:57:00 +00:00
|
|
|
struct virt_dma_chan vc;
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_desc *desc;
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_engine *sdma;
|
|
|
|
unsigned int channel;
|
2011-10-13 17:04:23 +00:00
|
|
|
enum dma_transfer_direction direction;
|
2018-10-25 14:15:28 +00:00
|
|
|
struct dma_slave_config slave_config;
|
2010-09-30 13:56:34 +00:00
|
|
|
enum sdma_peripheral_type peripheral_type;
|
|
|
|
unsigned int event_id0;
|
|
|
|
unsigned int event_id1;
|
|
|
|
enum dma_slave_buswidth word_size;
|
|
|
|
unsigned int pc_from_device, pc_to_device;
|
2015-07-10 09:08:16 +00:00
|
|
|
unsigned int device_to_device;
|
2018-07-23 17:46:11 +00:00
|
|
|
unsigned int pc_to_pc;
|
2010-09-30 13:56:34 +00:00
|
|
|
unsigned long flags;
|
2015-07-10 09:08:16 +00:00
|
|
|
dma_addr_t per_address, per_address2;
|
2012-01-13 03:10:01 +00:00
|
|
|
unsigned long event_mask[2];
|
|
|
|
unsigned long watermark_level;
|
2010-09-30 13:56:34 +00:00
|
|
|
u32 shp_addr, per_addr;
|
|
|
|
enum dma_status status;
|
2019-01-08 12:00:16 +00:00
|
|
|
bool context_loaded;
|
2014-06-16 03:32:29 +00:00
|
|
|
struct imx_dma_data data;
|
2018-11-06 03:40:33 +00:00
|
|
|
struct work_struct terminate_worker;
|
2010-09-30 13:56:34 +00:00
|
|
|
};
|
|
|
|
|
2012-01-13 03:10:01 +00:00
|
|
|
#define IMX_DMA_SG_LOOP BIT(0)
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
#define MAX_DMA_CHANNELS 32
|
|
|
|
#define MXC_SDMA_DEFAULT_PRIORITY 1
|
|
|
|
#define MXC_SDMA_MIN_PRIORITY 1
|
|
|
|
#define MXC_SDMA_MAX_PRIORITY 7
|
|
|
|
|
|
|
|
#define SDMA_FIRMWARE_MAGIC 0x414d4453
|
|
|
|
|
|
|
|
/**
|
|
|
|
* struct sdma_firmware_header - Layout of the firmware image
|
|
|
|
*
|
2018-07-04 10:06:42 +00:00
|
|
|
* @magic: "SDMA"
|
|
|
|
* @version_major: increased whenever layout of struct
|
|
|
|
* sdma_script_start_addrs changes.
|
|
|
|
* @version_minor: firmware minor version (for binary compatible changes)
|
|
|
|
* @script_addrs_start: offset of struct sdma_script_start_addrs in this image
|
|
|
|
* @num_script_addrs: Number of script addresses in this image
|
|
|
|
* @ram_code_start: offset of SDMA ram image in this firmware image
|
|
|
|
* @ram_code_size: size of SDMA ram image
|
|
|
|
* @script_addrs: Stores the start address of the SDMA scripts
|
2010-09-30 13:56:34 +00:00
|
|
|
* (in SDMA memory space)
|
|
|
|
*/
|
|
|
|
struct sdma_firmware_header {
|
|
|
|
u32 magic;
|
|
|
|
u32 version_major;
|
|
|
|
u32 version_minor;
|
|
|
|
u32 script_addrs_start;
|
|
|
|
u32 num_script_addrs;
|
|
|
|
u32 ram_code_start;
|
|
|
|
u32 ram_code_size;
|
|
|
|
};
|
|
|
|
|
2013-08-20 08:04:31 +00:00
|
|
|
struct sdma_driver_data {
|
|
|
|
int chnenbl0;
|
|
|
|
int num_events;
|
2013-08-20 08:04:32 +00:00
|
|
|
struct sdma_script_start_addrs *script_addrs;
|
2019-03-29 15:21:29 +00:00
|
|
|
bool check_ratio;
|
2011-07-13 13:33:17 +00:00
|
|
|
};
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_engine {
|
|
|
|
struct device *dev;
|
2011-01-12 11:12:31 +00:00
|
|
|
struct device_dma_parameters dma_parms;
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_channel channel[MAX_DMA_CHANNELS];
|
|
|
|
struct sdma_channel_control *channel_control;
|
|
|
|
void __iomem *regs;
|
|
|
|
struct sdma_context_data *context;
|
|
|
|
dma_addr_t context_phys;
|
|
|
|
struct dma_device dma_device;
|
2012-03-07 08:30:06 +00:00
|
|
|
struct clk *clk_ipg;
|
|
|
|
struct clk *clk_ahb;
|
2012-05-11 07:14:27 +00:00
|
|
|
spinlock_t channel_0_lock;
|
2013-11-13 14:55:24 +00:00
|
|
|
u32 script_number;
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_script_start_addrs *script_addrs;
|
2013-08-20 08:04:31 +00:00
|
|
|
const struct sdma_driver_data *drvdata;
|
2015-07-10 09:08:16 +00:00
|
|
|
u32 spba_start_addr;
|
|
|
|
u32 spba_end_addr;
|
2016-07-02 18:30:55 +00:00
|
|
|
unsigned int irq;
|
2018-06-19 16:56:59 +00:00
|
|
|
dma_addr_t bd0_phys;
|
|
|
|
struct sdma_buffer_descriptor *bd0;
|
2019-01-28 16:03:21 +00:00
|
|
|
/* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
|
|
|
|
bool clk_ratio;
|
2013-08-20 08:04:31 +00:00
|
|
|
};
|
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
static int sdma_config_write(struct dma_chan *chan,
|
|
|
|
struct dma_slave_config *dmaengine_cfg,
|
|
|
|
enum dma_transfer_direction direction);
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx31 = {
|
2013-08-20 08:04:31 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX31,
|
|
|
|
.num_events = 32,
|
|
|
|
};
|
|
|
|
|
2013-08-20 08:04:32 +00:00
|
|
|
static struct sdma_script_start_addrs sdma_script_imx25 = {
|
|
|
|
.ap_2_ap_addr = 729,
|
|
|
|
.uart_2_mcu_addr = 904,
|
|
|
|
.per_2_app_addr = 1255,
|
|
|
|
.mcu_2_app_addr = 834,
|
|
|
|
.uartsh_2_mcu_addr = 1120,
|
|
|
|
.per_2_shp_addr = 1329,
|
|
|
|
.mcu_2_shp_addr = 1048,
|
|
|
|
.ata_2_mcu_addr = 1560,
|
|
|
|
.mcu_2_ata_addr = 1479,
|
|
|
|
.app_2_per_addr = 1189,
|
|
|
|
.app_2_mcu_addr = 770,
|
|
|
|
.shp_2_per_addr = 1407,
|
|
|
|
.shp_2_mcu_addr = 979,
|
|
|
|
};
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx25 = {
|
2013-08-20 08:04:32 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx25,
|
|
|
|
};
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx35 = {
|
2013-08-20 08:04:31 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
2010-09-30 13:56:34 +00:00
|
|
|
};
|
|
|
|
|
2013-08-20 08:04:32 +00:00
|
|
|
static struct sdma_script_start_addrs sdma_script_imx51 = {
|
|
|
|
.ap_2_ap_addr = 642,
|
|
|
|
.uart_2_mcu_addr = 817,
|
|
|
|
.mcu_2_app_addr = 747,
|
|
|
|
.mcu_2_shp_addr = 961,
|
|
|
|
.ata_2_mcu_addr = 1473,
|
|
|
|
.mcu_2_ata_addr = 1392,
|
|
|
|
.app_2_per_addr = 1033,
|
|
|
|
.app_2_mcu_addr = 683,
|
|
|
|
.shp_2_per_addr = 1251,
|
|
|
|
.shp_2_mcu_addr = 892,
|
|
|
|
};
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx51 = {
|
2013-08-20 08:04:32 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx51,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct sdma_script_start_addrs sdma_script_imx53 = {
|
|
|
|
.ap_2_ap_addr = 642,
|
|
|
|
.app_2_mcu_addr = 683,
|
|
|
|
.mcu_2_app_addr = 747,
|
|
|
|
.uart_2_mcu_addr = 817,
|
|
|
|
.shp_2_mcu_addr = 891,
|
|
|
|
.mcu_2_shp_addr = 960,
|
|
|
|
.uartsh_2_mcu_addr = 1032,
|
|
|
|
.spdif_2_mcu_addr = 1100,
|
|
|
|
.mcu_2_spdif_addr = 1134,
|
|
|
|
.firi_2_mcu_addr = 1193,
|
|
|
|
.mcu_2_firi_addr = 1290,
|
|
|
|
};
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx53 = {
|
2013-08-20 08:04:32 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx53,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct sdma_script_start_addrs sdma_script_imx6q = {
|
|
|
|
.ap_2_ap_addr = 642,
|
|
|
|
.uart_2_mcu_addr = 817,
|
|
|
|
.mcu_2_app_addr = 747,
|
|
|
|
.per_2_per_addr = 6331,
|
|
|
|
.uartsh_2_mcu_addr = 1032,
|
|
|
|
.mcu_2_shp_addr = 960,
|
|
|
|
.app_2_mcu_addr = 683,
|
|
|
|
.shp_2_mcu_addr = 891,
|
|
|
|
.spdif_2_mcu_addr = 1100,
|
|
|
|
.mcu_2_spdif_addr = 1134,
|
|
|
|
};
|
|
|
|
|
2013-09-02 00:57:12 +00:00
|
|
|
static struct sdma_driver_data sdma_imx6q = {
|
2013-08-20 08:04:32 +00:00
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx6q,
|
|
|
|
};
|
|
|
|
|
2016-08-10 16:05:05 +00:00
|
|
|
static struct sdma_script_start_addrs sdma_script_imx7d = {
|
|
|
|
.ap_2_ap_addr = 644,
|
|
|
|
.uart_2_mcu_addr = 819,
|
|
|
|
.mcu_2_app_addr = 749,
|
|
|
|
.uartsh_2_mcu_addr = 1034,
|
|
|
|
.mcu_2_shp_addr = 962,
|
|
|
|
.app_2_mcu_addr = 685,
|
|
|
|
.shp_2_mcu_addr = 893,
|
|
|
|
.spdif_2_mcu_addr = 1102,
|
|
|
|
.mcu_2_spdif_addr = 1136,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct sdma_driver_data sdma_imx7d = {
|
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx7d,
|
|
|
|
};
|
|
|
|
|
2019-03-29 15:21:29 +00:00
|
|
|
static struct sdma_driver_data sdma_imx8mq = {
|
|
|
|
.chnenbl0 = SDMA_CHNENBL0_IMX35,
|
|
|
|
.num_events = 48,
|
|
|
|
.script_addrs = &sdma_script_imx7d,
|
|
|
|
.check_ratio = 1,
|
|
|
|
};
|
|
|
|
|
2015-05-01 15:57:46 +00:00
|
|
|
static const struct platform_device_id sdma_devtypes[] = {
|
2011-07-13 13:33:17 +00:00
|
|
|
{
|
2013-08-20 08:04:32 +00:00
|
|
|
.name = "imx25-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx25,
|
|
|
|
}, {
|
2011-07-13 13:33:17 +00:00
|
|
|
.name = "imx31-sdma",
|
2013-08-20 08:04:31 +00:00
|
|
|
.driver_data = (unsigned long)&sdma_imx31,
|
2011-07-13 13:33:17 +00:00
|
|
|
}, {
|
|
|
|
.name = "imx35-sdma",
|
2013-08-20 08:04:31 +00:00
|
|
|
.driver_data = (unsigned long)&sdma_imx35,
|
2013-08-20 08:04:32 +00:00
|
|
|
}, {
|
|
|
|
.name = "imx51-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx51,
|
|
|
|
}, {
|
|
|
|
.name = "imx53-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx53,
|
|
|
|
}, {
|
|
|
|
.name = "imx6q-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx6q,
|
2016-08-10 16:05:05 +00:00
|
|
|
}, {
|
|
|
|
.name = "imx7d-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx7d,
|
2019-03-29 15:21:29 +00:00
|
|
|
}, {
|
|
|
|
.name = "imx8mq-sdma",
|
|
|
|
.driver_data = (unsigned long)&sdma_imx8mq,
|
2011-07-13 13:33:17 +00:00
|
|
|
}, {
|
|
|
|
/* sentinel */
|
|
|
|
}
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(platform, sdma_devtypes);
|
|
|
|
|
2011-07-14 00:35:48 +00:00
|
|
|
static const struct of_device_id sdma_dt_ids[] = {
|
2013-08-20 08:04:32 +00:00
|
|
|
{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
|
|
|
|
{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
|
|
|
|
{ .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },
|
2013-08-20 08:04:31 +00:00
|
|
|
{ .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },
|
2013-08-20 08:04:32 +00:00
|
|
|
{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
|
2014-02-16 19:10:55 +00:00
|
|
|
{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
|
2016-08-10 16:05:05 +00:00
|
|
|
{ .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
|
2019-03-29 15:21:29 +00:00
|
|
|
{ .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
|
2011-07-14 00:35:48 +00:00
|
|
|
{ /* sentinel */ }
|
|
|
|
};
|
|
|
|
MODULE_DEVICE_TABLE(of, sdma_dt_ids);
|
|
|
|
|
2012-01-13 03:10:01 +00:00
|
|
|
#define SDMA_H_CONFIG_DSPDMA BIT(12) /* indicates if the DSPDMA is used */
|
|
|
|
#define SDMA_H_CONFIG_RTD_PINS BIT(11) /* indicates if Real-Time Debug pins are enabled */
|
|
|
|
#define SDMA_H_CONFIG_ACR BIT(4) /* indicates if AHB freq /core freq = 2 or 1 */
|
2010-09-30 13:56:34 +00:00
|
|
|
#define SDMA_H_CONFIG_CSM (3) /* indicates which context switch mode is selected*/
|
|
|
|
|
|
|
|
static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
|
|
|
|
{
|
2013-08-20 08:04:31 +00:00
|
|
|
u32 chnenbl0 = sdma->drvdata->chnenbl0;
|
2010-09-30 13:56:34 +00:00
|
|
|
return chnenbl0 + event * 4;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_config_ownership(struct sdma_channel *sdmac,
|
|
|
|
bool event_override, bool mcu_override, bool dsp_override)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
2012-01-13 03:10:01 +00:00
|
|
|
unsigned long evt, mcu, dsp;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (event_override && mcu_override && dsp_override)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
|
|
|
|
mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
|
|
|
|
dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (dsp_override)
|
2012-01-13 03:10:01 +00:00
|
|
|
__clear_bit(channel, &dsp);
|
2010-09-30 13:56:34 +00:00
|
|
|
else
|
2012-01-13 03:10:01 +00:00
|
|
|
__set_bit(channel, &dsp);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (event_override)
|
2012-01-13 03:10:01 +00:00
|
|
|
__clear_bit(channel, &evt);
|
2010-09-30 13:56:34 +00:00
|
|
|
else
|
2012-01-13 03:10:01 +00:00
|
|
|
__set_bit(channel, &evt);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (mcu_override)
|
2012-01-13 03:10:01 +00:00
|
|
|
__clear_bit(channel, &mcu);
|
2010-09-30 13:56:34 +00:00
|
|
|
else
|
2012-01-13 03:10:01 +00:00
|
|
|
__set_bit(channel, &mcu);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
|
|
|
|
writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
|
|
|
|
writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-01-13 03:09:56 +00:00
|
|
|
static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
|
|
|
|
{
|
2012-01-13 03:10:01 +00:00
|
|
|
writel(BIT(channel), sdma->regs + SDMA_H_START);
|
2012-01-13 03:09:56 +00:00
|
|
|
}
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/*
|
2012-05-11 07:14:27 +00:00
|
|
|
* sdma_run_channel0 - run a channel and wait till it's done
|
2010-09-30 13:56:34 +00:00
|
|
|
*/
|
2012-05-11 07:14:27 +00:00
|
|
|
static int sdma_run_channel0(struct sdma_engine *sdma)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
int ret;
|
dmaengine: imx-sdma: ack channel 0 IRQ in the interrupt handler
Currently the handler ignores the channel 0 interrupt and thus doesn't ack
it properly. This is done in order to allow sdma_run_channel0() to poll
on the irq status bit, as this function may be called in atomic context,
but needs to know when the channel has finished.
This works mostly, as the polling happens under a spinlock, disabling IRQs
on the local CPU, leaving only a very slight race window for a spurious
IRQ to happen if the handler is executed on another CPU in an SMP system.
Still this is clearly suboptimal.
This behavior turns into a real problem on an RT system, where the spinlock
doesn't disable IRQs on the local CPU. Not acking the IRQ in the handler
in such a setup is very likely to drown the CPU in an IRQ storm, leaving
it unable to make any progress in the polling loop, leading to the IRQ
never being acked.
Fix this by properly acknowledging the channel 0 IRQ in the handler.
As the IRQ status bit can no longer be used to poll for the channel
completion, switch over to using the SDMA_H_STATSTOP register for this
purpose, where bit 0 is cleared by the hardware when the channel is done.
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2016-07-07 09:35:51 +00:00
|
|
|
u32 reg;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
sdma_enable_channel(sdma, 0);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
dmaengine: imx-sdma: ack channel 0 IRQ in the interrupt handler
Currently the handler ignores the channel 0 interrupt and thus doesn't ack
it properly. This is done in order to allow sdma_run_channel0() to poll
on the irq status bit, as this function may be called in atomic context,
but needs to know when the channel has finished.
This works mostly, as the polling happens under a spinlock, disabling IRQs
on the local CPU, leaving only a very slight race window for a spurious
IRQ to happen if the handler is executed on another CPU in an SMP system.
Still this is clearly suboptimal.
This behavior turns into a real problem on an RT system, where the spinlock
doesn't disable IRQs on the local CPU. Not acking the IRQ in the handler
in such a setup is very likely to drown the CPU in an IRQ storm, leaving
it unable to make any progress in the polling loop, leading to the IRQ
never being acked.
Fix this by properly acknowledging the channel 0 IRQ in the handler.
As the IRQ status bit can no longer be used to poll for the channel
completion, switch over to using the SDMA_H_STATSTOP register for this
purpose, where bit 0 is cleared by the hardware when the channel is done.
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2016-07-07 09:35:51 +00:00
|
|
|
ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,
|
|
|
|
reg, !(reg & 1), 1, 500);
|
|
|
|
if (ret)
|
2012-05-11 07:14:27 +00:00
|
|
|
dev_err(sdma->dev, "Timeout waiting for CH0 ready\n");
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2015-02-15 02:00:35 +00:00
|
|
|
/* Set bits of CONFIG register with dynamic context switching */
|
2019-01-28 16:03:21 +00:00
|
|
|
reg = readl(sdma->regs + SDMA_H_CONFIG);
|
|
|
|
if ((reg & SDMA_H_CONFIG_CSM) == 0) {
|
|
|
|
reg |= SDMA_H_CONFIG_CSM;
|
|
|
|
writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
|
|
|
|
}
|
2015-02-15 02:00:35 +00:00
|
|
|
|
dmaengine: imx-sdma: ack channel 0 IRQ in the interrupt handler
Currently the handler ignores the channel 0 interrupt and thus doesn't ack
it properly. This is done in order to allow sdma_run_channel0() to poll
on the irq status bit, as this function may be called in atomic context,
but needs to know when the channel has finished.
This works mostly, as the polling happens under a spinlock, disabling IRQs
on the local CPU, leaving only a very slight race window for a spurious
IRQ to happen if the handler is executed on another CPU in an SMP system.
Still this is clearly suboptimal.
This behavior turns into a real problem on an RT system, where the spinlock
doesn't disable IRQs on the local CPU. Not acking the IRQ in the handler
in such a setup is very likely to drown the CPU in an IRQ storm, leaving
it unable to make any progress in the polling loop, leading to the IRQ
never being acked.
Fix this by properly acknowledging the channel 0 IRQ in the handler.
As the IRQ status bit can no longer be used to poll for the channel
completion, switch over to using the SDMA_H_STATSTOP register for this
purpose, where bit 0 is cleared by the hardware when the channel is done.
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2016-07-07 09:35:51 +00:00
|
|
|
return ret;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
|
|
|
|
u32 address)
|
|
|
|
{
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_buffer_descriptor *bd0 = sdma->bd0;
|
2010-09-30 13:56:34 +00:00
|
|
|
void *buf_virt;
|
|
|
|
dma_addr_t buf_phys;
|
|
|
|
int ret;
|
2012-05-11 07:14:27 +00:00
|
|
|
unsigned long flags;
|
2011-08-25 09:03:35 +00:00
|
|
|
|
2019-01-11 14:29:49 +00:00
|
|
|
buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL);
|
2011-08-25 09:03:35 +00:00
|
|
|
if (!buf_virt) {
|
2012-05-11 07:14:27 +00:00
|
|
|
return -ENOMEM;
|
2011-08-25 09:03:35 +00:00
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
spin_lock_irqsave(&sdma->channel_0_lock, flags);
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
bd0->mode.command = C0_SETPM;
|
2019-06-21 08:23:06 +00:00
|
|
|
bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
|
2010-09-30 13:56:34 +00:00
|
|
|
bd0->mode.count = size / 2;
|
|
|
|
bd0->buffer_addr = buf_phys;
|
|
|
|
bd0->ext_buffer_addr = address;
|
|
|
|
|
|
|
|
memcpy(buf_virt, buf, size);
|
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
ret = sdma_run_channel0(sdma);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2019-01-11 14:29:49 +00:00
|
|
|
dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);
|
2011-08-25 09:03:35 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
2012-01-13 03:10:01 +00:00
|
|
|
unsigned long val;
|
2010-09-30 13:56:34 +00:00
|
|
|
u32 chnenbl = chnenbl_ofs(sdma, event);
|
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
val = readl_relaxed(sdma->regs + chnenbl);
|
2012-01-13 03:10:01 +00:00
|
|
|
__set_bit(channel, &val);
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(val, sdma->regs + chnenbl);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
u32 chnenbl = chnenbl_ofs(sdma, event);
|
2012-01-13 03:10:01 +00:00
|
|
|
unsigned long val;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
val = readl_relaxed(sdma->regs + chnenbl);
|
2012-01-13 03:10:01 +00:00
|
|
|
__clear_bit(channel, &val);
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(val, sdma->regs + chnenbl);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
|
|
|
|
{
|
|
|
|
return container_of(t, struct sdma_desc, vd.tx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sdma_start_desc(struct sdma_channel *sdmac)
|
|
|
|
{
|
|
|
|
struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
|
|
|
|
struct sdma_desc *desc;
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
|
|
|
|
if (!vd) {
|
|
|
|
sdmac->desc = NULL;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
sdmac->desc = desc = to_sdma_desc(&vd->tx);
|
2019-12-16 10:53:28 +00:00
|
|
|
|
|
|
|
list_del(&vd->node);
|
2018-06-19 16:57:00 +00:00
|
|
|
|
|
|
|
sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
|
|
|
|
sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
|
|
|
|
sdma_enable_channel(sdma, sdmac->channel);
|
|
|
|
}
|
|
|
|
|
Update imx-sdma cyclic handling to report residue
I received a report this morning from one of the Novena developers that
the behaviour of the iMX6 ASoC codec driver (using imx-pcm-dma.c) was
sub-optimal under high system load.
While there are issues relating to system load remaining, upon reviewing
the ASoC imx-pcm-dma.c driver, it was noticed that it not using the
residue support, because SDMA doesn't support it. This has the effect
that SDMA has to make multiple calls into the ASoC and ALSA code, one
for each period.
Since ALSA's snd_pcm_elapsed() does not need to be called multiple times
and it is entirely sufficient to call it once to update ALSA with the
current buffer position via the pointer method, we can do better here.
We can also avoid stopping the DMA entirely, just like real cyclic DMA
implementations behave. While this means that we replay some old samples,
this is a nicer behaviour than having audio stop and restart.
The changes to achieve this are relatively minor - imx-sdma.c can track
where the DMA is to the nearest descriptor boundary - it does this
already when deciding how many callbacks to issue. In doing this,
buf_tail always points at the descriptor which will complete next.
The residue is defined by the bytes remaining to the end of the buffer,
when the buffer is viewed as a single block of memory [start...end].
So, when we start out, there's a full buffer worth of residue, and this
counts down as we approach the end of the buffer, eventually becoming
zero at the end, before returning to the full buffer worth when we
wrap back to the start.
Moving the walking of the descriptors into the interrupt handler means
that we can update the BD_DONE flag at interrupt time, thus avoiding
a delayed tasklet stopping the cyclic DMA.
This means that the residue can be calculated from (total descriptors -
buf_tail) * descriptor size. This is what the change below does. We
update imx-pcm-dma.c to remove the NO_RESIDUE flag since we now provide
the residue.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Tested-by: Shawn Guo <shawn.guo@linaro.org>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2014-06-25 12:00:33 +00:00
|
|
|
static void sdma_update_channel_loop(struct sdma_channel *sdmac)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
struct sdma_buffer_descriptor *bd;
|
2016-08-08 12:38:26 +00:00
|
|
|
int error = 0;
|
|
|
|
enum dma_status old_status = sdmac->status;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* loop mode. Iterate over descriptors, re-setup them and
|
|
|
|
* call callback function.
|
|
|
|
*/
|
2018-06-19 16:57:00 +00:00
|
|
|
while (sdmac->desc) {
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_desc *desc = sdmac->desc;
|
|
|
|
|
|
|
|
bd = &desc->bd[desc->buf_tail];
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (bd->mode.status & BD_DONE)
|
|
|
|
break;
|
|
|
|
|
2016-08-08 12:38:26 +00:00
|
|
|
if (bd->mode.status & BD_RROR) {
|
|
|
|
bd->mode.status &= ~BD_RROR;
|
2010-09-30 13:56:34 +00:00
|
|
|
sdmac->status = DMA_ERROR;
|
2016-08-08 12:38:26 +00:00
|
|
|
error = -EIO;
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2016-08-08 12:38:26 +00:00
|
|
|
/*
|
|
|
|
* We use bd->mode.count to calculate the residue, since contains
|
|
|
|
* the number of bytes present in the current buffer descriptor.
|
|
|
|
*/
|
|
|
|
|
2018-06-19 16:56:59 +00:00
|
|
|
desc->chn_real_count = bd->mode.count;
|
2010-09-30 13:56:34 +00:00
|
|
|
bd->mode.status |= BD_DONE;
|
2018-06-19 16:56:59 +00:00
|
|
|
bd->mode.count = desc->period_len;
|
|
|
|
desc->buf_ptail = desc->buf_tail;
|
|
|
|
desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;
|
2016-08-08 12:38:25 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The callback is called from the interrupt context in order
|
|
|
|
* to reduce latency and to avoid the risk of altering the
|
|
|
|
* SDMA transaction status by the time the client tasklet is
|
|
|
|
* executed.
|
|
|
|
*/
|
2018-06-19 16:57:00 +00:00
|
|
|
spin_unlock(&sdmac->vc.lock);
|
|
|
|
dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
|
|
|
|
spin_lock(&sdmac->vc.lock);
|
2016-08-08 12:38:25 +00:00
|
|
|
|
2016-08-08 12:38:26 +00:00
|
|
|
if (error)
|
|
|
|
sdmac->status = old_status;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2016-08-08 12:38:25 +00:00
|
|
|
struct sdma_channel *sdmac = (struct sdma_channel *) data;
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_buffer_descriptor *bd;
|
|
|
|
int i, error = 0;
|
|
|
|
|
2018-06-19 16:56:59 +00:00
|
|
|
sdmac->desc->chn_real_count = 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
/*
|
|
|
|
* non loop mode. Iterate over all descriptors, collect
|
|
|
|
* errors and call callback function
|
|
|
|
*/
|
2018-06-19 16:56:59 +00:00
|
|
|
for (i = 0; i < sdmac->desc->num_bd; i++) {
|
|
|
|
bd = &sdmac->desc->bd[i];
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
if (bd->mode.status & (BD_DONE | BD_RROR))
|
|
|
|
error = -EIO;
|
2018-06-19 16:56:59 +00:00
|
|
|
sdmac->desc->chn_real_count += bd->mode.count;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (error)
|
|
|
|
sdmac->status = DMA_ERROR;
|
|
|
|
else
|
2013-10-16 08:37:06 +00:00
|
|
|
sdmac->status = DMA_COMPLETE;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static irqreturn_t sdma_int_handler(int irq, void *dev_id)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = dev_id;
|
2012-01-13 03:10:01 +00:00
|
|
|
unsigned long stat;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
|
|
|
|
writel_relaxed(stat, sdma->regs + SDMA_H_INTR);
|
dmaengine: imx-sdma: ack channel 0 IRQ in the interrupt handler
Currently the handler ignores the channel 0 interrupt and thus doesn't ack
it properly. This is done in order to allow sdma_run_channel0() to poll
on the irq status bit, as this function may be called in atomic context,
but needs to know when the channel has finished.
This works mostly, as the polling happens under a spinlock, disabling IRQs
on the local CPU, leaving only a very slight race window for a spurious
IRQ to happen if the handler is executed on another CPU in an SMP system.
Still this is clearly suboptimal.
This behavior turns into a real problem on an RT system, where the spinlock
doesn't disable IRQs on the local CPU. Not acking the IRQ in the handler
in such a setup is very likely to drown the CPU in an IRQ storm, leaving
it unable to make any progress in the polling loop, leading to the IRQ
never being acked.
Fix this by properly acknowledging the channel 0 IRQ in the handler.
As the IRQ status bit can no longer be used to poll for the channel
completion, switch over to using the SDMA_H_STATSTOP register for this
purpose, where bit 0 is cleared by the hardware when the channel is done.
Signed-off-by: Michael Olbrich <m.olbrich@pengutronix.de>
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2016-07-07 09:35:51 +00:00
|
|
|
/* channel 0 is special and not handled here, see run_channel0() */
|
|
|
|
stat &= ~1;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
while (stat) {
|
|
|
|
int channel = fls(stat) - 1;
|
|
|
|
struct sdma_channel *sdmac = &sdma->channel[channel];
|
2018-06-19 16:57:00 +00:00
|
|
|
struct sdma_desc *desc;
|
|
|
|
|
|
|
|
spin_lock(&sdmac->vc.lock);
|
|
|
|
desc = sdmac->desc;
|
|
|
|
if (desc) {
|
|
|
|
if (sdmac->flags & IMX_DMA_SG_LOOP) {
|
|
|
|
sdma_update_channel_loop(sdmac);
|
|
|
|
} else {
|
|
|
|
mxc_sdma_handle_channel_normal(sdmac);
|
|
|
|
vchan_cookie_complete(&desc->vd);
|
|
|
|
sdma_start_desc(sdmac);
|
|
|
|
}
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
spin_unlock(&sdmac->vc.lock);
|
2012-01-13 03:10:01 +00:00
|
|
|
__clear_bit(channel, &stat);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* sets the pc of SDMA script according to the peripheral type
|
|
|
|
*/
|
|
|
|
static void sdma_get_pc(struct sdma_channel *sdmac,
|
|
|
|
enum sdma_peripheral_type peripheral_type)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int per_2_emi = 0, emi_2_per = 0;
|
|
|
|
/*
|
|
|
|
* These are needed once we start to support transfers between
|
|
|
|
* two peripherals or memory-to-memory transfers
|
|
|
|
*/
|
2018-07-23 17:46:11 +00:00
|
|
|
int per_2_per = 0, emi_2_emi = 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
sdmac->pc_from_device = 0;
|
|
|
|
sdmac->pc_to_device = 0;
|
2015-07-10 09:08:16 +00:00
|
|
|
sdmac->device_to_device = 0;
|
2018-07-23 17:46:11 +00:00
|
|
|
sdmac->pc_to_pc = 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
switch (peripheral_type) {
|
|
|
|
case IMX_DMATYPE_MEMORY:
|
2018-07-23 17:46:11 +00:00
|
|
|
emi_2_emi = sdma->script_addrs->ap_2_ap_addr;
|
2010-09-30 13:56:34 +00:00
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_DSP:
|
|
|
|
emi_2_per = sdma->script_addrs->bp_2_ap_addr;
|
|
|
|
per_2_emi = sdma->script_addrs->ap_2_bp_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_FIRI:
|
|
|
|
per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_UART:
|
|
|
|
per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_app_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_UART_SP:
|
|
|
|
per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_ATA:
|
|
|
|
per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_CSPI:
|
|
|
|
case IMX_DMATYPE_EXT:
|
|
|
|
case IMX_DMATYPE_SSI:
|
2014-10-24 19:37:41 +00:00
|
|
|
case IMX_DMATYPE_SAI:
|
2010-09-30 13:56:34 +00:00
|
|
|
per_2_emi = sdma->script_addrs->app_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_app_addr;
|
|
|
|
break;
|
2013-11-13 14:55:25 +00:00
|
|
|
case IMX_DMATYPE_SSI_DUAL:
|
|
|
|
per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;
|
|
|
|
break;
|
2010-09-30 13:56:34 +00:00
|
|
|
case IMX_DMATYPE_SSI_SP:
|
|
|
|
case IMX_DMATYPE_MMC:
|
|
|
|
case IMX_DMATYPE_SDHC:
|
|
|
|
case IMX_DMATYPE_CSPI_SP:
|
|
|
|
case IMX_DMATYPE_ESAI:
|
|
|
|
case IMX_DMATYPE_MSHC_SP:
|
|
|
|
per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_ASRC:
|
|
|
|
per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
|
|
|
|
per_2_per = sdma->script_addrs->per_2_per_addr;
|
|
|
|
break;
|
2014-06-16 03:31:05 +00:00
|
|
|
case IMX_DMATYPE_ASRC_SP:
|
|
|
|
per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
|
|
|
|
per_2_per = sdma->script_addrs->per_2_per_addr;
|
|
|
|
break;
|
2010-09-30 13:56:34 +00:00
|
|
|
case IMX_DMATYPE_MSHC:
|
|
|
|
per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_CCM:
|
|
|
|
per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_SPDIF:
|
|
|
|
per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
|
|
|
|
emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_IPU_MEMORY:
|
|
|
|
emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
sdmac->pc_from_device = per_2_emi;
|
|
|
|
sdmac->pc_to_device = emi_2_per;
|
2015-07-10 09:08:16 +00:00
|
|
|
sdmac->device_to_device = per_2_per;
|
2018-07-23 17:46:11 +00:00
|
|
|
sdmac->pc_to_pc = emi_2_emi;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_load_context(struct sdma_channel *sdmac)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
int load_address;
|
|
|
|
struct sdma_context_data *context = sdma->context;
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_buffer_descriptor *bd0 = sdma->bd0;
|
2010-09-30 13:56:34 +00:00
|
|
|
int ret;
|
2012-05-11 07:14:27 +00:00
|
|
|
unsigned long flags;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2019-01-08 12:00:16 +00:00
|
|
|
if (sdmac->context_loaded)
|
|
|
|
return 0;
|
|
|
|
|
2015-07-10 09:08:16 +00:00
|
|
|
if (sdmac->direction == DMA_DEV_TO_MEM)
|
2010-09-30 13:56:34 +00:00
|
|
|
load_address = sdmac->pc_from_device;
|
2015-07-10 09:08:16 +00:00
|
|
|
else if (sdmac->direction == DMA_DEV_TO_DEV)
|
|
|
|
load_address = sdmac->device_to_device;
|
2018-07-23 17:46:11 +00:00
|
|
|
else if (sdmac->direction == DMA_MEM_TO_MEM)
|
|
|
|
load_address = sdmac->pc_to_pc;
|
2015-07-10 09:08:16 +00:00
|
|
|
else
|
2010-09-30 13:56:34 +00:00
|
|
|
load_address = sdmac->pc_to_device;
|
|
|
|
|
|
|
|
if (load_address < 0)
|
|
|
|
return load_address;
|
|
|
|
|
|
|
|
dev_dbg(sdma->dev, "load_address = %d\n", load_address);
|
2012-01-13 03:10:01 +00:00
|
|
|
dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);
|
2010-09-30 13:56:34 +00:00
|
|
|
dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);
|
|
|
|
dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);
|
2012-01-13 03:10:01 +00:00
|
|
|
dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);
|
|
|
|
dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
spin_lock_irqsave(&sdma->channel_0_lock, flags);
|
2011-08-25 09:03:35 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
memset(context, 0, sizeof(*context));
|
|
|
|
context->channel_state.pc = load_address;
|
|
|
|
|
|
|
|
/* Send by context the event mask,base address for peripheral
|
|
|
|
* and watermark level
|
|
|
|
*/
|
2012-01-13 03:10:01 +00:00
|
|
|
context->gReg[0] = sdmac->event_mask[1];
|
|
|
|
context->gReg[1] = sdmac->event_mask[0];
|
2010-09-30 13:56:34 +00:00
|
|
|
context->gReg[2] = sdmac->per_addr;
|
|
|
|
context->gReg[6] = sdmac->shp_addr;
|
|
|
|
context->gReg[7] = sdmac->watermark_level;
|
|
|
|
|
|
|
|
bd0->mode.command = C0_SETDM;
|
2019-06-21 08:23:06 +00:00
|
|
|
bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
|
2010-09-30 13:56:34 +00:00
|
|
|
bd0->mode.count = sizeof(*context) / 4;
|
|
|
|
bd0->buffer_addr = sdma->context_phys;
|
|
|
|
bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;
|
2012-05-11 07:14:27 +00:00
|
|
|
ret = sdma_run_channel0(sdma);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
|
2011-08-25 09:03:35 +00:00
|
|
|
|
2019-01-08 12:00:16 +00:00
|
|
|
sdmac->context_loaded = true;
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-11-17 13:42:17 +00:00
|
|
|
static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
|
|
|
|
{
|
2018-06-19 16:57:00 +00:00
|
|
|
return container_of(chan, struct sdma_channel, vc.chan);
|
2014-11-17 13:42:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_disable_channel(struct dma_chan *chan)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2014-11-17 13:42:17 +00:00
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2010-09-30 13:56:34 +00:00
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
|
2012-01-13 03:10:01 +00:00
|
|
|
writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);
|
2010-09-30 13:56:34 +00:00
|
|
|
sdmac->status = DMA_ERROR;
|
2014-11-17 13:42:17 +00:00
|
|
|
|
|
|
|
return 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
2018-11-06 03:40:33 +00:00
|
|
|
static void sdma_channel_terminate_work(struct work_struct *work)
|
2017-03-17 06:12:09 +00:00
|
|
|
{
|
2018-11-06 03:40:33 +00:00
|
|
|
struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
|
|
|
|
terminate_worker);
|
2018-06-19 16:57:00 +00:00
|
|
|
unsigned long flags;
|
|
|
|
LIST_HEAD(head);
|
|
|
|
|
2017-03-17 06:12:09 +00:00
|
|
|
/*
|
|
|
|
* According to NXP R&D team a delay of one BD SDMA cost time
|
|
|
|
* (maximum is 1ms) should be added after disable of the channel
|
|
|
|
* bit, to ensure SDMA core has really been stopped after SDMA
|
|
|
|
* clients call .device_terminate_all.
|
|
|
|
*/
|
2018-11-06 03:40:33 +00:00
|
|
|
usleep_range(1000, 2000);
|
|
|
|
|
|
|
|
spin_lock_irqsave(&sdmac->vc.lock, flags);
|
|
|
|
vchan_get_all_descriptors(&sdmac->vc, &head);
|
|
|
|
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
|
|
|
|
vchan_dma_desc_free_list(&sdmac->vc, &head);
|
2019-01-08 12:00:16 +00:00
|
|
|
sdmac->context_loaded = false;
|
2018-11-06 03:40:33 +00:00
|
|
|
}
|
|
|
|
|
2019-12-16 10:53:26 +00:00
|
|
|
static int sdma_terminate_all(struct dma_chan *chan)
|
2018-11-06 03:40:33 +00:00
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2019-12-16 10:53:28 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&sdmac->vc.lock, flags);
|
2018-11-06 03:40:33 +00:00
|
|
|
|
|
|
|
sdma_disable_channel(chan);
|
|
|
|
|
2019-12-16 10:53:28 +00:00
|
|
|
if (sdmac->desc) {
|
|
|
|
vchan_terminate_vdesc(&sdmac->desc->vd);
|
|
|
|
sdmac->desc = NULL;
|
2018-11-06 03:40:33 +00:00
|
|
|
schedule_work(&sdmac->terminate_worker);
|
2019-12-16 10:53:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
|
2017-03-17 06:12:09 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-11-06 03:40:33 +00:00
|
|
|
static void sdma_channel_synchronize(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
|
|
|
|
vchan_synchronize(&sdmac->vc);
|
|
|
|
|
|
|
|
flush_work(&sdmac->terminate_worker);
|
|
|
|
}
|
|
|
|
|
2015-07-10 09:08:16 +00:00
|
|
|
static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
|
|
|
|
int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
|
|
|
|
int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
|
|
|
|
|
|
|
|
set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
|
|
|
|
set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
|
|
|
|
|
|
|
|
if (sdmac->event_id0 > 31)
|
|
|
|
sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
|
|
|
|
|
|
|
|
if (sdmac->event_id1 > 31)
|
|
|
|
sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If LWML(src_maxburst) > HWML(dst_maxburst), we need
|
|
|
|
* swap LWML and HWML of INFO(A.3.2.5.1), also need swap
|
|
|
|
* r0(event_mask[1]) and r1(event_mask[0]).
|
|
|
|
*/
|
|
|
|
if (lwml > hwml) {
|
|
|
|
sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
|
|
|
|
SDMA_WATERMARK_LEVEL_HWML);
|
|
|
|
sdmac->watermark_level |= hwml;
|
|
|
|
sdmac->watermark_level |= lwml << 16;
|
|
|
|
swap(sdmac->event_mask[0], sdmac->event_mask[1]);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sdmac->per_address2 >= sdma->spba_start_addr &&
|
|
|
|
sdmac->per_address2 <= sdma->spba_end_addr)
|
|
|
|
sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
|
|
|
|
|
|
|
|
if (sdmac->per_address >= sdma->spba_start_addr &&
|
|
|
|
sdmac->per_address <= sdma->spba_end_addr)
|
|
|
|
sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
|
|
|
|
|
|
|
|
sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
|
|
|
|
}
|
|
|
|
|
2014-11-17 13:42:17 +00:00
|
|
|
static int sdma_config_channel(struct dma_chan *chan)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2014-11-17 13:42:17 +00:00
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2010-09-30 13:56:34 +00:00
|
|
|
int ret;
|
|
|
|
|
2014-11-17 13:42:17 +00:00
|
|
|
sdma_disable_channel(chan);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-01-13 03:10:01 +00:00
|
|
|
sdmac->event_mask[0] = 0;
|
|
|
|
sdmac->event_mask[1] = 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
sdmac->shp_addr = 0;
|
|
|
|
sdmac->per_addr = 0;
|
|
|
|
|
|
|
|
switch (sdmac->peripheral_type) {
|
|
|
|
case IMX_DMATYPE_DSP:
|
|
|
|
sdma_config_ownership(sdmac, false, true, true);
|
|
|
|
break;
|
|
|
|
case IMX_DMATYPE_MEMORY:
|
|
|
|
sdma_config_ownership(sdmac, false, true, false);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
sdma_config_ownership(sdmac, true, true, false);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
sdma_get_pc(sdmac, sdmac->peripheral_type);
|
|
|
|
|
|
|
|
if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
|
|
|
|
(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
|
|
|
|
/* Handle multiple event channels differently */
|
|
|
|
if (sdmac->event_id1) {
|
2015-07-10 09:08:16 +00:00
|
|
|
if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
|
|
|
|
sdmac->peripheral_type == IMX_DMATYPE_ASRC)
|
|
|
|
sdma_set_watermarklevel_for_p2p(sdmac);
|
|
|
|
} else
|
2012-01-13 03:10:01 +00:00
|
|
|
__set_bit(sdmac->event_id0, sdmac->event_mask);
|
2015-07-10 09:08:16 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/* Address */
|
|
|
|
sdmac->shp_addr = sdmac->per_address;
|
2015-07-10 09:08:16 +00:00
|
|
|
sdmac->per_addr = sdmac->per_address2;
|
2010-09-30 13:56:34 +00:00
|
|
|
} else {
|
|
|
|
sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = sdma_load_context(sdmac);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_set_channel_priority(struct sdma_channel *sdmac,
|
|
|
|
unsigned int priority)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
|
|
|
|
if (priority < MXC_SDMA_MIN_PRIORITY
|
|
|
|
|| priority > MXC_SDMA_MAX_PRIORITY) {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
static int sdma_request_channel0(struct sdma_engine *sdma)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
int ret = -EBUSY;
|
|
|
|
|
dmaengine updates for v5.1-rc1
- dmatest updates for modularizing common struct and code
- remove SG support for VDMA xilinx IP and updates to driver
- Update to dw driver to support Intel iDMA controllers
multi-block support
- tegra updates for proper reporting of residue
- Add Snow Ridge ioatdma device id and support for IOATDMA v3.4
- struct_size() usage and useless LIST_HEAD cleanups in subsystem.
- qDMA controller driver for Layerscape SoCs
- stm32-dma PM Runtime support
- And usual updates to imx-sdma, sprd, Documentation, fsl-edma,
bcm2835, qcom_hidma etc
-----BEGIN PGP SIGNATURE-----
iQIcBAABAgAGBQJciVhkAAoJEHwUBw8lI4NHdVoQANDfaY1eruozJobfWX0w1b8U
0eSCN4M/gRzL/K1nOTWTV9dZFGycGokPBn25lryX6d6VIL+yO7Ptpwls/w0inn0e
RgVtESQodLZyxcD4iWACVfTGxqxe74/bXCRAq0OCrjt5oX+5KdsBsTrhHvB8dQin
JWT7Mq6tii5wZHZHl9b4Ds/crxM9+pIHmlzbu5MQiPDL37X9HX4KUoLQCKrVGgZt
3FjlKEiSS6CnBWP1cs/aHgANr/PjOIwL8SD1t5NPS3i7/2k9z1u16TmmI8SbcyHf
y9hVy1QPlxC5V85EPYK7JW7JLILotkMToxlX/QhfaFN0PWQq04rp6PCLvObKJuB2
36QJTwSXBM2/f9bWwkddPyo9Szb3L30K80Vx8zlxzgoXYWtFFB2BXAV57M/I48j2
gMnxEMZpHD3IupeqlykCmssClVVmCRT8qUZHLNHTdDNu48rLuNlZestFGyBS2Ma2
D4UcJPiA/IxENy1rz54XoCajL/BIJOsFXXVRikjj3vfbV0Uir0uB9puzwfemMtKz
tCrJbKwnnDtN30vBZtU7hVu/t4lFYYnl2c945+SpeQC69dysz3IFQGem0kxVHKnH
INdQH4Od7nQbVQer1EXg4+h3n/rUbcYx/0iLE6JAYyOT80wE+KSlrQ5etqyJ3/qh
1BxvU49PkUUPzw7aUGgS
=Ykvn
-----END PGP SIGNATURE-----
Merge tag 'dmaengine-5.1-rc1' of git://git.infradead.org/users/vkoul/slave-dma
Pull dmaengine updates from Vinod Koul:
- dmatest updates for modularizing common struct and code
- remove SG support for VDMA xilinx IP and updates to driver
- Update to dw driver to support Intel iDMA controllers multi-block
support
- tegra updates for proper reporting of residue
- Add Snow Ridge ioatdma device id and support for IOATDMA v3.4
- struct_size() usage and useless LIST_HEAD cleanups in subsystem.
- qDMA controller driver for Layerscape SoCs
- stm32-dma PM Runtime support
- And usual updates to imx-sdma, sprd, Documentation, fsl-edma,
bcm2835, qcom_hidma etc
* tag 'dmaengine-5.1-rc1' of git://git.infradead.org/users/vkoul/slave-dma: (81 commits)
dmaengine: imx-sdma: fix consistent dma test failures
dmaengine: imx-sdma: add a test for imx8mq multi sdma devices
dmaengine: imx-sdma: add clock ratio 1:1 check
dmaengine: dmatest: move test data alloc & free into functions
dmaengine: dmatest: add short-hand `buf_size` var in dmatest_func()
dmaengine: dmatest: wrap src & dst data into a struct
dmaengine: ioatdma: support latency tolerance report (LTR) for v3.4
dmaengine: ioatdma: add descriptor pre-fetch support for v3.4
dmaengine: ioatdma: disable DCA enabling on IOATDMA v3.4
dmaengine: ioatdma: Add Snow Ridge ioatdma device id
dmaengine: sprd: Change channel id to slave id for DMA cell specifier
dt-bindings: dmaengine: sprd: Change channel id to slave id for DMA cell specifier
dmaengine: mv_xor: Use correct device for DMA API
Documentation :dmaengine: clarify DMA desc. pointer after submission
Documentation: dmaengine: fix dmatest.rst warning
dmaengine: k3dma: Add support for dma-channel-mask
dmaengine: k3dma: Delete axi_config
dmaengine: k3dma: Upgrade k3dma driver to support hisi_asp_dma hardware
Documentation: bindings: dma: Add binding for dma-channel-mask
Documentation: bindings: k3dma: Extend the k3dma driver binding to support hisi-asp
...
2019-03-14 16:11:54 +00:00
|
|
|
sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys,
|
2018-06-19 16:57:00 +00:00
|
|
|
GFP_NOWAIT);
|
|
|
|
if (!sdma->bd0) {
|
2010-09-30 13:56:34 +00:00
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
|
|
|
|
sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);
|
2010-09-30 13:56:34 +00:00
|
|
|
return 0;
|
|
|
|
out:
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
|
|
|
|
static int sdma_alloc_bd(struct sdma_desc *desc)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2018-11-06 03:40:28 +00:00
|
|
|
u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
|
2018-06-19 16:57:00 +00:00
|
|
|
int ret = 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
dmaengine updates for v5.1-rc1
- dmatest updates for modularizing common struct and code
- remove SG support for VDMA xilinx IP and updates to driver
- Update to dw driver to support Intel iDMA controllers
multi-block support
- tegra updates for proper reporting of residue
- Add Snow Ridge ioatdma device id and support for IOATDMA v3.4
- struct_size() usage and useless LIST_HEAD cleanups in subsystem.
- qDMA controller driver for Layerscape SoCs
- stm32-dma PM Runtime support
- And usual updates to imx-sdma, sprd, Documentation, fsl-edma,
bcm2835, qcom_hidma etc
-----BEGIN PGP SIGNATURE-----
iQIcBAABAgAGBQJciVhkAAoJEHwUBw8lI4NHdVoQANDfaY1eruozJobfWX0w1b8U
0eSCN4M/gRzL/K1nOTWTV9dZFGycGokPBn25lryX6d6VIL+yO7Ptpwls/w0inn0e
RgVtESQodLZyxcD4iWACVfTGxqxe74/bXCRAq0OCrjt5oX+5KdsBsTrhHvB8dQin
JWT7Mq6tii5wZHZHl9b4Ds/crxM9+pIHmlzbu5MQiPDL37X9HX4KUoLQCKrVGgZt
3FjlKEiSS6CnBWP1cs/aHgANr/PjOIwL8SD1t5NPS3i7/2k9z1u16TmmI8SbcyHf
y9hVy1QPlxC5V85EPYK7JW7JLILotkMToxlX/QhfaFN0PWQq04rp6PCLvObKJuB2
36QJTwSXBM2/f9bWwkddPyo9Szb3L30K80Vx8zlxzgoXYWtFFB2BXAV57M/I48j2
gMnxEMZpHD3IupeqlykCmssClVVmCRT8qUZHLNHTdDNu48rLuNlZestFGyBS2Ma2
D4UcJPiA/IxENy1rz54XoCajL/BIJOsFXXVRikjj3vfbV0Uir0uB9puzwfemMtKz
tCrJbKwnnDtN30vBZtU7hVu/t4lFYYnl2c945+SpeQC69dysz3IFQGem0kxVHKnH
INdQH4Od7nQbVQer1EXg4+h3n/rUbcYx/0iLE6JAYyOT80wE+KSlrQ5etqyJ3/qh
1BxvU49PkUUPzw7aUGgS
=Ykvn
-----END PGP SIGNATURE-----
Merge tag 'dmaengine-5.1-rc1' of git://git.infradead.org/users/vkoul/slave-dma
Pull dmaengine updates from Vinod Koul:
- dmatest updates for modularizing common struct and code
- remove SG support for VDMA xilinx IP and updates to driver
- Update to dw driver to support Intel iDMA controllers multi-block
support
- tegra updates for proper reporting of residue
- Add Snow Ridge ioatdma device id and support for IOATDMA v3.4
- struct_size() usage and useless LIST_HEAD cleanups in subsystem.
- qDMA controller driver for Layerscape SoCs
- stm32-dma PM Runtime support
- And usual updates to imx-sdma, sprd, Documentation, fsl-edma,
bcm2835, qcom_hidma etc
* tag 'dmaengine-5.1-rc1' of git://git.infradead.org/users/vkoul/slave-dma: (81 commits)
dmaengine: imx-sdma: fix consistent dma test failures
dmaengine: imx-sdma: add a test for imx8mq multi sdma devices
dmaengine: imx-sdma: add clock ratio 1:1 check
dmaengine: dmatest: move test data alloc & free into functions
dmaengine: dmatest: add short-hand `buf_size` var in dmatest_func()
dmaengine: dmatest: wrap src & dst data into a struct
dmaengine: ioatdma: support latency tolerance report (LTR) for v3.4
dmaengine: ioatdma: add descriptor pre-fetch support for v3.4
dmaengine: ioatdma: disable DCA enabling on IOATDMA v3.4
dmaengine: ioatdma: Add Snow Ridge ioatdma device id
dmaengine: sprd: Change channel id to slave id for DMA cell specifier
dt-bindings: dmaengine: sprd: Change channel id to slave id for DMA cell specifier
dmaengine: mv_xor: Use correct device for DMA API
Documentation :dmaengine: clarify DMA desc. pointer after submission
Documentation: dmaengine: fix dmatest.rst warning
dmaengine: k3dma: Add support for dma-channel-mask
dmaengine: k3dma: Delete axi_config
dmaengine: k3dma: Upgrade k3dma driver to support hisi_asp_dma hardware
Documentation: bindings: dma: Add binding for dma-channel-mask
Documentation: bindings: k3dma: Extend the k3dma driver binding to support hisi-asp
...
2019-03-14 16:11:54 +00:00
|
|
|
desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,
|
2019-01-11 14:29:49 +00:00
|
|
|
&desc->bd_phys, GFP_NOWAIT);
|
2018-06-19 16:57:00 +00:00
|
|
|
if (!desc->bd) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
return ret;
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
static void sdma_free_bd(struct sdma_desc *desc)
|
|
|
|
{
|
2018-11-06 03:40:28 +00:00
|
|
|
u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);
|
|
|
|
|
2019-01-11 14:29:49 +00:00
|
|
|
dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,
|
|
|
|
desc->bd_phys);
|
2018-06-19 16:57:00 +00:00
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
static void sdma_desc_free(struct virt_dma_desc *vd)
|
|
|
|
{
|
|
|
|
struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
|
|
|
|
|
|
|
|
sdma_free_bd(desc);
|
|
|
|
kfree(desc);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int sdma_alloc_chan_resources(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
struct imx_dma_data *data = chan->private;
|
2018-07-23 17:46:11 +00:00
|
|
|
struct imx_dma_data mem_data;
|
2010-09-30 13:56:34 +00:00
|
|
|
int prio, ret;
|
|
|
|
|
2018-07-23 17:46:11 +00:00
|
|
|
/*
|
|
|
|
* MEMCPY may never setup chan->private by filter function such as
|
|
|
|
* dmatest, thus create 'struct imx_dma_data mem_data' for this case.
|
|
|
|
* Please note in any other slave case, you have to setup chan->private
|
|
|
|
* with 'struct imx_dma_data' in your own filter function if you want to
|
|
|
|
* request dma channel by dma_request_channel() rather than
|
|
|
|
* dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
|
|
|
|
* to warn you to correct your filter function.
|
|
|
|
*/
|
|
|
|
if (!data) {
|
|
|
|
dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");
|
|
|
|
mem_data.priority = 2;
|
|
|
|
mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
|
|
|
|
mem_data.dma_request = 0;
|
|
|
|
mem_data.dma_request2 = 0;
|
|
|
|
data = &mem_data;
|
|
|
|
|
|
|
|
sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
switch (data->priority) {
|
|
|
|
case DMA_PRIO_HIGH:
|
|
|
|
prio = 3;
|
|
|
|
break;
|
|
|
|
case DMA_PRIO_MEDIUM:
|
|
|
|
prio = 2;
|
|
|
|
break;
|
|
|
|
case DMA_PRIO_LOW:
|
|
|
|
default:
|
|
|
|
prio = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
sdmac->peripheral_type = data->peripheral_type;
|
|
|
|
sdmac->event_id0 = data->dma_request;
|
2015-07-10 09:08:16 +00:00
|
|
|
sdmac->event_id1 = data->dma_request2;
|
2012-01-13 03:09:59 +00:00
|
|
|
|
2015-07-30 00:03:49 +00:00
|
|
|
ret = clk_enable(sdmac->sdma->clk_ipg);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
ret = clk_enable(sdmac->sdma->clk_ahb);
|
|
|
|
if (ret)
|
|
|
|
goto disable_clk_ipg;
|
2012-01-13 03:09:59 +00:00
|
|
|
|
2012-01-13 03:09:58 +00:00
|
|
|
ret = sdma_set_channel_priority(sdmac, prio);
|
2010-09-30 13:56:34 +00:00
|
|
|
if (ret)
|
2015-07-30 00:03:49 +00:00
|
|
|
goto disable_clk_ahb;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
return 0;
|
2015-07-30 00:03:49 +00:00
|
|
|
|
|
|
|
disable_clk_ahb:
|
|
|
|
clk_disable(sdmac->sdma->clk_ahb);
|
|
|
|
disable_clk_ipg:
|
|
|
|
clk_disable(sdmac->sdma->clk_ipg);
|
|
|
|
return ret;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void sdma_free_chan_resources(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
|
2019-12-16 10:53:26 +00:00
|
|
|
sdma_terminate_all(chan);
|
2018-11-06 03:40:33 +00:00
|
|
|
|
|
|
|
sdma_channel_synchronize(chan);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2020-02-25 08:23:20 +00:00
|
|
|
if (sdmac->event_id0 >= 0)
|
2010-09-30 13:56:34 +00:00
|
|
|
sdma_event_disable(sdmac, sdmac->event_id0);
|
|
|
|
if (sdmac->event_id1)
|
|
|
|
sdma_event_disable(sdmac, sdmac->event_id1);
|
|
|
|
|
|
|
|
sdmac->event_id0 = 0;
|
|
|
|
sdmac->event_id1 = 0;
|
2020-01-29 13:40:06 +00:00
|
|
|
sdmac->context_loaded = false;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
sdma_set_channel_priority(sdmac, 0);
|
|
|
|
|
2012-03-07 08:30:06 +00:00
|
|
|
clk_disable(sdma->clk_ipg);
|
|
|
|
clk_disable(sdma->clk_ahb);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:03 +00:00
|
|
|
static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
|
|
|
|
enum dma_transfer_direction direction, u32 bds)
|
|
|
|
{
|
|
|
|
struct sdma_desc *desc;
|
|
|
|
|
|
|
|
desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
|
|
|
|
if (!desc)
|
|
|
|
goto err_out;
|
|
|
|
|
|
|
|
sdmac->status = DMA_IN_PROGRESS;
|
|
|
|
sdmac->direction = direction;
|
|
|
|
sdmac->flags = 0;
|
|
|
|
|
|
|
|
desc->chn_count = 0;
|
|
|
|
desc->chn_real_count = 0;
|
|
|
|
desc->buf_tail = 0;
|
|
|
|
desc->buf_ptail = 0;
|
|
|
|
desc->sdmac = sdmac;
|
|
|
|
desc->num_bd = bds;
|
|
|
|
|
|
|
|
if (sdma_alloc_bd(desc))
|
|
|
|
goto err_desc_out;
|
|
|
|
|
2018-07-23 17:46:11 +00:00
|
|
|
/* No slave_config called in MEMCPY case, so do here */
|
|
|
|
if (direction == DMA_MEM_TO_MEM)
|
|
|
|
sdma_config_ownership(sdmac, false, true, false);
|
|
|
|
|
2018-06-19 16:57:03 +00:00
|
|
|
if (sdma_load_context(sdmac))
|
|
|
|
goto err_desc_out;
|
|
|
|
|
|
|
|
return desc;
|
|
|
|
|
|
|
|
err_desc_out:
|
|
|
|
kfree(desc);
|
|
|
|
err_out:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-07-23 17:46:11 +00:00
|
|
|
static struct dma_async_tx_descriptor *sdma_prep_memcpy(
|
|
|
|
struct dma_chan *chan, dma_addr_t dma_dst,
|
|
|
|
dma_addr_t dma_src, size_t len, unsigned long flags)
|
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int channel = sdmac->channel;
|
|
|
|
size_t count;
|
|
|
|
int i = 0, param;
|
|
|
|
struct sdma_buffer_descriptor *bd;
|
|
|
|
struct sdma_desc *desc;
|
|
|
|
|
|
|
|
if (!chan || !len)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n",
|
|
|
|
&dma_src, &dma_dst, len, channel);
|
|
|
|
|
|
|
|
desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
|
|
|
|
len / SDMA_BD_MAX_CNT + 1);
|
|
|
|
if (!desc)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
do {
|
|
|
|
count = min_t(size_t, len, SDMA_BD_MAX_CNT);
|
|
|
|
bd = &desc->bd[i];
|
|
|
|
bd->buffer_addr = dma_src;
|
|
|
|
bd->ext_buffer_addr = dma_dst;
|
|
|
|
bd->mode.count = count;
|
|
|
|
desc->chn_count += count;
|
|
|
|
bd->mode.command = 0;
|
|
|
|
|
|
|
|
dma_src += count;
|
|
|
|
dma_dst += count;
|
|
|
|
len -= count;
|
|
|
|
i++;
|
|
|
|
|
|
|
|
param = BD_DONE | BD_EXTD | BD_CONT;
|
|
|
|
/* last bd */
|
|
|
|
if (!len) {
|
|
|
|
param |= BD_INTR;
|
|
|
|
param |= BD_LAST;
|
|
|
|
param &= ~BD_CONT;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n",
|
|
|
|
i, count, bd->buffer_addr,
|
|
|
|
param & BD_WRAP ? "wrap" : "",
|
|
|
|
param & BD_INTR ? " intr" : "");
|
|
|
|
|
|
|
|
bd->mode.status = param;
|
|
|
|
} while (len);
|
|
|
|
|
|
|
|
return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
|
|
|
|
}
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
|
|
|
|
struct dma_chan *chan, struct scatterlist *sgl,
|
2011-10-13 17:04:23 +00:00
|
|
|
unsigned int sg_len, enum dma_transfer_direction direction,
|
2012-03-08 20:35:13 +00:00
|
|
|
unsigned long flags, void *context)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
2018-07-02 13:12:51 +00:00
|
|
|
int i, count;
|
2011-01-31 09:56:58 +00:00
|
|
|
int channel = sdmac->channel;
|
2010-09-30 13:56:34 +00:00
|
|
|
struct scatterlist *sg;
|
2018-06-19 16:57:00 +00:00
|
|
|
struct sdma_desc *desc;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
sdma_config_write(chan, &sdmac->slave_config, direction);
|
|
|
|
|
2018-06-19 16:57:03 +00:00
|
|
|
desc = sdma_transfer_init(sdmac, direction, sg_len);
|
2018-06-19 16:57:00 +00:00
|
|
|
if (!desc)
|
|
|
|
goto err_out;
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n",
|
|
|
|
sg_len, channel);
|
|
|
|
|
|
|
|
for_each_sg(sgl, sg, sg_len, i) {
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_buffer_descriptor *bd = &desc->bd[i];
|
2010-09-30 13:56:34 +00:00
|
|
|
int param;
|
|
|
|
|
2010-11-22 17:35:18 +00:00
|
|
|
bd->buffer_addr = sg->dma_address;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-04-25 18:50:52 +00:00
|
|
|
count = sg_dma_len(sg);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-07-23 17:46:10 +00:00
|
|
|
if (count > SDMA_BD_MAX_CNT) {
|
2010-09-30 13:56:34 +00:00
|
|
|
dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n",
|
2018-07-23 17:46:10 +00:00
|
|
|
channel, count, SDMA_BD_MAX_CNT);
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bd->mode.count = count;
|
2018-06-19 16:56:59 +00:00
|
|
|
desc->chn_count += count;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-07-02 13:12:51 +00:00
|
|
|
if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2011-01-12 12:02:28 +00:00
|
|
|
|
|
|
|
switch (sdmac->word_size) {
|
|
|
|
case DMA_SLAVE_BUSWIDTH_4_BYTES:
|
2010-09-30 13:56:34 +00:00
|
|
|
bd->mode.command = 0;
|
2011-01-12 12:02:28 +00:00
|
|
|
if (count & 3 || sg->dma_address & 3)
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2011-01-12 12:02:28 +00:00
|
|
|
break;
|
|
|
|
case DMA_SLAVE_BUSWIDTH_2_BYTES:
|
|
|
|
bd->mode.command = 2;
|
|
|
|
if (count & 1 || sg->dma_address & 1)
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2011-01-12 12:02:28 +00:00
|
|
|
break;
|
|
|
|
case DMA_SLAVE_BUSWIDTH_1_BYTE:
|
|
|
|
bd->mode.command = 1;
|
|
|
|
break;
|
|
|
|
default:
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2011-01-12 12:02:28 +00:00
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
param = BD_DONE | BD_EXTD | BD_CONT;
|
|
|
|
|
2011-01-19 21:50:39 +00:00
|
|
|
if (i + 1 == sg_len) {
|
2010-09-30 13:56:34 +00:00
|
|
|
param |= BD_INTR;
|
2011-01-19 21:50:39 +00:00
|
|
|
param |= BD_LAST;
|
|
|
|
param &= ~BD_CONT;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2013-11-13 06:30:44 +00:00
|
|
|
dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n",
|
|
|
|
i, count, (u64)sg->dma_address,
|
2010-09-30 13:56:34 +00:00
|
|
|
param & BD_WRAP ? "wrap" : "",
|
|
|
|
param & BD_INTR ? " intr" : "");
|
|
|
|
|
|
|
|
bd->mode.status = param;
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
|
|
|
|
err_bd_out:
|
|
|
|
sdma_free_bd(desc);
|
|
|
|
kfree(desc);
|
2010-09-30 13:56:34 +00:00
|
|
|
err_out:
|
2011-01-19 21:50:36 +00:00
|
|
|
sdmac->status = DMA_ERROR;
|
2010-09-30 13:56:34 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
|
|
|
|
struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
|
2012-03-08 20:35:13 +00:00
|
|
|
size_t period_len, enum dma_transfer_direction direction,
|
2014-08-01 10:20:10 +00:00
|
|
|
unsigned long flags)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
struct sdma_engine *sdma = sdmac->sdma;
|
|
|
|
int num_periods = buf_len / period_len;
|
2011-01-31 09:56:58 +00:00
|
|
|
int channel = sdmac->channel;
|
2018-06-19 16:57:03 +00:00
|
|
|
int i = 0, buf = 0;
|
2018-06-19 16:57:00 +00:00
|
|
|
struct sdma_desc *desc;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel);
|
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
sdma_config_write(chan, &sdmac->slave_config, direction);
|
|
|
|
|
2018-06-19 16:57:03 +00:00
|
|
|
desc = sdma_transfer_init(sdmac, direction, num_periods);
|
2018-06-19 16:57:00 +00:00
|
|
|
if (!desc)
|
|
|
|
goto err_out;
|
|
|
|
|
2018-06-19 16:56:59 +00:00
|
|
|
desc->period_len = period_len;
|
2012-06-04 01:17:24 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
sdmac->flags |= IMX_DMA_SG_LOOP;
|
|
|
|
|
2018-07-23 17:46:10 +00:00
|
|
|
if (period_len > SDMA_BD_MAX_CNT) {
|
2017-05-24 06:49:06 +00:00
|
|
|
dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n",
|
2018-07-23 17:46:10 +00:00
|
|
|
channel, period_len, SDMA_BD_MAX_CNT);
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
while (buf < buf_len) {
|
2018-06-19 16:56:59 +00:00
|
|
|
struct sdma_buffer_descriptor *bd = &desc->bd[i];
|
2010-09-30 13:56:34 +00:00
|
|
|
int param;
|
|
|
|
|
|
|
|
bd->buffer_addr = dma_addr;
|
|
|
|
|
|
|
|
bd->mode.count = period_len;
|
|
|
|
|
|
|
|
if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)
|
2018-06-19 16:57:00 +00:00
|
|
|
goto err_bd_out;
|
2010-09-30 13:56:34 +00:00
|
|
|
if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
|
|
|
|
bd->mode.command = 0;
|
|
|
|
else
|
|
|
|
bd->mode.command = sdmac->word_size;
|
|
|
|
|
|
|
|
param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
|
|
|
|
if (i + 1 == num_periods)
|
|
|
|
param |= BD_WRAP;
|
|
|
|
|
2017-05-24 06:49:06 +00:00
|
|
|
dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s\n",
|
2013-11-13 06:30:44 +00:00
|
|
|
i, period_len, (u64)dma_addr,
|
2010-09-30 13:56:34 +00:00
|
|
|
param & BD_WRAP ? "wrap" : "",
|
|
|
|
param & BD_INTR ? " intr" : "");
|
|
|
|
|
|
|
|
bd->mode.status = param;
|
|
|
|
|
|
|
|
dma_addr += period_len;
|
|
|
|
buf += period_len;
|
|
|
|
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
|
|
|
|
err_bd_out:
|
|
|
|
sdma_free_bd(desc);
|
|
|
|
kfree(desc);
|
2010-09-30 13:56:34 +00:00
|
|
|
err_out:
|
|
|
|
sdmac->status = DMA_ERROR;
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
static int sdma_config_write(struct dma_chan *chan,
|
|
|
|
struct dma_slave_config *dmaengine_cfg,
|
|
|
|
enum dma_transfer_direction direction)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
if (direction == DMA_DEV_TO_MEM) {
|
2014-11-17 13:42:17 +00:00
|
|
|
sdmac->per_address = dmaengine_cfg->src_addr;
|
|
|
|
sdmac->watermark_level = dmaengine_cfg->src_maxburst *
|
|
|
|
dmaengine_cfg->src_addr_width;
|
|
|
|
sdmac->word_size = dmaengine_cfg->src_addr_width;
|
2018-10-25 14:15:28 +00:00
|
|
|
} else if (direction == DMA_DEV_TO_DEV) {
|
2015-07-10 09:08:16 +00:00
|
|
|
sdmac->per_address2 = dmaengine_cfg->src_addr;
|
|
|
|
sdmac->per_address = dmaengine_cfg->dst_addr;
|
|
|
|
sdmac->watermark_level = dmaengine_cfg->src_maxburst &
|
|
|
|
SDMA_WATERMARK_LEVEL_LWML;
|
|
|
|
sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
|
|
|
|
SDMA_WATERMARK_LEVEL_HWML;
|
|
|
|
sdmac->word_size = dmaengine_cfg->dst_addr_width;
|
2014-11-17 13:42:17 +00:00
|
|
|
} else {
|
|
|
|
sdmac->per_address = dmaengine_cfg->dst_addr;
|
|
|
|
sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
|
|
|
|
dmaengine_cfg->dst_addr_width;
|
|
|
|
sdmac->word_size = dmaengine_cfg->dst_addr_width;
|
|
|
|
}
|
2018-10-25 14:15:28 +00:00
|
|
|
sdmac->direction = direction;
|
2014-11-17 13:42:17 +00:00
|
|
|
return sdma_config_channel(chan);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2018-10-25 14:15:28 +00:00
|
|
|
static int sdma_config(struct dma_chan *chan,
|
|
|
|
struct dma_slave_config *dmaengine_cfg)
|
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
|
|
|
|
|
|
|
memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
|
|
|
|
|
|
|
|
/* Set ENBLn earlier to make sure dma request triggered after that */
|
2020-02-25 08:23:20 +00:00
|
|
|
if (sdmac->event_id0 >= 0) {
|
2018-10-25 14:15:28 +00:00
|
|
|
if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
|
|
|
|
return -EINVAL;
|
|
|
|
sdma_event_enable(sdmac, sdmac->event_id0);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sdmac->event_id1) {
|
|
|
|
if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
|
|
|
|
return -EINVAL;
|
|
|
|
sdma_event_enable(sdmac, sdmac->event_id1);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
static enum dma_status sdma_tx_status(struct dma_chan *chan,
|
2013-05-27 12:14:31 +00:00
|
|
|
dma_cookie_t cookie,
|
|
|
|
struct dma_tx_state *txstate)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2019-12-16 10:53:27 +00:00
|
|
|
struct sdma_desc *desc = NULL;
|
Update imx-sdma cyclic handling to report residue
I received a report this morning from one of the Novena developers that
the behaviour of the iMX6 ASoC codec driver (using imx-pcm-dma.c) was
sub-optimal under high system load.
While there are issues relating to system load remaining, upon reviewing
the ASoC imx-pcm-dma.c driver, it was noticed that it not using the
residue support, because SDMA doesn't support it. This has the effect
that SDMA has to make multiple calls into the ASoC and ALSA code, one
for each period.
Since ALSA's snd_pcm_elapsed() does not need to be called multiple times
and it is entirely sufficient to call it once to update ALSA with the
current buffer position via the pointer method, we can do better here.
We can also avoid stopping the DMA entirely, just like real cyclic DMA
implementations behave. While this means that we replay some old samples,
this is a nicer behaviour than having audio stop and restart.
The changes to achieve this are relatively minor - imx-sdma.c can track
where the DMA is to the nearest descriptor boundary - it does this
already when deciding how many callbacks to issue. In doing this,
buf_tail always points at the descriptor which will complete next.
The residue is defined by the bytes remaining to the end of the buffer,
when the buffer is viewed as a single block of memory [start...end].
So, when we start out, there's a full buffer worth of residue, and this
counts down as we approach the end of the buffer, eventually becoming
zero at the end, before returning to the full buffer worth when we
wrap back to the start.
Moving the walking of the descriptors into the interrupt handler means
that we can update the BD_DONE flag at interrupt time, thus avoiding
a delayed tasklet stopping the cyclic DMA.
This means that the residue can be calculated from (total descriptors -
buf_tail) * descriptor size. This is what the change below does. We
update imx-pcm-dma.c to remove the NO_RESIDUE flag since we now provide
the residue.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Tested-by: Shawn Guo <shawn.guo@linaro.org>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2014-06-25 12:00:33 +00:00
|
|
|
u32 residue;
|
2018-06-19 16:57:00 +00:00
|
|
|
struct virt_dma_desc *vd;
|
|
|
|
enum dma_status ret;
|
|
|
|
unsigned long flags;
|
Update imx-sdma cyclic handling to report residue
I received a report this morning from one of the Novena developers that
the behaviour of the iMX6 ASoC codec driver (using imx-pcm-dma.c) was
sub-optimal under high system load.
While there are issues relating to system load remaining, upon reviewing
the ASoC imx-pcm-dma.c driver, it was noticed that it not using the
residue support, because SDMA doesn't support it. This has the effect
that SDMA has to make multiple calls into the ASoC and ALSA code, one
for each period.
Since ALSA's snd_pcm_elapsed() does not need to be called multiple times
and it is entirely sufficient to call it once to update ALSA with the
current buffer position via the pointer method, we can do better here.
We can also avoid stopping the DMA entirely, just like real cyclic DMA
implementations behave. While this means that we replay some old samples,
this is a nicer behaviour than having audio stop and restart.
The changes to achieve this are relatively minor - imx-sdma.c can track
where the DMA is to the nearest descriptor boundary - it does this
already when deciding how many callbacks to issue. In doing this,
buf_tail always points at the descriptor which will complete next.
The residue is defined by the bytes remaining to the end of the buffer,
when the buffer is viewed as a single block of memory [start...end].
So, when we start out, there's a full buffer worth of residue, and this
counts down as we approach the end of the buffer, eventually becoming
zero at the end, before returning to the full buffer worth when we
wrap back to the start.
Moving the walking of the descriptors into the interrupt handler means
that we can update the BD_DONE flag at interrupt time, thus avoiding
a delayed tasklet stopping the cyclic DMA.
This means that the residue can be calculated from (total descriptors -
buf_tail) * descriptor size. This is what the change below does. We
update imx-pcm-dma.c to remove the NO_RESIDUE flag since we now provide
the residue.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Tested-by: Shawn Guo <shawn.guo@linaro.org>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2014-06-25 12:00:33 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
ret = dma_cookie_status(chan, cookie, txstate);
|
|
|
|
if (ret == DMA_COMPLETE || !txstate)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&sdmac->vc.lock, flags);
|
2019-12-16 10:53:27 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
vd = vchan_find_desc(&sdmac->vc, cookie);
|
2019-12-16 10:53:27 +00:00
|
|
|
if (vd)
|
2018-06-19 16:57:00 +00:00
|
|
|
desc = to_sdma_desc(&vd->tx);
|
2019-12-16 10:53:27 +00:00
|
|
|
else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
|
|
|
|
desc = sdmac->desc;
|
|
|
|
|
|
|
|
if (desc) {
|
2018-06-19 16:57:00 +00:00
|
|
|
if (sdmac->flags & IMX_DMA_SG_LOOP)
|
|
|
|
residue = (desc->num_bd - desc->buf_ptail) *
|
|
|
|
desc->period_len - desc->chn_real_count;
|
|
|
|
else
|
|
|
|
residue = desc->chn_count - desc->chn_real_count;
|
|
|
|
} else {
|
|
|
|
residue = 0;
|
|
|
|
}
|
2019-12-16 10:53:27 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2013-05-27 12:14:31 +00:00
|
|
|
dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,
|
Update imx-sdma cyclic handling to report residue
I received a report this morning from one of the Novena developers that
the behaviour of the iMX6 ASoC codec driver (using imx-pcm-dma.c) was
sub-optimal under high system load.
While there are issues relating to system load remaining, upon reviewing
the ASoC imx-pcm-dma.c driver, it was noticed that it not using the
residue support, because SDMA doesn't support it. This has the effect
that SDMA has to make multiple calls into the ASoC and ALSA code, one
for each period.
Since ALSA's snd_pcm_elapsed() does not need to be called multiple times
and it is entirely sufficient to call it once to update ALSA with the
current buffer position via the pointer method, we can do better here.
We can also avoid stopping the DMA entirely, just like real cyclic DMA
implementations behave. While this means that we replay some old samples,
this is a nicer behaviour than having audio stop and restart.
The changes to achieve this are relatively minor - imx-sdma.c can track
where the DMA is to the nearest descriptor boundary - it does this
already when deciding how many callbacks to issue. In doing this,
buf_tail always points at the descriptor which will complete next.
The residue is defined by the bytes remaining to the end of the buffer,
when the buffer is viewed as a single block of memory [start...end].
So, when we start out, there's a full buffer worth of residue, and this
counts down as we approach the end of the buffer, eventually becoming
zero at the end, before returning to the full buffer worth when we
wrap back to the start.
Moving the walking of the descriptors into the interrupt handler means
that we can update the BD_DONE flag at interrupt time, thus avoiding
a delayed tasklet stopping the cyclic DMA.
This means that the residue can be calculated from (total descriptors -
buf_tail) * descriptor size. This is what the change below does. We
update imx-pcm-dma.c to remove the NO_RESIDUE flag since we now provide
the residue.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Tested-by: Shawn Guo <shawn.guo@linaro.org>
Signed-off-by: Vinod Koul <vinod.koul@intel.com>
2014-06-25 12:00:33 +00:00
|
|
|
residue);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2011-01-19 21:50:37 +00:00
|
|
|
return sdmac->status;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void sdma_issue_pending(struct dma_chan *chan)
|
|
|
|
{
|
2012-01-09 09:32:50 +00:00
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2018-06-19 16:57:00 +00:00
|
|
|
unsigned long flags;
|
2012-01-09 09:32:50 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
spin_lock_irqsave(&sdmac->vc.lock, flags);
|
|
|
|
if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
|
|
|
|
sdma_start_desc(sdmac);
|
|
|
|
spin_unlock_irqrestore(&sdmac->vc.lock, flags);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2010-10-06 13:41:15 +00:00
|
|
|
#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34
|
2013-11-13 14:55:24 +00:00
|
|
|
#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38
|
2015-03-11 15:30:58 +00:00
|
|
|
#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41
|
2016-08-10 16:05:05 +00:00
|
|
|
#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42
|
2010-10-06 13:41:15 +00:00
|
|
|
|
|
|
|
static void sdma_add_scripts(struct sdma_engine *sdma,
|
|
|
|
const struct sdma_script_start_addrs *addr)
|
|
|
|
{
|
|
|
|
s32 *addr_arr = (u32 *)addr;
|
|
|
|
s32 *saddr_arr = (u32 *)sdma->script_addrs;
|
|
|
|
int i;
|
|
|
|
|
2014-01-08 08:45:56 +00:00
|
|
|
/* use the default firmware in ROM if missing external firmware */
|
|
|
|
if (!sdma->script_number)
|
|
|
|
sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
|
|
|
|
|
2019-09-24 09:49:18 +00:00
|
|
|
if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
|
|
|
|
/ sizeof(s32)) {
|
|
|
|
dev_err(sdma->dev,
|
|
|
|
"SDMA script number %d not match with firmware.\n",
|
|
|
|
sdma->script_number);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2013-11-13 14:55:24 +00:00
|
|
|
for (i = 0; i < sdma->script_number; i++)
|
2010-10-06 13:41:15 +00:00
|
|
|
if (addr_arr[i] > 0)
|
|
|
|
saddr_arr[i] = addr_arr[i];
|
|
|
|
}
|
|
|
|
|
2011-08-25 09:03:37 +00:00
|
|
|
static void sdma_load_firmware(const struct firmware *fw, void *context)
|
2010-10-06 13:41:15 +00:00
|
|
|
{
|
2011-08-25 09:03:37 +00:00
|
|
|
struct sdma_engine *sdma = context;
|
2010-10-06 13:41:15 +00:00
|
|
|
const struct sdma_firmware_header *header;
|
|
|
|
const struct sdma_script_start_addrs *addr;
|
|
|
|
unsigned short *ram_code;
|
|
|
|
|
2011-08-25 09:03:37 +00:00
|
|
|
if (!fw) {
|
2014-11-12 22:04:29 +00:00
|
|
|
dev_info(sdma->dev, "external firmware not found, using ROM firmware\n");
|
|
|
|
/* In this case we just use the ROM firmware. */
|
2011-08-25 09:03:37 +00:00
|
|
|
return;
|
|
|
|
}
|
2010-10-06 13:41:15 +00:00
|
|
|
|
|
|
|
if (fw->size < sizeof(*header))
|
|
|
|
goto err_firmware;
|
|
|
|
|
|
|
|
header = (struct sdma_firmware_header *)fw->data;
|
|
|
|
|
|
|
|
if (header->magic != SDMA_FIRMWARE_MAGIC)
|
|
|
|
goto err_firmware;
|
|
|
|
if (header->ram_code_start + header->ram_code_size > fw->size)
|
|
|
|
goto err_firmware;
|
2013-11-13 14:55:24 +00:00
|
|
|
switch (header->version_major) {
|
2014-12-10 08:00:36 +00:00
|
|
|
case 1:
|
|
|
|
sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
|
|
|
|
break;
|
2015-03-11 15:30:58 +00:00
|
|
|
case 3:
|
|
|
|
sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
|
|
|
|
break;
|
2016-08-10 16:05:05 +00:00
|
|
|
case 4:
|
|
|
|
sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
|
|
|
|
break;
|
2014-12-10 08:00:36 +00:00
|
|
|
default:
|
|
|
|
dev_err(sdma->dev, "unknown firmware version\n");
|
|
|
|
goto err_firmware;
|
2013-11-13 14:55:24 +00:00
|
|
|
}
|
2010-10-06 13:41:15 +00:00
|
|
|
|
|
|
|
addr = (void *)header + header->script_addrs_start;
|
|
|
|
ram_code = (void *)header + header->ram_code_start;
|
|
|
|
|
2012-03-07 08:30:06 +00:00
|
|
|
clk_enable(sdma->clk_ipg);
|
|
|
|
clk_enable(sdma->clk_ahb);
|
2010-10-06 13:41:15 +00:00
|
|
|
/* download the RAM image for SDMA */
|
|
|
|
sdma_load_script(sdma, ram_code,
|
|
|
|
header->ram_code_size,
|
2011-01-12 10:18:14 +00:00
|
|
|
addr->ram_code_start_addr);
|
2012-03-07 08:30:06 +00:00
|
|
|
clk_disable(sdma->clk_ipg);
|
|
|
|
clk_disable(sdma->clk_ahb);
|
2010-10-06 13:41:15 +00:00
|
|
|
|
|
|
|
sdma_add_scripts(sdma, addr);
|
|
|
|
|
|
|
|
dev_info(sdma->dev, "loaded firmware %d.%d\n",
|
|
|
|
header->version_major,
|
|
|
|
header->version_minor);
|
|
|
|
|
|
|
|
err_firmware:
|
|
|
|
release_firmware(fw);
|
2011-08-25 09:03:37 +00:00
|
|
|
}
|
|
|
|
|
2015-07-23 03:40:49 +00:00
|
|
|
#define EVENT_REMAP_CELLS 3
|
|
|
|
|
2015-11-11 09:20:49 +00:00
|
|
|
static int sdma_event_remap(struct sdma_engine *sdma)
|
2015-07-23 03:40:49 +00:00
|
|
|
{
|
|
|
|
struct device_node *np = sdma->dev->of_node;
|
|
|
|
struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
|
|
|
|
struct property *event_remap;
|
|
|
|
struct regmap *gpr;
|
|
|
|
char propname[] = "fsl,sdma-event-remap";
|
|
|
|
u32 reg, val, shift, num_map, i;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
if (IS_ERR(np) || IS_ERR(gpr_np))
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
event_remap = of_find_property(np, propname, NULL);
|
|
|
|
num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
|
|
|
|
if (!num_map) {
|
2015-10-03 22:37:58 +00:00
|
|
|
dev_dbg(sdma->dev, "no event needs to be remapped\n");
|
2015-07-23 03:40:49 +00:00
|
|
|
goto out;
|
|
|
|
} else if (num_map % EVENT_REMAP_CELLS) {
|
|
|
|
dev_err(sdma->dev, "the property %s must modulo %d\n",
|
|
|
|
propname, EVENT_REMAP_CELLS);
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
gpr = syscon_node_to_regmap(gpr_np);
|
|
|
|
if (IS_ERR(gpr)) {
|
|
|
|
dev_err(sdma->dev, "failed to get gpr regmap\n");
|
|
|
|
ret = PTR_ERR(gpr);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
|
|
|
|
ret = of_property_read_u32_index(np, propname, i, ®);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(sdma->dev, "failed to read property %s index %d\n",
|
|
|
|
propname, i);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = of_property_read_u32_index(np, propname, i + 1, &shift);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(sdma->dev, "failed to read property %s index %d\n",
|
|
|
|
propname, i + 1);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = of_property_read_u32_index(np, propname, i + 2, &val);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(sdma->dev, "failed to read property %s index %d\n",
|
|
|
|
propname, i + 2);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
regmap_update_bits(gpr, reg, BIT(shift), val << shift);
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
if (!IS_ERR(gpr_np))
|
|
|
|
of_node_put(gpr_np);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-09-26 21:24:00 +00:00
|
|
|
static int sdma_get_firmware(struct sdma_engine *sdma,
|
2011-08-25 09:03:37 +00:00
|
|
|
const char *fw_name)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = request_firmware_nowait(THIS_MODULE,
|
|
|
|
FW_ACTION_HOTPLUG, fw_name, sdma->dev,
|
|
|
|
GFP_KERNEL, sdma, sdma_load_firmware);
|
2010-10-06 13:41:15 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-11-06 01:10:09 +00:00
|
|
|
static int sdma_init(struct sdma_engine *sdma)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
|
|
|
int i, ret;
|
|
|
|
dma_addr_t ccb_phys;
|
|
|
|
|
2015-07-30 00:03:49 +00:00
|
|
|
ret = clk_enable(sdma->clk_ipg);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
ret = clk_enable(sdma->clk_ahb);
|
|
|
|
if (ret)
|
|
|
|
goto disable_clk_ipg;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2019-03-29 15:21:29 +00:00
|
|
|
if (sdma->drvdata->check_ratio &&
|
|
|
|
(clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
|
2019-01-28 16:03:21 +00:00
|
|
|
sdma->clk_ratio = 1;
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
/* Be sure SDMA has not started yet */
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2019-01-11 14:29:49 +00:00
|
|
|
sdma->channel_control = dma_alloc_coherent(sdma->dev,
|
2010-09-30 13:56:34 +00:00
|
|
|
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) +
|
|
|
|
sizeof(struct sdma_context_data),
|
|
|
|
&ccb_phys, GFP_KERNEL);
|
|
|
|
|
|
|
|
if (!sdma->channel_control) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_dma_alloc;
|
|
|
|
}
|
|
|
|
|
|
|
|
sdma->context = (void *)sdma->channel_control +
|
|
|
|
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
|
|
|
|
sdma->context_phys = ccb_phys +
|
|
|
|
MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control);
|
|
|
|
|
|
|
|
/* disable all channels */
|
2013-08-20 08:04:31 +00:00
|
|
|
for (i = 0; i < sdma->drvdata->num_events; i++)
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
/* All channels have priority 0 */
|
|
|
|
for (i = 0; i < MAX_DMA_CHANNELS; i++)
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
ret = sdma_request_channel0(sdma);
|
2010-09-30 13:56:34 +00:00
|
|
|
if (ret)
|
|
|
|
goto err_dma_alloc;
|
|
|
|
|
|
|
|
sdma_config_ownership(&sdma->channel[0], false, true, false);
|
|
|
|
|
|
|
|
/* Set Command Channel (Channel Zero) */
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
/* Set bits of CONFIG register but with static context switching */
|
2019-01-28 16:03:21 +00:00
|
|
|
if (sdma->clk_ratio)
|
|
|
|
writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
|
|
|
|
else
|
|
|
|
writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-01-13 03:09:57 +00:00
|
|
|
writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
/* Initializes channel's priorities */
|
|
|
|
sdma_set_channel_priority(&sdma->channel[0], 7);
|
|
|
|
|
2012-03-07 08:30:06 +00:00
|
|
|
clk_disable(sdma->clk_ipg);
|
|
|
|
clk_disable(sdma->clk_ahb);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_dma_alloc:
|
2012-03-07 08:30:06 +00:00
|
|
|
clk_disable(sdma->clk_ahb);
|
2015-07-30 00:03:49 +00:00
|
|
|
disable_clk_ipg:
|
|
|
|
clk_disable(sdma->clk_ipg);
|
2010-09-30 13:56:34 +00:00
|
|
|
dev_err(sdma->dev, "initialisation failed with %d\n", ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2013-05-30 14:23:32 +00:00
|
|
|
static bool sdma_filter_fn(struct dma_chan *chan, void *fn_param)
|
|
|
|
{
|
2014-06-16 03:32:29 +00:00
|
|
|
struct sdma_channel *sdmac = to_sdma_chan(chan);
|
2013-05-30 14:23:32 +00:00
|
|
|
struct imx_dma_data *data = fn_param;
|
|
|
|
|
|
|
|
if (!imx_dma_is_general_purpose(chan))
|
|
|
|
return false;
|
|
|
|
|
2014-06-16 03:32:29 +00:00
|
|
|
sdmac->data = *data;
|
|
|
|
chan->private = &sdmac->data;
|
2013-05-30 14:23:32 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
|
|
|
|
struct of_dma *ofdma)
|
|
|
|
{
|
|
|
|
struct sdma_engine *sdma = ofdma->of_dma_data;
|
|
|
|
dma_cap_mask_t mask = sdma->dma_device.cap_mask;
|
|
|
|
struct imx_dma_data data;
|
|
|
|
|
|
|
|
if (dma_spec->args_count != 3)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
data.dma_request = dma_spec->args[0];
|
|
|
|
data.peripheral_type = dma_spec->args[1];
|
|
|
|
data.priority = dma_spec->args[2];
|
2015-07-10 09:08:16 +00:00
|
|
|
/*
|
|
|
|
* init dma_request2 to zero, which is not used by the dts.
|
|
|
|
* For P2P, dma_request2 is init from dma_request_channel(),
|
|
|
|
* chan->private will point to the imx_dma_data, and in
|
|
|
|
* device_alloc_chan_resources(), imx_dma_data.dma_request2 will
|
|
|
|
* be set to sdmac->event_id1.
|
|
|
|
*/
|
|
|
|
data.dma_request2 = 0;
|
2013-05-30 14:23:32 +00:00
|
|
|
|
2019-05-20 11:32:16 +00:00
|
|
|
return __dma_request_channel(&mask, sdma_filter_fn, &data,
|
|
|
|
ofdma->of_node);
|
2013-05-30 14:23:32 +00:00
|
|
|
}
|
|
|
|
|
2014-08-27 10:55:53 +00:00
|
|
|
static int sdma_probe(struct platform_device *pdev)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2011-07-14 00:35:48 +00:00
|
|
|
const struct of_device_id *of_id =
|
|
|
|
of_match_device(sdma_dt_ids, &pdev->dev);
|
|
|
|
struct device_node *np = pdev->dev.of_node;
|
2015-07-10 09:08:16 +00:00
|
|
|
struct device_node *spba_bus;
|
2011-07-14 00:35:48 +00:00
|
|
|
const char *fw_name;
|
2010-09-30 13:56:34 +00:00
|
|
|
int ret;
|
|
|
|
int irq;
|
|
|
|
struct resource *iores;
|
2015-07-10 09:08:16 +00:00
|
|
|
struct resource spba_res;
|
2013-07-30 08:09:11 +00:00
|
|
|
struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev);
|
2010-09-30 13:56:34 +00:00
|
|
|
int i;
|
|
|
|
struct sdma_engine *sdma;
|
2011-08-25 09:03:36 +00:00
|
|
|
s32 *saddr_arr;
|
2013-08-20 08:04:31 +00:00
|
|
|
const struct sdma_driver_data *drvdata = NULL;
|
|
|
|
|
|
|
|
if (of_id)
|
|
|
|
drvdata = of_id->data;
|
|
|
|
else if (pdev->id_entry)
|
|
|
|
drvdata = (void *)pdev->id_entry->driver_data;
|
|
|
|
|
|
|
|
if (!drvdata) {
|
|
|
|
dev_err(&pdev->dev, "unable to find driver data\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2013-10-14 08:45:17 +00:00
|
|
|
ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2014-12-29 17:20:52 +00:00
|
|
|
sdma = devm_kzalloc(&pdev->dev, sizeof(*sdma), GFP_KERNEL);
|
2010-09-30 13:56:34 +00:00
|
|
|
if (!sdma)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2012-05-11 07:14:27 +00:00
|
|
|
spin_lock_init(&sdma->channel_0_lock);
|
2011-08-25 09:03:35 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
sdma->dev = &pdev->dev;
|
2013-08-20 08:04:31 +00:00
|
|
|
sdma->drvdata = drvdata;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
irq = platform_get_irq(pdev, 0);
|
2014-12-29 17:20:52 +00:00
|
|
|
if (irq < 0)
|
2014-12-29 17:20:53 +00:00
|
|
|
return irq;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2014-12-29 17:20:52 +00:00
|
|
|
iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
|
|
|
sdma->regs = devm_ioremap_resource(&pdev->dev, iores);
|
|
|
|
if (IS_ERR(sdma->regs))
|
|
|
|
return PTR_ERR(sdma->regs);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-03-07 08:30:06 +00:00
|
|
|
sdma->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
|
2014-12-29 17:20:52 +00:00
|
|
|
if (IS_ERR(sdma->clk_ipg))
|
|
|
|
return PTR_ERR(sdma->clk_ipg);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2012-03-07 08:30:06 +00:00
|
|
|
sdma->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
|
2014-12-29 17:20:52 +00:00
|
|
|
if (IS_ERR(sdma->clk_ahb))
|
|
|
|
return PTR_ERR(sdma->clk_ahb);
|
2012-03-07 08:30:06 +00:00
|
|
|
|
2017-05-24 06:39:53 +00:00
|
|
|
ret = clk_prepare(sdma->clk_ipg);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = clk_prepare(sdma->clk_ahb);
|
|
|
|
if (ret)
|
|
|
|
goto err_clk;
|
2012-03-07 08:30:06 +00:00
|
|
|
|
2014-12-29 17:20:52 +00:00
|
|
|
ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma",
|
|
|
|
sdma);
|
2010-09-30 13:56:34 +00:00
|
|
|
if (ret)
|
2017-05-24 06:39:53 +00:00
|
|
|
goto err_irq;
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2016-07-02 18:30:55 +00:00
|
|
|
sdma->irq = irq;
|
|
|
|
|
2010-10-06 13:41:15 +00:00
|
|
|
sdma->script_addrs = kzalloc(sizeof(*sdma->script_addrs), GFP_KERNEL);
|
2017-05-24 06:39:53 +00:00
|
|
|
if (!sdma->script_addrs) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_irq;
|
|
|
|
}
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2011-08-25 09:03:36 +00:00
|
|
|
/* initially no scripts available */
|
|
|
|
saddr_arr = (s32 *)sdma->script_addrs;
|
2020-05-13 06:04:05 +00:00
|
|
|
for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++)
|
2011-08-25 09:03:36 +00:00
|
|
|
saddr_arr[i] = -EINVAL;
|
|
|
|
|
2011-01-31 09:21:35 +00:00
|
|
|
dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask);
|
|
|
|
dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask);
|
2018-07-23 17:46:11 +00:00
|
|
|
dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask);
|
2011-01-31 09:21:35 +00:00
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
INIT_LIST_HEAD(&sdma->dma_device.channels);
|
|
|
|
/* Initialize channel parameters */
|
|
|
|
for (i = 0; i < MAX_DMA_CHANNELS; i++) {
|
|
|
|
struct sdma_channel *sdmac = &sdma->channel[i];
|
|
|
|
|
|
|
|
sdmac->sdma = sdma;
|
|
|
|
|
|
|
|
sdmac->channel = i;
|
2018-06-19 16:57:00 +00:00
|
|
|
sdmac->vc.desc_free = sdma_desc_free;
|
2018-11-06 03:40:33 +00:00
|
|
|
INIT_WORK(&sdmac->terminate_worker,
|
|
|
|
sdma_channel_terminate_work);
|
2011-01-31 09:56:58 +00:00
|
|
|
/*
|
|
|
|
* Add the channel to the DMAC list. Do not add channel 0 though
|
|
|
|
* because we need it internally in the SDMA driver. This also means
|
|
|
|
* that channel 0 in dmaengine counting matches sdma channel 1.
|
|
|
|
*/
|
|
|
|
if (i)
|
2018-06-19 16:57:00 +00:00
|
|
|
vchan_init(&sdmac->vc, &sdma->dma_device);
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2010-10-06 13:41:15 +00:00
|
|
|
ret = sdma_init(sdma);
|
2010-09-30 13:56:34 +00:00
|
|
|
if (ret)
|
|
|
|
goto err_init;
|
|
|
|
|
2015-07-23 03:40:49 +00:00
|
|
|
ret = sdma_event_remap(sdma);
|
|
|
|
if (ret)
|
|
|
|
goto err_init;
|
|
|
|
|
2013-08-20 08:04:32 +00:00
|
|
|
if (sdma->drvdata->script_addrs)
|
|
|
|
sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
|
2011-07-14 00:35:48 +00:00
|
|
|
if (pdata && pdata->script_addrs)
|
2010-10-06 13:41:15 +00:00
|
|
|
sdma_add_scripts(sdma, pdata->script_addrs);
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
sdma->dma_device.dev = &pdev->dev;
|
|
|
|
|
|
|
|
sdma->dma_device.device_alloc_chan_resources = sdma_alloc_chan_resources;
|
|
|
|
sdma->dma_device.device_free_chan_resources = sdma_free_chan_resources;
|
|
|
|
sdma->dma_device.device_tx_status = sdma_tx_status;
|
|
|
|
sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg;
|
|
|
|
sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic;
|
2014-11-17 13:42:17 +00:00
|
|
|
sdma->dma_device.device_config = sdma_config;
|
2019-12-16 10:53:26 +00:00
|
|
|
sdma->dma_device.device_terminate_all = sdma_terminate_all;
|
2018-11-06 03:40:33 +00:00
|
|
|
sdma->dma_device.device_synchronize = sdma_channel_synchronize;
|
2017-09-14 18:46:43 +00:00
|
|
|
sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS;
|
|
|
|
sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS;
|
|
|
|
sdma->dma_device.directions = SDMA_DMA_DIRECTIONS;
|
2017-03-08 09:13:09 +00:00
|
|
|
sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
|
2018-07-23 17:46:11 +00:00
|
|
|
sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy;
|
2010-09-30 13:56:34 +00:00
|
|
|
sdma->dma_device.device_issue_pending = sdma_issue_pending;
|
2011-01-12 11:12:31 +00:00
|
|
|
sdma->dma_device.dev->dma_parms = &sdma->dma_parms;
|
2019-01-28 16:03:23 +00:00
|
|
|
sdma->dma_device.copy_align = 2;
|
2018-07-23 17:46:10 +00:00
|
|
|
dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
2014-08-05 13:09:41 +00:00
|
|
|
platform_set_drvdata(pdev, sdma);
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
ret = dma_async_device_register(&sdma->dma_device);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(&pdev->dev, "unable to register\n");
|
|
|
|
goto err_init;
|
|
|
|
}
|
|
|
|
|
2013-05-30 14:23:32 +00:00
|
|
|
if (np) {
|
|
|
|
ret = of_dma_controller_register(np, sdma_xlate, sdma);
|
|
|
|
if (ret) {
|
|
|
|
dev_err(&pdev->dev, "failed to register controller\n");
|
|
|
|
goto err_register;
|
|
|
|
}
|
2015-07-10 09:08:16 +00:00
|
|
|
|
|
|
|
spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
|
|
|
|
ret = of_address_to_resource(spba_bus, 0, &spba_res);
|
|
|
|
if (!ret) {
|
|
|
|
sdma->spba_start_addr = spba_res.start;
|
|
|
|
sdma->spba_end_addr = spba_res.end;
|
|
|
|
}
|
|
|
|
of_node_put(spba_bus);
|
2013-05-30 14:23:32 +00:00
|
|
|
}
|
|
|
|
|
2019-06-24 14:07:31 +00:00
|
|
|
/*
|
|
|
|
* Kick off firmware loading as the very last step:
|
|
|
|
* attempt to load firmware only if we're not on the error path, because
|
|
|
|
* the firmware callback requires a fully functional and allocated sdma
|
|
|
|
* instance.
|
|
|
|
*/
|
|
|
|
if (pdata) {
|
|
|
|
ret = sdma_get_firmware(sdma, pdata->fw_name);
|
|
|
|
if (ret)
|
|
|
|
dev_warn(&pdev->dev, "failed to get firmware from platform data\n");
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Because that device tree does not encode ROM script address,
|
|
|
|
* the RAM script in firmware is mandatory for device tree
|
|
|
|
* probe, otherwise it fails.
|
|
|
|
*/
|
|
|
|
ret = of_property_read_string(np, "fsl,sdma-ram-script-name",
|
|
|
|
&fw_name);
|
|
|
|
if (ret) {
|
|
|
|
dev_warn(&pdev->dev, "failed to get firmware name\n");
|
|
|
|
} else {
|
|
|
|
ret = sdma_get_firmware(sdma, fw_name);
|
|
|
|
if (ret)
|
|
|
|
dev_warn(&pdev->dev, "failed to get firmware from device tree\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-09-30 13:56:34 +00:00
|
|
|
return 0;
|
|
|
|
|
2013-05-30 14:23:32 +00:00
|
|
|
err_register:
|
|
|
|
dma_async_device_unregister(&sdma->dma_device);
|
2010-09-30 13:56:34 +00:00
|
|
|
err_init:
|
|
|
|
kfree(sdma->script_addrs);
|
2017-05-24 06:39:53 +00:00
|
|
|
err_irq:
|
|
|
|
clk_unprepare(sdma->clk_ahb);
|
|
|
|
err_clk:
|
|
|
|
clk_unprepare(sdma->clk_ipg);
|
2011-01-19 11:13:06 +00:00
|
|
|
return ret;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
2013-02-20 00:07:04 +00:00
|
|
|
static int sdma_remove(struct platform_device *pdev)
|
2010-09-30 13:56:34 +00:00
|
|
|
{
|
2014-08-05 13:09:41 +00:00
|
|
|
struct sdma_engine *sdma = platform_get_drvdata(pdev);
|
2014-08-05 13:09:42 +00:00
|
|
|
int i;
|
2014-08-05 13:09:41 +00:00
|
|
|
|
2016-07-02 18:30:55 +00:00
|
|
|
devm_free_irq(&pdev->dev, sdma->irq, sdma);
|
2014-08-05 13:09:41 +00:00
|
|
|
dma_async_device_unregister(&sdma->dma_device);
|
|
|
|
kfree(sdma->script_addrs);
|
2017-05-24 06:39:53 +00:00
|
|
|
clk_unprepare(sdma->clk_ahb);
|
|
|
|
clk_unprepare(sdma->clk_ipg);
|
2014-08-05 13:09:42 +00:00
|
|
|
/* Kill the tasklet */
|
|
|
|
for (i = 0; i < MAX_DMA_CHANNELS; i++) {
|
|
|
|
struct sdma_channel *sdmac = &sdma->channel[i];
|
|
|
|
|
2018-06-19 16:57:00 +00:00
|
|
|
tasklet_kill(&sdmac->vc.task);
|
|
|
|
sdma_free_chan_resources(&sdmac->vc.chan);
|
2014-08-05 13:09:42 +00:00
|
|
|
}
|
2014-08-05 13:09:41 +00:00
|
|
|
|
|
|
|
platform_set_drvdata(pdev, NULL);
|
|
|
|
return 0;
|
2010-09-30 13:56:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct platform_driver sdma_driver = {
|
|
|
|
.driver = {
|
|
|
|
.name = "imx-sdma",
|
2011-07-14 00:35:48 +00:00
|
|
|
.of_match_table = sdma_dt_ids,
|
2010-09-30 13:56:34 +00:00
|
|
|
},
|
2011-07-13 13:33:17 +00:00
|
|
|
.id_table = sdma_devtypes,
|
2013-02-20 00:07:04 +00:00
|
|
|
.remove = sdma_remove,
|
2014-08-05 13:09:41 +00:00
|
|
|
.probe = sdma_probe,
|
2010-09-30 13:56:34 +00:00
|
|
|
};
|
|
|
|
|
2014-08-05 13:09:41 +00:00
|
|
|
module_platform_driver(sdma_driver);
|
2010-09-30 13:56:34 +00:00
|
|
|
|
|
|
|
MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
|
|
|
|
MODULE_DESCRIPTION("i.MX SDMA driver");
|
2017-12-13 15:50:33 +00:00
|
|
|
#if IS_ENABLED(CONFIG_SOC_IMX6Q)
|
|
|
|
MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
|
|
|
|
#endif
|
|
|
|
#if IS_ENABLED(CONFIG_SOC_IMX7D)
|
|
|
|
MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
|
|
|
|
#endif
|
2010-09-30 13:56:34 +00:00
|
|
|
MODULE_LICENSE("GPL");
|