* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
+#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dmaengine.h>
#include <linux/slab.h>
#include "dw_dmac_regs.h"
+#include "dmaengine.h"
/*
* This supports the Synopsys "DesignWare AHB Central DMA Controller",
* which does not support descriptor writeback.
*/
-#define DWC_DEFAULT_CTLLO(private) ({ \
- struct dw_dma_slave *__slave = (private); \
- int dms = __slave ? __slave->dst_master : 0; \
- int sms = __slave ? __slave->src_master : 1; \
- u8 smsize = __slave ? __slave->src_msize : DW_DMA_MSIZE_16; \
- u8 dmsize = __slave ? __slave->dst_msize : DW_DMA_MSIZE_16; \
+#define DWC_DEFAULT_CTLLO(_chan) ({ \
+ struct dw_dma_slave *__slave = (_chan->private); \
+ struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan); \
+ struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
+ int _dms = __slave ? __slave->dst_master : 0; \
+ int _sms = __slave ? __slave->src_master : 1; \
+ u8 _smsize = __slave ? _sconfig->src_maxburst : \
+ DW_DMA_MSIZE_16; \
+ u8 _dmsize = __slave ? _sconfig->dst_maxburst : \
+ DW_DMA_MSIZE_16; \
\
- (DWC_CTLL_DST_MSIZE(dmsize) \
- | DWC_CTLL_SRC_MSIZE(smsize) \
+ (DWC_CTLL_DST_MSIZE(_dmsize) \
+ | DWC_CTLL_SRC_MSIZE(_smsize) \
| DWC_CTLL_LLP_D_EN \
| DWC_CTLL_LLP_S_EN \
- | DWC_CTLL_DMS(dms) \
- | DWC_CTLL_SMS(sms)); \
+ | DWC_CTLL_DMS(_dms) \
+ | DWC_CTLL_SMS(_sms)); \
})
/*
}
}
-/* Called with dwc->lock held and bh disabled */
-static dma_cookie_t
-dwc_assign_cookie(struct dw_dma_chan *dwc, struct dw_desc *desc)
-{
- dma_cookie_t cookie = dwc->chan.cookie;
-
- if (++cookie < 0)
- cookie = 1;
-
- dwc->chan.cookie = cookie;
- desc->txd.cookie = cookie;
-
- return cookie;
-}
-
static void dwc_initialize(struct dw_dma_chan *dwc)
{
struct dw_dma *dw = to_dw_dma(dwc->chan.device);
/* Enable interrupts */
channel_set_bit(dw, MASK.XFER, dwc->mask);
- channel_set_bit(dw, MASK.BLOCK, dwc->mask);
channel_set_bit(dw, MASK.ERROR, dwc->mask);
dwc->initialized = true;
dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
spin_lock_irqsave(&dwc->lock, flags);
- dwc->completed = txd->cookie;
+ dma_cookie_complete(txd);
if (callback_required) {
callback = txd->callback;
param = txd->callback_param;
unsigned long flags;
spin_lock_irqsave(&dwc->lock, flags);
- /*
- * Clear block interrupt flag before scanning so that we don't
- * miss any, and read LLP before RAW_XFER to ensure it is
- * valid if we decide to scan the list.
- */
- dma_writel(dw, CLEAR.BLOCK, dwc->mask);
llp = channel_readl(dwc, LLP);
status_xfer = dma_readl(dw, RAW.XFER);
/* called with dwc->lock held and all DMAC interrupts disabled */
static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
- u32 status_block, u32 status_err, u32 status_xfer)
+ u32 status_err, u32 status_xfer)
{
unsigned long flags;
- if (status_block & dwc->mask) {
+ if (dwc->mask) {
void (*callback)(void *param);
void *callback_param;
dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
channel_readl(dwc, LLP));
- dma_writel(dw, CLEAR.BLOCK, dwc->mask);
callback = dwc->cdesc->period_callback;
callback_param = dwc->cdesc->period_callback_param;
channel_writel(dwc, CTL_LO, 0);
channel_writel(dwc, CTL_HI, 0);
- dma_writel(dw, CLEAR.BLOCK, dwc->mask);
dma_writel(dw, CLEAR.ERROR, dwc->mask);
dma_writel(dw, CLEAR.XFER, dwc->mask);
{
struct dw_dma *dw = (struct dw_dma *)data;
struct dw_dma_chan *dwc;
- u32 status_block;
u32 status_xfer;
u32 status_err;
int i;
- status_block = dma_readl(dw, RAW.BLOCK);
status_xfer = dma_readl(dw, RAW.XFER);
status_err = dma_readl(dw, RAW.ERROR);
- dev_vdbg(dw->dma.dev, "tasklet: status_block=%x status_err=%x\n",
- status_block, status_err);
+ dev_vdbg(dw->dma.dev, "tasklet: status_err=%x\n", status_err);
for (i = 0; i < dw->dma.chancnt; i++) {
dwc = &dw->chan[i];
if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
- dwc_handle_cyclic(dw, dwc, status_block, status_err,
- status_xfer);
+ dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
else if (status_err & (1 << i))
dwc_handle_error(dw, dwc);
- else if ((status_block | status_xfer) & (1 << i))
+ else if (status_xfer & (1 << i))
dwc_scan_descriptors(dw, dwc);
}
/*
- * Re-enable interrupts. Block Complete interrupts are only
- * enabled if the INT_EN bit in the descriptor is set. This
- * will trigger a scan before the whole list is done.
+ * Re-enable interrupts.
*/
channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
- channel_set_bit(dw, MASK.BLOCK, dw->all_chan_mask);
channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
}
* softirq handler.
*/
channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
- channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
status = dma_readl(dw, STATUS_INT);
/* Try to recover */
channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
- channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
unsigned long flags;
spin_lock_irqsave(&dwc->lock, flags);
- cookie = dwc_assign_cookie(dwc, desc);
+ cookie = dma_cookie_assign(tx);
/*
* REVISIT: We should attempt to chain as many descriptors as
else
src_width = dst_width = 0;
- ctllo = DWC_DEFAULT_CTLLO(chan->private)
+ ctllo = DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_DST_WIDTH(dst_width)
| DWC_CTLL_SRC_WIDTH(src_width)
| DWC_CTLL_DST_INC
static struct dma_async_tx_descriptor *
dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
- unsigned long flags)
+ unsigned long flags, void *context)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
struct dw_dma_slave *dws = chan->private;
+ struct dma_slave_config *sconfig = &dwc->dma_sconfig;
struct dw_desc *prev;
struct dw_desc *first;
u32 ctllo;
if (unlikely(!dws || !sg_len))
return NULL;
- reg_width = dws->reg_width;
prev = first = NULL;
switch (direction) {
case DMA_MEM_TO_DEV:
- ctllo = (DWC_DEFAULT_CTLLO(chan->private)
+ reg_width = __fls(sconfig->dst_addr_width);
+ reg = sconfig->dst_addr;
+ ctllo = (DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_DST_FIX
- | DWC_CTLL_SRC_INC
- | DWC_CTLL_FC(dws->fc));
- reg = dws->tx_reg;
+ | DWC_CTLL_SRC_INC);
+
+ ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
+ DWC_CTLL_FC(DW_DMA_FC_D_M2P);
+
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
u32 len, dlen, mem;
mem = sg_phys(sg);
len = sg_dma_len(sg);
- mem_width = 2;
- if (unlikely(mem & 3 || len & 3))
+
+ if (!((mem | len) & 7))
+ mem_width = 3;
+ else if (!((mem | len) & 3))
+ mem_width = 2;
+ else if (!((mem | len) & 1))
+ mem_width = 1;
+ else
mem_width = 0;
slave_sg_todev_fill_desc:
}
break;
case DMA_DEV_TO_MEM:
- ctllo = (DWC_DEFAULT_CTLLO(chan->private)
+ reg_width = __fls(sconfig->src_addr_width);
+ reg = sconfig->src_addr;
+ ctllo = (DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_INC
- | DWC_CTLL_SRC_FIX
- | DWC_CTLL_FC(dws->fc));
+ | DWC_CTLL_SRC_FIX);
+
+ ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
+ DWC_CTLL_FC(DW_DMA_FC_D_P2M);
- reg = dws->rx_reg;
for_each_sg(sgl, sg, sg_len, i) {
struct dw_desc *desc;
u32 len, dlen, mem;
mem = sg_phys(sg);
len = sg_dma_len(sg);
- mem_width = 2;
- if (unlikely(mem & 3 || len & 3))
+
+ if (!((mem | len) & 7))
+ mem_width = 3;
+ else if (!((mem | len) & 3))
+ mem_width = 2;
+ else if (!((mem | len) & 1))
+ mem_width = 1;
+ else
mem_width = 0;
slave_sg_fromdev_fill_desc:
return NULL;
}
+/*
+ * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
+ * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
+ *
+ * NOTE: burst size 2 is not supported by controller.
+ *
+ * This can be done by finding least significant bit set: n & (n - 1)
+ */
+static inline void convert_burst(u32 *maxburst)
+{
+ if (*maxburst > 1)
+ *maxburst = fls(*maxburst) - 2;
+ else
+ *maxburst = 0;
+}
+
+static int
+set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
+{
+ struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
+
+ /* Check if it is chan is configured for slave transfers */
+ if (!chan->private)
+ return -EINVAL;
+
+ memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
+
+ convert_burst(&dwc->dma_sconfig.src_maxburst);
+ convert_burst(&dwc->dma_sconfig.dst_maxburst);
+
+ return 0;
+}
+
static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
unsigned long arg)
{
/* Flush all pending and queued descriptors */
list_for_each_entry_safe(desc, _desc, &list, desc_node)
dwc_descriptor_complete(dwc, desc, false);
- } else
+ } else if (cmd == DMA_SLAVE_CONFIG) {
+ return set_runtime_config(chan, (struct dma_slave_config *)arg);
+ } else {
return -ENXIO;
+ }
return 0;
}
struct dma_tx_state *txstate)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
- dma_cookie_t last_used;
- dma_cookie_t last_complete;
- int ret;
-
- last_complete = dwc->completed;
- last_used = chan->cookie;
+ enum dma_status ret;
- ret = dma_async_is_complete(cookie, last_complete, last_used);
+ ret = dma_cookie_status(chan, cookie, txstate);
if (ret != DMA_SUCCESS) {
dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
- last_complete = dwc->completed;
- last_used = chan->cookie;
-
- ret = dma_async_is_complete(cookie, last_complete, last_used);
+ ret = dma_cookie_status(chan, cookie, txstate);
}
if (ret != DMA_SUCCESS)
- dma_set_tx_state(txstate, last_complete, last_used,
- dwc_first_active(dwc)->len);
- else
- dma_set_tx_state(txstate, last_complete, last_used, 0);
+ dma_set_residue(txstate, dwc_first_active(dwc)->len);
if (dwc->paused)
return DMA_PAUSED;
return -EIO;
}
- dwc->completed = chan->cookie = 1;
+ dma_cookie_init(chan);
/*
* NOTE: some controllers may have additional features that we
/* Disable interrupts */
channel_clear_bit(dw, MASK.XFER, dwc->mask);
- channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
channel_clear_bit(dw, MASK.ERROR, dwc->mask);
spin_unlock_irqrestore(&dwc->lock, flags);
return -EBUSY;
}
- dma_writel(dw, CLEAR.BLOCK, dwc->mask);
dma_writel(dw, CLEAR.ERROR, dwc->mask);
dma_writel(dw, CLEAR.XFER, dwc->mask);
enum dma_transfer_direction direction)
{
struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
+ struct dma_slave_config *sconfig = &dwc->dma_sconfig;
struct dw_cyclic_desc *cdesc;
struct dw_cyclic_desc *retval = NULL;
struct dw_desc *desc;
struct dw_desc *last = NULL;
- struct dw_dma_slave *dws = chan->private;
unsigned long was_cyclic;
unsigned int reg_width;
unsigned int periods;
}
retval = ERR_PTR(-EINVAL);
- reg_width = dws->reg_width;
+
+ if (direction == DMA_MEM_TO_DEV)
+ reg_width = __ffs(sconfig->dst_addr_width);
+ else
+ reg_width = __ffs(sconfig->src_addr_width);
+
periods = buf_len / period_len;
/* Check for too big/unaligned periods and unaligned DMA buffer. */
switch (direction) {
case DMA_MEM_TO_DEV:
- desc->lli.dar = dws->tx_reg;
+ desc->lli.dar = sconfig->dst_addr;
desc->lli.sar = buf_addr + (period_len * i);
- desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
+ desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_FIX
| DWC_CTLL_SRC_INC
- | DWC_CTLL_FC(dws->fc)
| DWC_CTLL_INT_EN);
+
+ desc->lli.ctllo |= sconfig->device_fc ?
+ DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
+ DWC_CTLL_FC(DW_DMA_FC_D_M2P);
+
break;
case DMA_DEV_TO_MEM:
desc->lli.dar = buf_addr + (period_len * i);
- desc->lli.sar = dws->rx_reg;
- desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan->private)
+ desc->lli.sar = sconfig->src_addr;
+ desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
| DWC_CTLL_SRC_WIDTH(reg_width)
| DWC_CTLL_DST_WIDTH(reg_width)
| DWC_CTLL_DST_INC
| DWC_CTLL_SRC_FIX
- | DWC_CTLL_FC(dws->fc)
| DWC_CTLL_INT_EN);
+
+ desc->lli.ctllo |= sconfig->device_fc ?
+ DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
+ DWC_CTLL_FC(DW_DMA_FC_D_P2M);
+
break;
default:
break;
while (dma_readl(dw, CH_EN) & dwc->mask)
cpu_relax();
- dma_writel(dw, CLEAR.BLOCK, dwc->mask);
dma_writel(dw, CLEAR.ERROR, dwc->mask);
dma_writel(dw, CLEAR.XFER, dwc->mask);
dma_writel(dw, CFG, 0);
channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
- channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
int err;
int i;
- pdata = pdev->dev.platform_data;
+ pdata = dev_get_platdata(&pdev->dev);
if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
return -EINVAL;
struct dw_dma_chan *dwc = &dw->chan[i];
dwc->chan.device = &dw->dma;
- dwc->chan.cookie = dwc->completed = 1;
+ dma_cookie_init(&dwc->chan);
if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
list_add_tail(&dwc->chan.device_node,
&dw->dma.channels);
/* 7 is highest priority & 0 is lowest. */
if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
- dwc->priority = 7 - i;
+ dwc->priority = pdata->nr_channels - i - 1;
else
dwc->priority = i;
/* Clear/disable all interrupts on all channels. */
dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
- dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
- channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
static const struct dev_pm_ops dw_dev_pm_ops = {
.suspend_noirq = dw_suspend_noirq,
.resume_noirq = dw_resume_noirq,
+ .freeze_noirq = dw_suspend_noirq,
+ .thaw_noirq = dw_resume_noirq,
+ .restore_noirq = dw_resume_noirq,
+ .poweroff_noirq = dw_suspend_noirq,
};
static struct platform_driver dw_driver = {
projects / ~andy / linux / blobdiff