X-Git-Url: https://git.karo-electronics.de/?a=blobdiff_plain;f=drivers%2Fdma%2Fshdma.c;h=2a638f9f09a25d5ed37411d7a997172b928491ce;hb=d205df9955feb9c2ae63ac1387804e225ff22605;hp=d50da41ac328c6fdf82484371232db31a701febf;hpb=11ad2f52826ac6d58d6780d3d8a3e098c88d9142;p=karo-tx-linux.git diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index d50da41ac328..2a638f9f09a2 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -48,7 +48,7 @@ enum sh_dmae_desc_status { /* * Used for write-side mutual exclusion for the global device list, - * read-side synchronization by way of RCU. + * read-side synchronization by way of RCU, and per-controller data. */ static DEFINE_SPINLOCK(sh_dmae_lock); static LIST_HEAD(sh_dmae_devices); @@ -85,22 +85,35 @@ static void dmaor_write(struct sh_dmae_device *shdev, u16 data) */ static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) { - unsigned short dmaor = dmaor_read(shdev); + unsigned short dmaor; + unsigned long flags; + + spin_lock_irqsave(&sh_dmae_lock, flags); + dmaor = dmaor_read(shdev); dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); + + spin_unlock_irqrestore(&sh_dmae_lock, flags); } static int sh_dmae_rst(struct sh_dmae_device *shdev) { unsigned short dmaor; + unsigned long flags; - sh_dmae_ctl_stop(shdev); - dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init; + spin_lock_irqsave(&sh_dmae_lock, flags); - dmaor_write(shdev, dmaor); - if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) { - pr_warning("dma-sh: Can't initialize DMAOR.\n"); - return -EINVAL; + dmaor = dmaor_read(shdev) & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME); + + dmaor_write(shdev, dmaor | shdev->pdata->dmaor_init); + + dmaor = dmaor_read(shdev); + + spin_unlock_irqrestore(&sh_dmae_lock, flags); + + if (dmaor & (DMAOR_AE | DMAOR_NMIF)) { + dev_warn(shdev->common.dev, "Can't initialize DMAOR.\n"); + return -EIO; } return 0; } @@ -184,7 +197,7 @@ static void dmae_init(struct sh_dmae_chan *sh_chan) static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) { - /* When DMA was working, can not set data to CHCR */ + /* If DMA is active, cannot set CHCR. TODO: remove this superfluous check */ if (dmae_is_busy(sh_chan)) return -EBUSY; @@ -200,12 +213,17 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) struct sh_dmae_device, common); struct sh_dmae_pdata *pdata = shdev->pdata; const struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; - u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16); + u16 __iomem *addr = shdev->dmars; int shift = chan_pdata->dmars_bit; if (dmae_is_busy(sh_chan)) return -EBUSY; + /* in the case of a missing DMARS resource use first memory window */ + if (!addr) + addr = (u16 __iomem *)shdev->chan_reg; + addr += chan_pdata->dmars / sizeof(u16); + __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), addr); @@ -325,7 +343,7 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) dmae_set_dmars(sh_chan, cfg->mid_rid); dmae_set_chcr(sh_chan, cfg->chcr); - } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) { + } else { dmae_init(sh_chan); } @@ -374,7 +392,12 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) LIST_HEAD(list); int descs = sh_chan->descs_allocated; + /* Protect against ISR */ + spin_lock_irq(&sh_chan->desc_lock); dmae_halt(sh_chan); + spin_unlock_irq(&sh_chan->desc_lock); + + /* Now no new interrupts will occur */ /* Prepared and not submitted descriptors can still be on the queue */ if (!list_empty(&sh_chan->ld_queue)) @@ -384,6 +407,7 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) /* The caller is holding dma_list_mutex */ struct sh_dmae_slave *param = chan->private; clear_bit(param->slave_id, sh_dmae_slave_used); + chan->private = NULL; } spin_lock_bh(&sh_chan->desc_lock); @@ -563,8 +587,6 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( if (!chan || !len) return NULL; - chan->private = NULL; - sh_chan = to_sh_chan(chan); sg_init_table(&sg, 1); @@ -620,9 +642,9 @@ static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, if (!chan) return -EINVAL; + spin_lock_bh(&sh_chan->desc_lock); dmae_halt(sh_chan); - spin_lock_bh(&sh_chan->desc_lock); if (!list_empty(&sh_chan->ld_queue)) { /* Record partial transfer */ struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, @@ -716,6 +738,14 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all list_move(&desc->node, &sh_chan->ld_free); } } + + if (all && !callback) + /* + * Terminating and the loop completed normally: forgive + * uncompleted cookies + */ + sh_chan->completed_cookie = sh_chan->common.cookie; + spin_unlock_bh(&sh_chan->desc_lock); if (callback) @@ -733,10 +763,6 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) { while (__ld_cleanup(sh_chan, all)) ; - - if (all) - /* Terminating - forgive uncompleted cookies */ - sh_chan->completed_cookie = sh_chan->common.cookie; } static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) @@ -782,8 +808,10 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, sh_dmae_chan_ld_cleanup(sh_chan, false); - last_used = chan->cookie; + /* First read completed cookie to avoid a skew */ last_complete = sh_chan->completed_cookie; + rmb(); + last_used = chan->cookie; BUG_ON(last_complete < 0); dma_set_tx_state(txstate, last_complete, last_used, 0); @@ -813,8 +841,12 @@ static enum dma_status sh_dmae_tx_status(struct dma_chan *chan, static irqreturn_t sh_dmae_interrupt(int irq, void *data) { irqreturn_t ret = IRQ_NONE; - struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; - u32 chcr = sh_dmae_readl(sh_chan, CHCR); + struct sh_dmae_chan *sh_chan = data; + u32 chcr; + + spin_lock(&sh_chan->desc_lock); + + chcr = sh_dmae_readl(sh_chan, CHCR); if (chcr & CHCR_TE) { /* DMA stop */ @@ -824,10 +856,13 @@ static irqreturn_t sh_dmae_interrupt(int irq, void *data) tasklet_schedule(&sh_chan->tasklet); } + spin_unlock(&sh_chan->desc_lock); + return ret; } -static unsigned int sh_dmae_reset(struct sh_dmae_device *shdev) +/* Called from error IRQ or NMI */ +static bool sh_dmae_reset(struct sh_dmae_device *shdev) { unsigned int handled = 0; int i; @@ -839,22 +874,32 @@ static unsigned int sh_dmae_reset(struct sh_dmae_device *shdev) for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { struct sh_dmae_chan *sh_chan = shdev->chan[i]; struct sh_desc *desc; + LIST_HEAD(dl); if (!sh_chan) continue; + spin_lock(&sh_chan->desc_lock); + /* Stop the channel */ dmae_halt(sh_chan); + list_splice_init(&sh_chan->ld_queue, &dl); + + spin_unlock(&sh_chan->desc_lock); + /* Complete all */ - list_for_each_entry(desc, &sh_chan->ld_queue, node) { + list_for_each_entry(desc, &dl, node) { struct dma_async_tx_descriptor *tx = &desc->async_tx; desc->mark = DESC_IDLE; if (tx->callback) tx->callback(tx->callback_param); } - list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free); + spin_lock(&sh_chan->desc_lock); + list_splice(&dl, &sh_chan->ld_free); + spin_unlock(&sh_chan->desc_lock); + handled++; } @@ -867,10 +912,11 @@ static irqreturn_t sh_dmae_err(int irq, void *data) { struct sh_dmae_device *shdev = data; - if (dmaor_read(shdev) & DMAOR_AE) - return IRQ_RETVAL(sh_dmae_reset(data)); - else + if (!(dmaor_read(shdev) & DMAOR_AE)) return IRQ_NONE; + + sh_dmae_reset(data); + return IRQ_HANDLED; } static void dmae_do_tasklet(unsigned long data) @@ -902,17 +948,11 @@ static void dmae_do_tasklet(unsigned long data) static bool sh_dmae_nmi_notify(struct sh_dmae_device *shdev) { - unsigned int handled; - /* Fast path out if NMIF is not asserted for this controller */ if ((dmaor_read(shdev) & DMAOR_NMIF) == 0) return false; - handled = sh_dmae_reset(shdev); - if (handled) - return true; - - return false; + return sh_dmae_reset(shdev); } static int sh_dmae_nmi_handler(struct notifier_block *self, @@ -982,9 +1022,6 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, (unsigned long)new_sh_chan); - /* Init the channel */ - dmae_init(new_sh_chan); - spin_lock_init(&new_sh_chan->desc_lock); /* Init descripter manage list */ @@ -1045,9 +1082,8 @@ static int __init sh_dmae_probe(struct platform_device *pdev) struct sh_dmae_pdata *pdata = pdev->dev.platform_data; unsigned long irqflags = IRQF_DISABLED, chan_flag[SH_DMAC_MAX_CHANNELS] = {}; - unsigned long flags; int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; - int err, i, irq_cnt = 0, irqres = 0; + int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0; struct sh_dmae_device *shdev; struct resource *chan, *dmars, *errirq_res, *chanirq_res; @@ -1056,7 +1092,7 @@ static int __init sh_dmae_probe(struct platform_device *pdev) return -ENODEV; chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); - /* DMARS area is optional, if absent, this controller cannot do slave DMA */ + /* DMARS area is optional */ dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); /* * IRQ resources: @@ -1108,14 +1144,16 @@ static int __init sh_dmae_probe(struct platform_device *pdev) /* platform data */ shdev->pdata = pdata; + platform_set_drvdata(pdev, shdev); + pm_runtime_enable(&pdev->dev); pm_runtime_get_sync(&pdev->dev); - spin_lock_irqsave(&sh_dmae_lock, flags); + spin_lock_irq(&sh_dmae_lock); list_add_tail_rcu(&shdev->node, &sh_dmae_devices); - spin_unlock_irqrestore(&sh_dmae_lock, flags); + spin_unlock_irq(&sh_dmae_lock); - /* reset dma controller */ + /* reset dma controller - only needed as a test */ err = sh_dmae_rst(shdev); if (err) goto rst_err; @@ -1123,7 +1161,7 @@ static int __init sh_dmae_probe(struct platform_device *pdev) INIT_LIST_HEAD(&shdev->common.channels); dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); - if (dmars) + if (pdata->slave && pdata->slave_num) dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); shdev->common.device_alloc_chan_resources @@ -1172,8 +1210,13 @@ static int __init sh_dmae_probe(struct platform_device *pdev) !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { /* Special case - all multiplexed */ for (; irq_cnt < pdata->channel_num; irq_cnt++) { - chan_irq[irq_cnt] = chanirq_res->start; - chan_flag[irq_cnt] = IRQF_SHARED; + if (irq_cnt < SH_DMAC_MAX_CHANNELS) { + chan_irq[irq_cnt] = chanirq_res->start; + chan_flag[irq_cnt] = IRQF_SHARED; + } else { + irq_cap = 1; + break; + } } } else { do { @@ -1187,46 +1230,60 @@ static int __init sh_dmae_probe(struct platform_device *pdev) "Found IRQ %d for channel %d\n", i, irq_cnt); chan_irq[irq_cnt++] = i; + + if (irq_cnt >= SH_DMAC_MAX_CHANNELS) + break; + } + + if (irq_cnt >= SH_DMAC_MAX_CHANNELS) { + irq_cap = 1; + break; } chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, ++irqres); } while (irq_cnt < pdata->channel_num && chanirq_res); } - if (irq_cnt < pdata->channel_num) - goto eirqres; - /* Create DMA Channel */ - for (i = 0; i < pdata->channel_num; i++) { + for (i = 0; i < irq_cnt; i++) { err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); if (err) goto chan_probe_err; } + if (irq_cap) + dev_notice(&pdev->dev, "Attempting to register %d DMA " + "channels when a maximum of %d are supported.\n", + pdata->channel_num, SH_DMAC_MAX_CHANNELS); + pm_runtime_put(&pdev->dev); - platform_set_drvdata(pdev, shdev); dma_async_device_register(&shdev->common); return err; chan_probe_err: sh_dmae_chan_remove(shdev); -eirqres: + #if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARCH_SHMOBILE) free_irq(errirq, shdev); eirq_err: #endif rst_err: - spin_lock_irqsave(&sh_dmae_lock, flags); + spin_lock_irq(&sh_dmae_lock); list_del_rcu(&shdev->node); - spin_unlock_irqrestore(&sh_dmae_lock, flags); + spin_unlock_irq(&sh_dmae_lock); pm_runtime_put(&pdev->dev); + pm_runtime_disable(&pdev->dev); + if (dmars) iounmap(shdev->dmars); + + platform_set_drvdata(pdev, NULL); emapdmars: iounmap(shdev->chan_reg); + synchronize_rcu(); emapchan: kfree(shdev); ealloc: @@ -1242,7 +1299,6 @@ static int __exit sh_dmae_remove(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); struct resource *res; - unsigned long flags; int errirq = platform_get_irq(pdev, 0); dma_async_device_unregister(&shdev->common); @@ -1250,9 +1306,9 @@ static int __exit sh_dmae_remove(struct platform_device *pdev) if (errirq > 0) free_irq(errirq, shdev); - spin_lock_irqsave(&sh_dmae_lock, flags); + spin_lock_irq(&sh_dmae_lock); list_del_rcu(&shdev->node); - spin_unlock_irqrestore(&sh_dmae_lock, flags); + spin_unlock_irq(&sh_dmae_lock); /* channel data remove */ sh_dmae_chan_remove(shdev); @@ -1263,6 +1319,9 @@ static int __exit sh_dmae_remove(struct platform_device *pdev) iounmap(shdev->dmars); iounmap(shdev->chan_reg); + platform_set_drvdata(pdev, NULL); + + synchronize_rcu(); kfree(shdev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -1281,12 +1340,78 @@ static void sh_dmae_shutdown(struct platform_device *pdev) sh_dmae_ctl_stop(shdev); } +static int sh_dmae_runtime_suspend(struct device *dev) +{ + return 0; +} + +static int sh_dmae_runtime_resume(struct device *dev) +{ + struct sh_dmae_device *shdev = dev_get_drvdata(dev); + + return sh_dmae_rst(shdev); +} + +#ifdef CONFIG_PM +static int sh_dmae_suspend(struct device *dev) +{ + struct sh_dmae_device *shdev = dev_get_drvdata(dev); + int i; + + for (i = 0; i < shdev->pdata->channel_num; i++) { + struct sh_dmae_chan *sh_chan = shdev->chan[i]; + if (sh_chan->descs_allocated) + sh_chan->pm_error = pm_runtime_put_sync(dev); + } + + return 0; +} + +static int sh_dmae_resume(struct device *dev) +{ + struct sh_dmae_device *shdev = dev_get_drvdata(dev); + int i; + + for (i = 0; i < shdev->pdata->channel_num; i++) { + struct sh_dmae_chan *sh_chan = shdev->chan[i]; + struct sh_dmae_slave *param = sh_chan->common.private; + + if (!sh_chan->descs_allocated) + continue; + + if (!sh_chan->pm_error) + pm_runtime_get_sync(dev); + + if (param) { + const struct sh_dmae_slave_config *cfg = param->config; + dmae_set_dmars(sh_chan, cfg->mid_rid); + dmae_set_chcr(sh_chan, cfg->chcr); + } else { + dmae_init(sh_chan); + } + } + + return 0; +} +#else +#define sh_dmae_suspend NULL +#define sh_dmae_resume NULL +#endif + +const struct dev_pm_ops sh_dmae_pm = { + .suspend = sh_dmae_suspend, + .resume = sh_dmae_resume, + .runtime_suspend = sh_dmae_runtime_suspend, + .runtime_resume = sh_dmae_runtime_resume, +}; + static struct platform_driver sh_dmae_driver = { .remove = __exit_p(sh_dmae_remove), .shutdown = sh_dmae_shutdown, .driver = { .owner = THIS_MODULE, .name = "sh-dma-engine", + .pm = &sh_dmae_pm, }, };