+
+/*
+ * This DMA functionality is only compiled in if we have
+ * access to the generic DMA devices/DMA engine.
+ */
+#ifdef CONFIG_DMA_ENGINE
+static void unmap_free_dma_scatter(struct pl022 *pl022)
+{
+ /* Unmap and free the SG tables */
+ dma_unmap_sg(&pl022->adev->dev, pl022->sgt_tx.sgl,
+ pl022->sgt_tx.nents, DMA_TO_DEVICE);
+ dma_unmap_sg(&pl022->adev->dev, pl022->sgt_rx.sgl,
+ pl022->sgt_rx.nents, DMA_FROM_DEVICE);
+ sg_free_table(&pl022->sgt_rx);
+ sg_free_table(&pl022->sgt_tx);
+}
+
+static void dma_callback(void *data)
+{
+ struct pl022 *pl022 = data;
+ struct spi_message *msg = pl022->cur_msg;
+
+ BUG_ON(!pl022->sgt_rx.sgl);
+
+#ifdef VERBOSE_DEBUG
+ /*
+ * Optionally dump out buffers to inspect contents, this is
+ * good if you want to convince yourself that the loopback
+ * read/write contents are the same, when adopting to a new
+ * DMA engine.
+ */
+ {
+ struct scatterlist *sg;
+ unsigned int i;
+
+ dma_sync_sg_for_cpu(&pl022->adev->dev,
+ pl022->sgt_rx.sgl,
+ pl022->sgt_rx.nents,
+ DMA_FROM_DEVICE);
+
+ for_each_sg(pl022->sgt_rx.sgl, sg, pl022->sgt_rx.nents, i) {
+ dev_dbg(&pl022->adev->dev, "SPI RX SG ENTRY: %d", i);
+ print_hex_dump(KERN_ERR, "SPI RX: ",
+ DUMP_PREFIX_OFFSET,
+ 16,
+ 1,
+ sg_virt(sg),
+ sg_dma_len(sg),
+ 1);
+ }
+ for_each_sg(pl022->sgt_tx.sgl, sg, pl022->sgt_tx.nents, i) {
+ dev_dbg(&pl022->adev->dev, "SPI TX SG ENTRY: %d", i);
+ print_hex_dump(KERN_ERR, "SPI TX: ",
+ DUMP_PREFIX_OFFSET,
+ 16,
+ 1,
+ sg_virt(sg),
+ sg_dma_len(sg),
+ 1);
+ }
+ }
+#endif
+
+ unmap_free_dma_scatter(pl022);
+
+ /* Update total bytes transfered */
+ msg->actual_length += pl022->cur_transfer->len;
+ if (pl022->cur_transfer->cs_change)
+ pl022->cur_chip->
+ cs_control(SSP_CHIP_DESELECT);
+
+ /* Move to next transfer */
+ msg->state = next_transfer(pl022);
+ tasklet_schedule(&pl022->pump_transfers);
+}
+
+static void setup_dma_scatter(struct pl022 *pl022,
+ void *buffer,
+ unsigned int length,
+ struct sg_table *sgtab)
+{
+ struct scatterlist *sg;
+ int bytesleft = length;
+ void *bufp = buffer;
+ int mapbytes;
+ int i;
+
+ if (buffer) {
+ for_each_sg(sgtab->sgl, sg, sgtab->nents, i) {
+ /*
+ * If there are less bytes left than what fits
+ * in the current page (plus page alignment offset)
+ * we just feed in this, else we stuff in as much
+ * as we can.
+ */
+ if (bytesleft < (PAGE_SIZE - offset_in_page(bufp)))
+ mapbytes = bytesleft;
+ else
+ mapbytes = PAGE_SIZE - offset_in_page(bufp);
+ sg_set_page(sg, virt_to_page(bufp),
+ mapbytes, offset_in_page(bufp));
+ bufp += mapbytes;
+ bytesleft -= mapbytes;
+ dev_dbg(&pl022->adev->dev,
+ "set RX/TX target page @ %p, %d bytes, %d left\n",
+ bufp, mapbytes, bytesleft);
+ }
+ } else {
+ /* Map the dummy buffer on every page */
+ for_each_sg(sgtab->sgl, sg, sgtab->nents, i) {
+ if (bytesleft < PAGE_SIZE)
+ mapbytes = bytesleft;
+ else
+ mapbytes = PAGE_SIZE;
+ sg_set_page(sg, virt_to_page(pl022->dummypage),
+ mapbytes, 0);
+ bytesleft -= mapbytes;
+ dev_dbg(&pl022->adev->dev,
+ "set RX/TX to dummy page %d bytes, %d left\n",
+ mapbytes, bytesleft);
+
+ }
+ }
+ BUG_ON(bytesleft);
+}
+
+/**
+ * configure_dma - configures the channels for the next transfer
+ * @pl022: SSP driver's private data structure
+ */
+static int configure_dma(struct pl022 *pl022)
+{
+ struct dma_slave_config rx_conf = {
+ .src_addr = SSP_DR(pl022->phybase),
+ .direction = DMA_FROM_DEVICE,
+ .src_maxburst = pl022->vendor->fifodepth >> 1,
+ };
+ struct dma_slave_config tx_conf = {
+ .dst_addr = SSP_DR(pl022->phybase),
+ .direction = DMA_TO_DEVICE,
+ .dst_maxburst = pl022->vendor->fifodepth >> 1,
+ };
+ unsigned int pages;
+ int ret;
+ int sglen;
+ struct dma_chan *rxchan = pl022->dma_rx_channel;
+ struct dma_chan *txchan = pl022->dma_tx_channel;
+ struct dma_async_tx_descriptor *rxdesc;
+ struct dma_async_tx_descriptor *txdesc;
+ dma_cookie_t cookie;
+
+ /* Check that the channels are available */
+ if (!rxchan || !txchan)
+ return -ENODEV;
+
+ switch (pl022->read) {
+ case READING_NULL:
+ /* Use the same as for writing */
+ rx_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
+ break;
+ case READING_U8:
+ rx_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+ break;
+ case READING_U16:
+ rx_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+ break;
+ case READING_U32:
+ rx_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ break;
+ }
+
+ switch (pl022->write) {
+ case WRITING_NULL:
+ /* Use the same as for reading */
+ tx_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
+ break;
+ case WRITING_U8:
+ tx_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+ break;
+ case WRITING_U16:
+ tx_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+ break;
+ case WRITING_U32:
+ tx_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;;
+ break;
+ }
+
+ /* SPI pecularity: we need to read and write the same width */
+ if (rx_conf.src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
+ rx_conf.src_addr_width = tx_conf.dst_addr_width;
+ if (tx_conf.dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
+ tx_conf.dst_addr_width = rx_conf.src_addr_width;
+ BUG_ON(rx_conf.src_addr_width != tx_conf.dst_addr_width);
+
+ rxchan->device->device_control(rxchan, DMA_SLAVE_CONFIG,
+ (unsigned long) &rx_conf);
+ txchan->device->device_control(txchan, DMA_SLAVE_CONFIG,
+ (unsigned long) &tx_conf);
+
+ /* Create sglists for the transfers */
+ pages = (pl022->cur_transfer->len >> PAGE_SHIFT) + 1;
+ dev_dbg(&pl022->adev->dev, "using %d pages for transfer\n", pages);
+
+ ret = sg_alloc_table(&pl022->sgt_rx, pages, GFP_KERNEL);
+ if (ret)
+ goto err_alloc_rx_sg;
+
+ ret = sg_alloc_table(&pl022->sgt_tx, pages, GFP_KERNEL);
+ if (ret)
+ goto err_alloc_tx_sg;
+
+ /* Fill in the scatterlists for the RX+TX buffers */
+ setup_dma_scatter(pl022, pl022->rx,
+ pl022->cur_transfer->len, &pl022->sgt_rx);
+ setup_dma_scatter(pl022, pl022->tx,
+ pl022->cur_transfer->len, &pl022->sgt_tx);
+
+ /* Map DMA buffers */
+ sglen = dma_map_sg(&pl022->adev->dev, pl022->sgt_rx.sgl,
+ pl022->sgt_rx.nents, DMA_FROM_DEVICE);
+ if (!sglen)
+ goto err_rx_sgmap;
+
+ sglen = dma_map_sg(&pl022->adev->dev, pl022->sgt_tx.sgl,
+ pl022->sgt_tx.nents, DMA_TO_DEVICE);
+ if (!sglen)
+ goto err_tx_sgmap;
+
+ /* Send both scatterlists */
+ rxdesc = rxchan->device->device_prep_slave_sg(rxchan,
+ pl022->sgt_rx.sgl,
+ pl022->sgt_rx.nents,
+ DMA_FROM_DEVICE,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!rxdesc)
+ goto err_rxdesc;
+
+ txdesc = txchan->device->device_prep_slave_sg(txchan,
+ pl022->sgt_tx.sgl,
+ pl022->sgt_tx.nents,
+ DMA_TO_DEVICE,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!txdesc)
+ goto err_txdesc;
+
+ /* Put the callback on the RX transfer only, that should finish last */
+ rxdesc->callback = dma_callback;
+ rxdesc->callback_param = pl022;
+
+ /* Submit and fire RX and TX with TX last so we're ready to read! */
+ cookie = rxdesc->tx_submit(rxdesc);
+ if (dma_submit_error(cookie))
+ goto err_submit_rx;
+ cookie = txdesc->tx_submit(txdesc);
+ if (dma_submit_error(cookie))
+ goto err_submit_tx;
+ rxchan->device->device_issue_pending(rxchan);
+ txchan->device->device_issue_pending(txchan);
+
+ return 0;
+
+err_submit_tx:
+err_submit_rx:
+err_txdesc:
+ txchan->device->device_control(txchan, DMA_TERMINATE_ALL, 0);
+err_rxdesc:
+ rxchan->device->device_control(rxchan, DMA_TERMINATE_ALL, 0);
+ dma_unmap_sg(&pl022->adev->dev, pl022->sgt_tx.sgl,
+ pl022->sgt_tx.nents, DMA_TO_DEVICE);
+err_tx_sgmap:
+ dma_unmap_sg(&pl022->adev->dev, pl022->sgt_rx.sgl,
+ pl022->sgt_tx.nents, DMA_FROM_DEVICE);
+err_rx_sgmap:
+ sg_free_table(&pl022->sgt_tx);
+err_alloc_tx_sg:
+ sg_free_table(&pl022->sgt_rx);
+err_alloc_rx_sg:
+ return -ENOMEM;
+}
+
+static int __init pl022_dma_probe(struct pl022 *pl022)
+{
+ dma_cap_mask_t mask;
+
+ /* Try to acquire a generic DMA engine slave channel */
+ dma_cap_zero(mask);
+ dma_cap_set(DMA_SLAVE, mask);
+ /*
+ * We need both RX and TX channels to do DMA, else do none
+ * of them.
+ */
+ pl022->dma_rx_channel = dma_request_channel(mask,
+ pl022->master_info->dma_filter,
+ pl022->master_info->dma_rx_param);
+ if (!pl022->dma_rx_channel) {
+ dev_err(&pl022->adev->dev, "no RX DMA channel!\n");
+ goto err_no_rxchan;
+ }
+
+ pl022->dma_tx_channel = dma_request_channel(mask,
+ pl022->master_info->dma_filter,
+ pl022->master_info->dma_tx_param);
+ if (!pl022->dma_tx_channel) {
+ dev_err(&pl022->adev->dev, "no TX DMA channel!\n");
+ goto err_no_txchan;
+ }
+
+ pl022->dummypage = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!pl022->dummypage) {
+ dev_err(&pl022->adev->dev, "no DMA dummypage!\n");
+ goto err_no_dummypage;
+ }
+
+ dev_info(&pl022->adev->dev, "setup for DMA on RX %s, TX %s\n",
+ dma_chan_name(pl022->dma_rx_channel),
+ dma_chan_name(pl022->dma_tx_channel));
+
+ return 0;
+
+err_no_dummypage:
+ dma_release_channel(pl022->dma_tx_channel);
+err_no_txchan:
+ dma_release_channel(pl022->dma_rx_channel);
+ pl022->dma_rx_channel = NULL;
+err_no_rxchan:
+ return -ENODEV;
+}
+
+static void terminate_dma(struct pl022 *pl022)
+{
+ struct dma_chan *rxchan = pl022->dma_rx_channel;
+ struct dma_chan *txchan = pl022->dma_tx_channel;
+
+ rxchan->device->device_control(rxchan, DMA_TERMINATE_ALL, 0);
+ txchan->device->device_control(txchan, DMA_TERMINATE_ALL, 0);
+ unmap_free_dma_scatter(pl022);
+}
+
+static void pl022_dma_remove(struct pl022 *pl022)
+{
+ if (pl022->busy)
+ terminate_dma(pl022);
+ if (pl022->dma_tx_channel)
+ dma_release_channel(pl022->dma_tx_channel);
+ if (pl022->dma_rx_channel)
+ dma_release_channel(pl022->dma_rx_channel);
+ kfree(pl022->dummypage);
+}
+
+#else
+static inline int configure_dma(struct pl022 *pl022)
+{
+ return -ENODEV;
+}
+
+static inline int pl022_dma_probe(struct pl022 *pl022)
+{
+ return 0;
+}
+
+static inline void pl022_dma_remove(struct pl022 *pl022)
+{
+}
+#endif
+