drivers/dma/pl330.c

   1 /* linux/drivers/dma/pl330.c
   2  *
   3  * Copyright (C) 2010 Samsung Electronics Co. Ltd.
   4  *      Jaswinder Singh <jassi.brar@samsung.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <linux/io.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/module.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/amba/bus.h>
  19 #include <linux/amba/pl330.h>
  20 #include <linux/pm_runtime.h>
  21 #include <linux/scatterlist.h>
  22 #include <linux/of.h>
  23
  24 #include "dmaengine.h"
  25
  26 #define NR_DEFAULT_DESC 16
  27
  28 enum desc_status {
  29         /* In the DMAC pool */
  30         FREE,
  31         /*
  32          * Allocted to some channel during prep_xxx
  33          * Also may be sitting on the work_list.
  34          */
  35         PREP,
  36         /*
  37          * Sitting on the work_list and already submitted
  38          * to the PL330 core. Not more than two descriptors
  39          * of a channel can be BUSY at any time.
  40          */
  41         BUSY,
  42         /*
  43          * Sitting on the channel work_list but xfer done
  44          * by PL330 core
  45          */
  46         DONE,
  47 };
  48
  49 struct dma_pl330_chan {
  50         /* Schedule desc completion */
  51         struct tasklet_struct task;
  52
  53         /* DMA-Engine Channel */
  54         struct dma_chan chan;
  55
  56         /* List of to be xfered descriptors */
  57         struct list_head work_list;
  58
  59         /* Pointer to the DMAC that manages this channel,
  60          * NULL if the channel is available to be acquired.
  61          * As the parent, this DMAC also provides descriptors
  62          * to the channel.
  63          */
  64         struct dma_pl330_dmac *dmac;
  65
  66         /* To protect channel manipulation */
  67         spinlock_t lock;
  68
  69         /* Token of a hardware channel thread of PL330 DMAC
  70          * NULL if the channel is available to be acquired.
  71          */
  72         void *pl330_chid;
  73
  74         /* For D-to-M and M-to-D channels */
  75         int burst_sz; /* the peripheral fifo width */
  76         int burst_len; /* the number of burst */
  77         dma_addr_t fifo_addr;
  78
  79         /* for cyclic capability */
  80         bool cyclic;
  81 };
  82
  83 struct dma_pl330_dmac {
  84         struct pl330_info pif;
  85
  86         /* DMA-Engine Device */
  87         struct dma_device ddma;
  88
  89         /* Pool of descriptors available for the DMAC's channels */
  90         struct list_head desc_pool;
  91         /* To protect desc_pool manipulation */
  92         spinlock_t pool_lock;
  93
  94         /* Peripheral channels connected to this DMAC */
  95         struct dma_pl330_chan *peripherals; /* keep at end */
  96
  97         struct clk *clk;
  98 };
  99
 100 struct dma_pl330_desc {
 101         /* To attach to a queue as child */
 102         struct list_head node;
 103
 104         /* Descriptor for the DMA Engine API */
 105         struct dma_async_tx_descriptor txd;
 106
 107         /* Xfer for PL330 core */
 108         struct pl330_xfer px;
 109
 110         struct pl330_reqcfg rqcfg;
 111         struct pl330_req req;
 112
 113         enum desc_status status;
 114
 115         /* The channel which currently holds this desc */
 116         struct dma_pl330_chan *pchan;
 117 };
 118
 119 /* forward declaration */
 120 static struct amba_driver pl330_driver;
 121
 122 static inline struct dma_pl330_chan *
 123 to_pchan(struct dma_chan *ch)
 124 {
 125         if (!ch)
 126                 return NULL;
 127
 128         return container_of(ch, struct dma_pl330_chan, chan);
 129 }
 130
 131 static inline struct dma_pl330_desc *
 132 to_desc(struct dma_async_tx_descriptor *tx)
 133 {
 134         return container_of(tx, struct dma_pl330_desc, txd);
 135 }
 136
 137 static inline void free_desc_list(struct list_head *list)
 138 {
 139         struct dma_pl330_dmac *pdmac;
 140         struct dma_pl330_desc *desc;
 141         struct dma_pl330_chan *pch;
 142         unsigned long flags;
 143
 144         if (list_empty(list))
 145                 return;
 146
 147         /* Finish off the work list */
 148         list_for_each_entry(desc, list, node) {
 149                 dma_async_tx_callback callback;
 150                 void *param;
 151
 152                 /* All desc in a list belong to same channel */
 153                 pch = desc->pchan;
 154                 callback = desc->txd.callback;
 155                 param = desc->txd.callback_param;
 156
 157                 if (callback)
 158                         callback(param);
 159
 160                 desc->pchan = NULL;
 161         }
 162
 163         pdmac = pch->dmac;
 164
 165         spin_lock_irqsave(&pdmac->pool_lock, flags);
 166         list_splice_tail_init(list, &pdmac->desc_pool);
 167         spin_unlock_irqrestore(&pdmac->pool_lock, flags);
 168 }
 169
 170 static inline void handle_cyclic_desc_list(struct list_head *list)
 171 {
 172         struct dma_pl330_desc *desc;
 173         struct dma_pl330_chan *pch;
 174         unsigned long flags;
 175
 176         if (list_empty(list))
 177                 return;
 178
 179         list_for_each_entry(desc, list, node) {
 180                 dma_async_tx_callback callback;
 181
 182                 /* Change status to reload it */
 183                 desc->status = PREP;
 184                 pch = desc->pchan;
 185                 callback = desc->txd.callback;
 186                 if (callback)
 187                         callback(desc->txd.callback_param);
 188         }
 189
 190         spin_lock_irqsave(&pch->lock, flags);
 191         list_splice_tail_init(list, &pch->work_list);
 192         spin_unlock_irqrestore(&pch->lock, flags);
 193 }
 194
 195 static inline void fill_queue(struct dma_pl330_chan *pch)
 196 {
 197         struct dma_pl330_desc *desc;
 198         int ret;
 199
 200         list_for_each_entry(desc, &pch->work_list, node) {
 201
 202                 /* If already submitted */
 203                 if (desc->status == BUSY)
 204                         break;
 205
 206                 ret = pl330_submit_req(pch->pl330_chid,
 207                                                 &desc->req);
 208                 if (!ret) {
 209                         desc->status = BUSY;
 210                         break;
 211                 } else if (ret == -EAGAIN) {
 212                         /* QFull or DMAC Dying */
 213                         break;
 214                 } else {
 215                         /* Unacceptable request */
 216                         desc->status = DONE;
 217                         dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
 218                                         __func__, __LINE__, desc->txd.cookie);
 219                         tasklet_schedule(&pch->task);
 220                 }
 221         }
 222 }
 223
 224 static void pl330_tasklet(unsigned long data)
 225 {
 226         struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
 227         struct dma_pl330_desc *desc, *_dt;
 228         unsigned long flags;
 229         LIST_HEAD(list);
 230
 231         spin_lock_irqsave(&pch->lock, flags);
 232
 233         /* Pick up ripe tomatoes */
 234         list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
 235                 if (desc->status == DONE) {
 236                         pch->chan.completed_cookie = desc->txd.cookie;
 237                         list_move_tail(&desc->node, &list);
 238                 }
 239
 240         /* Try to submit a req imm. next to the last completed cookie */
 241         fill_queue(pch);
 242
 243         /* Make sure the PL330 Channel thread is active */
 244         pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
 245
 246         spin_unlock_irqrestore(&pch->lock, flags);
 247
 248         if (pch->cyclic)
 249                 handle_cyclic_desc_list(&list);
 250         else
 251                 free_desc_list(&list);
 252 }
 253
 254 static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
 255 {
 256         struct dma_pl330_desc *desc = token;
 257         struct dma_pl330_chan *pch = desc->pchan;
 258         unsigned long flags;
 259
 260         /* If desc aborted */
 261         if (!pch)
 262                 return;
 263
 264         spin_lock_irqsave(&pch->lock, flags);
 265
 266         desc->status = DONE;
 267
 268         spin_unlock_irqrestore(&pch->lock, flags);
 269
 270         tasklet_schedule(&pch->task);
 271 }
 272
 273 bool pl330_filter(struct dma_chan *chan, void *param)
 274 {
 275         u8 *peri_id;
 276
 277         if (chan->device->dev->driver != &pl330_driver.drv)
 278                 return false;
 279
 280 #ifdef CONFIG_OF
 281         if (chan->device->dev->of_node) {
 282                 const __be32 *prop_value;
 283                 phandle phandle;
 284                 struct device_node *node;
 285
 286                 prop_value = ((struct property *)param)->value;
 287                 phandle = be32_to_cpup(prop_value++);
 288                 node = of_find_node_by_phandle(phandle);
 289                 return ((chan->private == node) &&
 290                                 (chan->chan_id == be32_to_cpup(prop_value)));
 291         }
 292 #endif
 293
 294         peri_id = chan->private;
 295         return *peri_id == (unsigned)param;
 296 }
 297 EXPORT_SYMBOL(pl330_filter);
 298
 299 static int pl330_alloc_chan_resources(struct dma_chan *chan)
 300 {
 301         struct dma_pl330_chan *pch = to_pchan(chan);
 302         struct dma_pl330_dmac *pdmac = pch->dmac;
 303         unsigned long flags;
 304
 305         spin_lock_irqsave(&pch->lock, flags);
 306
 307         chan->completed_cookie = chan->cookie = 1;
 308         pch->cyclic = false;
 309
 310         pch->pl330_chid = pl330_request_channel(&pdmac->pif);
 311         if (!pch->pl330_chid) {
 312                 spin_unlock_irqrestore(&pch->lock, flags);
 313                 return 0;
 314         }
 315
 316         tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
 317
 318         spin_unlock_irqrestore(&pch->lock, flags);
 319
 320         return 1;
 321 }
 322
 323 static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
 324 {
 325         struct dma_pl330_chan *pch = to_pchan(chan);
 326         struct dma_pl330_desc *desc, *_dt;
 327         unsigned long flags;
 328         struct dma_pl330_dmac *pdmac = pch->dmac;
 329         struct dma_slave_config *slave_config;
 330         LIST_HEAD(list);
 331
 332         switch (cmd) {
 333         case DMA_TERMINATE_ALL:
 334                 spin_lock_irqsave(&pch->lock, flags);
 335
 336                 /* FLUSH the PL330 Channel thread */
 337                 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
 338
 339                 /* Mark all desc done */
 340                 list_for_each_entry_safe(desc, _dt, &pch->work_list , node) {
 341                         desc->status = DONE;
 342                         pch->completed = desc->txd.cookie;
 343                         list_move_tail(&desc->node, &list);
 344                 }
 345
 346                 list_splice_tail_init(&list, &pdmac->desc_pool);
 347                 spin_unlock_irqrestore(&pch->lock, flags);
 348                 break;
 349         case DMA_SLAVE_CONFIG:
 350                 slave_config = (struct dma_slave_config *)arg;
 351
 352                 if (slave_config->direction == DMA_MEM_TO_DEV) {
 353                         if (slave_config->dst_addr)
 354                                 pch->fifo_addr = slave_config->dst_addr;
 355                         if (slave_config->dst_addr_width)
 356                                 pch->burst_sz = __ffs(slave_config->dst_addr_width);
 357                         if (slave_config->dst_maxburst)
 358                                 pch->burst_len = slave_config->dst_maxburst;
 359                 } else if (slave_config->direction == DMA_DEV_TO_MEM) {
 360                         if (slave_config->src_addr)
 361                                 pch->fifo_addr = slave_config->src_addr;
 362                         if (slave_config->src_addr_width)
 363                                 pch->burst_sz = __ffs(slave_config->src_addr_width);
 364                         if (slave_config->src_maxburst)
 365                                 pch->burst_len = slave_config->src_maxburst;
 366                 }
 367                 break;
 368         default:
 369                 dev_err(pch->dmac->pif.dev, "Not supported command.\n");
 370                 return -ENXIO;
 371         }
 372
 373         return 0;
 374 }
 375
 376 static void pl330_free_chan_resources(struct dma_chan *chan)
 377 {
 378         struct dma_pl330_chan *pch = to_pchan(chan);
 379         unsigned long flags;
 380
 381         spin_lock_irqsave(&pch->lock, flags);
 382
 383         tasklet_kill(&pch->task);
 384
 385         pl330_release_channel(pch->pl330_chid);
 386         pch->pl330_chid = NULL;
 387
 388         if (pch->cyclic)
 389                 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
 390
 391         spin_unlock_irqrestore(&pch->lock, flags);
 392 }
 393
 394 static enum dma_status
 395 pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 396                  struct dma_tx_state *txstate)
 397 {
 398         struct dma_pl330_chan *pch = to_pchan(chan);
 399         dma_cookie_t last_done, last_used;
 400         int ret;
 401
 402         last_done = chan->completed_cookie;
 403         last_used = chan->cookie;
 404
 405         ret = dma_async_is_complete(cookie, last_done, last_used);
 406
 407         dma_set_tx_state(txstate, last_done, last_used, 0);
 408
 409         return ret;
 410 }
 411
 412 static void pl330_issue_pending(struct dma_chan *chan)
 413 {
 414         pl330_tasklet((unsigned long) to_pchan(chan));
 415 }
 416
 417 /*
 418  * We returned the last one of the circular list of descriptor(s)
 419  * from prep_xxx, so the argument to submit corresponds to the last
 420  * descriptor of the list.
 421  */
 422 static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
 423 {
 424         struct dma_pl330_desc *desc, *last = to_desc(tx);
 425         struct dma_pl330_chan *pch = to_pchan(tx->chan);
 426         dma_cookie_t cookie;
 427         unsigned long flags;
 428
 429         spin_lock_irqsave(&pch->lock, flags);
 430
 431         /* Assign cookies to all nodes */
 432         cookie = tx->chan->cookie;
 433
 434         while (!list_empty(&last->node)) {
 435                 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
 436
 437                 if (++cookie < 0)
 438                         cookie = 1;
 439                 desc->txd.cookie = cookie;
 440
 441                 list_move_tail(&desc->node, &pch->work_list);
 442         }
 443
 444         if (++cookie < 0)
 445                 cookie = 1;
 446         last->txd.cookie = cookie;
 447
 448         list_add_tail(&last->node, &pch->work_list);
 449
 450         tx->chan->cookie = cookie;
 451
 452         spin_unlock_irqrestore(&pch->lock, flags);
 453
 454         return cookie;
 455 }
 456
 457 static inline void _init_desc(struct dma_pl330_desc *desc)
 458 {
 459         desc->pchan = NULL;
 460         desc->req.x = &desc->px;
 461         desc->req.token = desc;
 462         desc->rqcfg.swap = SWAP_NO;
 463         desc->rqcfg.privileged = 0;
 464         desc->rqcfg.insnaccess = 0;
 465         desc->rqcfg.scctl = SCCTRL0;
 466         desc->rqcfg.dcctl = DCCTRL0;
 467         desc->req.cfg = &desc->rqcfg;
 468         desc->req.xfer_cb = dma_pl330_rqcb;
 469         desc->txd.tx_submit = pl330_tx_submit;
 470
 471         INIT_LIST_HEAD(&desc->node);
 472 }
 473
 474 /* Returns the number of descriptors added to the DMAC pool */
 475 int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
 476 {
 477         struct dma_pl330_desc *desc;
 478         unsigned long flags;
 479         int i;
 480
 481         if (!pdmac)
 482                 return 0;
 483
 484         desc = kmalloc(count * sizeof(*desc), flg);
 485         if (!desc)
 486                 return 0;
 487
 488         spin_lock_irqsave(&pdmac->pool_lock, flags);
 489
 490         for (i = 0; i < count; i++) {
 491                 _init_desc(&desc[i]);
 492                 list_add_tail(&desc[i].node, &pdmac->desc_pool);
 493         }
 494
 495         spin_unlock_irqrestore(&pdmac->pool_lock, flags);
 496
 497         return count;
 498 }
 499
 500 static struct dma_pl330_desc *
 501 pluck_desc(struct dma_pl330_dmac *pdmac)
 502 {
 503         struct dma_pl330_desc *desc = NULL;
 504         unsigned long flags;
 505
 506         if (!pdmac)
 507                 return NULL;
 508
 509         spin_lock_irqsave(&pdmac->pool_lock, flags);
 510
 511         if (!list_empty(&pdmac->desc_pool)) {
 512                 desc = list_entry(pdmac->desc_pool.next,
 513                                 struct dma_pl330_desc, node);
 514
 515                 list_del_init(&desc->node);
 516
 517                 desc->status = PREP;
 518                 desc->txd.callback = NULL;
 519         }
 520
 521         spin_unlock_irqrestore(&pdmac->pool_lock, flags);
 522
 523         return desc;
 524 }
 525
 526 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
 527 {
 528         struct dma_pl330_dmac *pdmac = pch->dmac;
 529         u8 *peri_id = pch->chan.private;
 530         struct dma_pl330_desc *desc;
 531
 532         /* Pluck one desc from the pool of DMAC */
 533         desc = pluck_desc(pdmac);
 534
 535         /* If the DMAC pool is empty, alloc new */
 536         if (!desc) {
 537                 if (!add_desc(pdmac, GFP_ATOMIC, 1))
 538                         return NULL;
 539
 540                 /* Try again */
 541                 desc = pluck_desc(pdmac);
 542                 if (!desc) {
 543                         dev_err(pch->dmac->pif.dev,
 544                                 "%s:%d ALERT!\n", __func__, __LINE__);
 545                         return NULL;
 546                 }
 547         }
 548
 549         /* Initialize the descriptor */
 550         desc->pchan = pch;
 551         desc->txd.cookie = 0;
 552         async_tx_ack(&desc->txd);
 553
 554         desc->req.peri = peri_id ? pch->chan.chan_id : 0;
 555
 556         dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
 557
 558         return desc;
 559 }
 560
 561 static inline void fill_px(struct pl330_xfer *px,
 562                 dma_addr_t dst, dma_addr_t src, size_t len)
 563 {
 564         px->next = NULL;
 565         px->bytes = len;
 566         px->dst_addr = dst;
 567         px->src_addr = src;
 568 }
 569
 570 static struct dma_pl330_desc *
 571 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
 572                 dma_addr_t src, size_t len)
 573 {
 574         struct dma_pl330_desc *desc = pl330_get_desc(pch);
 575
 576         if (!desc) {
 577                 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
 578                         __func__, __LINE__);
 579                 return NULL;
 580         }
 581
 582         /*
 583          * Ideally we should lookout for reqs bigger than
 584          * those that can be programmed with 256 bytes of
 585          * MC buffer, but considering a req size is seldom
 586          * going to be word-unaligned and more than 200MB,
 587          * we take it easy.
 588          * Also, should the limit is reached we'd rather
 589          * have the platform increase MC buffer size than
 590          * complicating this API driver.
 591          */
 592         fill_px(&desc->px, dst, src, len);
 593
 594         return desc;
 595 }
 596
 597 /* Call after fixing burst size */
 598 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
 599 {
 600         struct dma_pl330_chan *pch = desc->pchan;
 601         struct pl330_info *pi = &pch->dmac->pif;
 602         int burst_len;
 603
 604         burst_len = pi->pcfg.data_bus_width / 8;
 605         burst_len *= pi->pcfg.data_buf_dep;
 606         burst_len >>= desc->rqcfg.brst_size;
 607
 608         /* src/dst_burst_len can't be more than 16 */
 609         if (burst_len > 16)
 610                 burst_len = 16;
 611
 612         while (burst_len > 1) {
 613                 if (!(len % (burst_len << desc->rqcfg.brst_size)))
 614                         break;
 615                 burst_len--;
 616         }
 617
 618         return burst_len;
 619 }
 620
 621 static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
 622                 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
 623                 size_t period_len, enum dma_transfer_direction direction)
 624 {
 625         struct dma_pl330_desc *desc;
 626         struct dma_pl330_chan *pch = to_pchan(chan);
 627         dma_addr_t dst;
 628         dma_addr_t src;
 629
 630         desc = pl330_get_desc(pch);
 631         if (!desc) {
 632                 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
 633                         __func__, __LINE__);
 634                 return NULL;
 635         }
 636
 637         switch (direction) {
 638         case DMA_MEM_TO_DEV:
 639                 desc->rqcfg.src_inc = 1;
 640                 desc->rqcfg.dst_inc = 0;
 641                 desc->req.rqtype = MEMTODEV;
 642                 src = dma_addr;
 643                 dst = pch->fifo_addr;
 644                 break;
 645         case DMA_DEV_TO_MEM:
 646                 desc->rqcfg.src_inc = 0;
 647                 desc->rqcfg.dst_inc = 1;
 648                 desc->req.rqtype = DEVTOMEM;
 649                 src = pch->fifo_addr;
 650                 dst = dma_addr;
 651                 break;
 652         default:
 653                 dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n",
 654                 __func__, __LINE__);
 655                 return NULL;
 656         }
 657
 658         desc->rqcfg.brst_size = pch->burst_sz;
 659         desc->rqcfg.brst_len = 1;
 660
 661         pch->cyclic = true;
 662
 663         fill_px(&desc->px, dst, src, period_len);
 664
 665         return &desc->txd;
 666 }
 667
 668 static struct dma_async_tx_descriptor *
 669 pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
 670                 dma_addr_t src, size_t len, unsigned long flags)
 671 {
 672         struct dma_pl330_desc *desc;
 673         struct dma_pl330_chan *pch = to_pchan(chan);
 674         struct pl330_info *pi;
 675         int burst;
 676
 677         if (unlikely(!pch || !len))
 678                 return NULL;
 679
 680         pi = &pch->dmac->pif;
 681
 682         desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
 683         if (!desc)
 684                 return NULL;
 685
 686         desc->rqcfg.src_inc = 1;
 687         desc->rqcfg.dst_inc = 1;
 688         desc->req.rqtype = MEMTOMEM;
 689
 690         /* Select max possible burst size */
 691         burst = pi->pcfg.data_bus_width / 8;
 692
 693         while (burst > 1) {
 694                 if (!(len % burst))
 695                         break;
 696                 burst /= 2;
 697         }
 698
 699         desc->rqcfg.brst_size = 0;
 700         while (burst != (1 << desc->rqcfg.brst_size))
 701                 desc->rqcfg.brst_size++;
 702
 703         desc->rqcfg.brst_len = get_burst_len(desc, len);
 704
 705         desc->txd.flags = flags;
 706
 707         return &desc->txd;
 708 }
 709
 710 static struct dma_async_tx_descriptor *
 711 pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 712                 unsigned int sg_len, enum dma_transfer_direction direction,
 713                 unsigned long flg)
 714 {
 715         struct dma_pl330_desc *first, *desc = NULL;
 716         struct dma_pl330_chan *pch = to_pchan(chan);
 717         struct scatterlist *sg;
 718         unsigned long flags;
 719         int i;
 720         dma_addr_t addr;
 721
 722         if (unlikely(!pch || !sgl || !sg_len))
 723                 return NULL;
 724
 725         addr = pch->fifo_addr;
 726
 727         first = NULL;
 728
 729         for_each_sg(sgl, sg, sg_len, i) {
 730
 731                 desc = pl330_get_desc(pch);
 732                 if (!desc) {
 733                         struct dma_pl330_dmac *pdmac = pch->dmac;
 734
 735                         dev_err(pch->dmac->pif.dev,
 736                                 "%s:%d Unable to fetch desc\n",
 737                                 __func__, __LINE__);
 738                         if (!first)
 739                                 return NULL;
 740
 741                         spin_lock_irqsave(&pdmac->pool_lock, flags);
 742
 743                         while (!list_empty(&first->node)) {
 744                                 desc = list_entry(first->node.next,
 745                                                 struct dma_pl330_desc, node);
 746                                 list_move_tail(&desc->node, &pdmac->desc_pool);
 747                         }
 748
 749                         list_move_tail(&first->node, &pdmac->desc_pool);
 750
 751                         spin_unlock_irqrestore(&pdmac->pool_lock, flags);
 752
 753                         return NULL;
 754                 }
 755
 756                 if (!first)
 757                         first = desc;
 758                 else
 759                         list_add_tail(&desc->node, &first->node);
 760
 761                 if (direction == DMA_MEM_TO_DEV) {
 762                         desc->rqcfg.src_inc = 1;
 763                         desc->rqcfg.dst_inc = 0;
 764                         desc->req.rqtype = MEMTODEV;
 765                         fill_px(&desc->px,
 766                                 addr, sg_dma_address(sg), sg_dma_len(sg));
 767                 } else {
 768                         desc->rqcfg.src_inc = 0;
 769                         desc->rqcfg.dst_inc = 1;
 770                         desc->req.rqtype = DEVTOMEM;
 771                         fill_px(&desc->px,
 772                                 sg_dma_address(sg), addr, sg_dma_len(sg));
 773                 }
 774
 775                 desc->rqcfg.brst_size = pch->burst_sz;
 776                 desc->rqcfg.brst_len = 1;
 777         }
 778
 779         /* Return the last desc in the chain */
 780         desc->txd.flags = flg;
 781         return &desc->txd;
 782 }
 783
 784 static irqreturn_t pl330_irq_handler(int irq, void *data)
 785 {
 786         if (pl330_update(data))
 787                 return IRQ_HANDLED;
 788         else
 789                 return IRQ_NONE;
 790 }
 791
 792 static int __devinit
 793 pl330_probe(struct amba_device *adev, const struct amba_id *id)
 794 {
 795         struct dma_pl330_platdata *pdat;
 796         struct dma_pl330_dmac *pdmac;
 797         struct dma_pl330_chan *pch;
 798         struct pl330_info *pi;
 799         struct dma_device *pd;
 800         struct resource *res;
 801         int i, ret, irq;
 802         int num_chan;
 803
 804         pdat = adev->dev.platform_data;
 805
 806         /* Allocate a new DMAC and its Channels */
 807         pdmac = kzalloc(sizeof(*pdmac), GFP_KERNEL);
 808         if (!pdmac) {
 809                 dev_err(&adev->dev, "unable to allocate mem\n");
 810                 return -ENOMEM;
 811         }
 812
 813         pi = &pdmac->pif;
 814         pi->dev = &adev->dev;
 815         pi->pl330_data = NULL;
 816         pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
 817
 818         res = &adev->res;
 819         request_mem_region(res->start, resource_size(res), "dma-pl330");
 820
 821         pi->base = ioremap(res->start, resource_size(res));
 822         if (!pi->base) {
 823                 ret = -ENXIO;
 824                 goto probe_err1;
 825         }
 826
 827         pdmac->clk = clk_get(&adev->dev, "dma");
 828         if (IS_ERR(pdmac->clk)) {
 829                 dev_err(&adev->dev, "Cannot get operation clock.\n");
 830                 ret = -EINVAL;
 831                 goto probe_err2;
 832         }
 833
 834         amba_set_drvdata(adev, pdmac);
 835
 836 #ifndef CONFIG_PM_RUNTIME
 837         /* enable dma clk */
 838         clk_enable(pdmac->clk);
 839 #endif
 840
 841         irq = adev->irq[0];
 842         ret = request_irq(irq, pl330_irq_handler, 0,
 843                         dev_name(&adev->dev), pi);
 844         if (ret)
 845                 goto probe_err3;
 846
 847         ret = pl330_add(pi);
 848         if (ret)
 849                 goto probe_err4;
 850
 851         INIT_LIST_HEAD(&pdmac->desc_pool);
 852         spin_lock_init(&pdmac->pool_lock);
 853
 854         /* Create a descriptor pool of default size */
 855         if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
 856                 dev_warn(&adev->dev, "unable to allocate desc\n");
 857
 858         pd = &pdmac->ddma;
 859         INIT_LIST_HEAD(&pd->channels);
 860
 861         /* Initialize channel parameters */
 862         num_chan = max(pdat ? pdat->nr_valid_peri : (u8)pi->pcfg.num_peri,
 863                         (u8)pi->pcfg.num_chan);
 864         pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
 865
 866         for (i = 0; i < num_chan; i++) {
 867                 pch = &pdmac->peripherals[i];
 868                 if (!adev->dev.of_node)
 869                         pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
 870                 else
 871                         pch->chan.private = adev->dev.of_node;
 872
 873                 INIT_LIST_HEAD(&pch->work_list);
 874                 spin_lock_init(&pch->lock);
 875                 pch->pl330_chid = NULL;
 876                 pch->chan.device = pd;
 877                 pch->dmac = pdmac;
 878
 879                 /* Add the channel to the DMAC list */
 880                 list_add_tail(&pch->chan.device_node, &pd->channels);
 881         }
 882
 883         pd->dev = &adev->dev;
 884         if (pdat) {
 885                 pd->cap_mask = pdat->cap_mask;
 886         } else {
 887                 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
 888                 if (pi->pcfg.num_peri) {
 889                         dma_cap_set(DMA_SLAVE, pd->cap_mask);
 890                         dma_cap_set(DMA_CYCLIC, pd->cap_mask);
 891                 }
 892         }
 893
 894         pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
 895         pd->device_free_chan_resources = pl330_free_chan_resources;
 896         pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
 897         pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
 898         pd->device_tx_status = pl330_tx_status;
 899         pd->device_prep_slave_sg = pl330_prep_slave_sg;
 900         pd->device_control = pl330_control;
 901         pd->device_issue_pending = pl330_issue_pending;
 902
 903         ret = dma_async_device_register(pd);
 904         if (ret) {
 905                 dev_err(&adev->dev, "unable to register DMAC\n");
 906                 goto probe_err5;
 907         }
 908
 909         dev_info(&adev->dev,
 910                 "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
 911         dev_info(&adev->dev,
 912                 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
 913                 pi->pcfg.data_buf_dep,
 914                 pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
 915                 pi->pcfg.num_peri, pi->pcfg.num_events);
 916
 917         return 0;
 918
 919 probe_err5:
 920         pl330_del(pi);
 921 probe_err4:
 922         free_irq(irq, pi);
 923 probe_err3:
 924 #ifndef CONFIG_PM_RUNTIME
 925         clk_disable(pdmac->clk);
 926 #endif
 927         clk_put(pdmac->clk);
 928 probe_err2:
 929         iounmap(pi->base);
 930 probe_err1:
 931         release_mem_region(res->start, resource_size(res));
 932         kfree(pdmac);
 933
 934         return ret;
 935 }
 936
 937 static int __devexit pl330_remove(struct amba_device *adev)
 938 {
 939         struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
 940         struct dma_pl330_chan *pch, *_p;
 941         struct pl330_info *pi;
 942         struct resource *res;
 943         int irq;
 944
 945         if (!pdmac)
 946                 return 0;
 947
 948         amba_set_drvdata(adev, NULL);
 949
 950         /* Idle the DMAC */
 951         list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
 952                         chan.device_node) {
 953
 954                 /* Remove the channel */
 955                 list_del(&pch->chan.device_node);
 956
 957                 /* Flush the channel */
 958                 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
 959                 pl330_free_chan_resources(&pch->chan);
 960         }
 961
 962         pi = &pdmac->pif;
 963
 964         pl330_del(pi);
 965
 966         irq = adev->irq[0];
 967         free_irq(irq, pi);
 968
 969         iounmap(pi->base);
 970
 971         res = &adev->res;
 972         release_mem_region(res->start, resource_size(res));
 973
 974 #ifndef CONFIG_PM_RUNTIME
 975         clk_disable(pdmac->clk);
 976 #endif
 977
 978         kfree(pdmac);
 979
 980         return 0;
 981 }
 982
 983 static struct amba_id pl330_ids[] = {
 984         {
 985                 .id     = 0x00041330,
 986                 .mask   = 0x000fffff,
 987         },
 988         { 0, 0 },
 989 };
 990
 991 MODULE_DEVICE_TABLE(amba, pl330_ids);
 992
 993 #ifdef CONFIG_PM_RUNTIME
 994 static int pl330_runtime_suspend(struct device *dev)
 995 {
 996         struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
 997
 998         if (!pdmac) {
 999                 dev_err(dev, "failed to get dmac\n");
1000                 return -ENODEV;
1001         }
1002
1003         clk_disable(pdmac->clk);
1004
1005         return 0;
1006 }
1007
1008 static int pl330_runtime_resume(struct device *dev)
1009 {
1010         struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
1011
1012         if (!pdmac) {
1013                 dev_err(dev, "failed to get dmac\n");
1014                 return -ENODEV;
1015         }
1016
1017         clk_enable(pdmac->clk);
1018
1019         return 0;
1020 }
1021 #else
1022 #define pl330_runtime_suspend   NULL
1023 #define pl330_runtime_resume    NULL
1024 #endif /* CONFIG_PM_RUNTIME */
1025
1026 static const struct dev_pm_ops pl330_pm_ops = {
1027         .runtime_suspend = pl330_runtime_suspend,
1028         .runtime_resume = pl330_runtime_resume,
1029 };
1030
1031 static struct amba_driver pl330_driver = {
1032         .drv = {
1033                 .owner = THIS_MODULE,
1034                 .name = "dma-pl330",
1035                 .pm = &pl330_pm_ops,
1036         },
1037         .id_table = pl330_ids,
1038         .probe = pl330_probe,
1039         .remove = pl330_remove,
1040 };
1041
1042 static int __init pl330_init(void)
1043 {
1044         return amba_driver_register(&pl330_driver);
1045 }
1046 module_init(pl330_init);
1047
1048 static void __exit pl330_exit(void)
1049 {
1050         amba_driver_unregister(&pl330_driver);
1051         return;
1052 }
1053 module_exit(pl330_exit);
1054
1055 MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
1056 MODULE_DESCRIPTION("API Driver for PL330 DMAC");
1057 MODULE_LICENSE("GPL");