drivers/dma/dw/core.c

   1 /*
   2  * Core driver for the Synopsys DesignWare DMA Controller
   3  *
   4  * Copyright (C) 2007-2008 Atmel Corporation
   5  * Copyright (C) 2010-2011 ST Microelectronics
   6  * Copyright (C) 2013 Intel Corporation
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/bitops.h>
  14 #include <linux/clk.h>
  15 #include <linux/delay.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmapool.h>
  19 #include <linux/err.h>
  20 #include <linux/init.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io.h>
  23 #include <linux/mm.h>
  24 #include <linux/module.h>
  25 #include <linux/slab.h>
  26
  27 #include "../dmaengine.h"
  28 #include "internal.h"
  29
  30 /*
  31  * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  32  * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  33  * of which use ARM any more).  See the "Databook" from Synopsys for
  34  * information beyond what licensees probably provide.
  35  *
  36  * The driver has currently been tested only with the Atmel AT32AP7000,
  37  * which does not support descriptor writeback.
  38  */
  39
  40 static inline bool is_request_line_unset(struct dw_dma_chan *dwc)
  41 {
  42         return dwc->request_line == (typeof(dwc->request_line))~0;
  43 }
  44
  45 static inline void dwc_set_masters(struct dw_dma_chan *dwc)
  46 {
  47         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
  48         struct dw_dma_slave *dws = dwc->chan.private;
  49         unsigned char mmax = dw->nr_masters - 1;
  50
  51         if (!is_request_line_unset(dwc))
  52                 return;
  53
  54         dwc->src_master = min_t(unsigned char, mmax, dwc_get_sms(dws));
  55         dwc->dst_master = min_t(unsigned char, mmax, dwc_get_dms(dws));
  56 }
  57
  58 #define DWC_DEFAULT_CTLLO(_chan) ({                             \
  59                 struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan);       \
  60                 struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
  61                 bool _is_slave = is_slave_direction(_dwc->direction);   \
  62                 u8 _smsize = _is_slave ? _sconfig->src_maxburst :       \
  63                         DW_DMA_MSIZE_16;                        \
  64                 u8 _dmsize = _is_slave ? _sconfig->dst_maxburst :       \
  65                         DW_DMA_MSIZE_16;                        \
  66                                                                 \
  67                 (DWC_CTLL_DST_MSIZE(_dmsize)                    \
  68                  | DWC_CTLL_SRC_MSIZE(_smsize)                  \
  69                  | DWC_CTLL_LLP_D_EN                            \
  70                  | DWC_CTLL_LLP_S_EN                            \
  71                  | DWC_CTLL_DMS(_dwc->dst_master)               \
  72                  | DWC_CTLL_SMS(_dwc->src_master));             \
  73         })
  74
  75 /*
  76  * Number of descriptors to allocate for each channel. This should be
  77  * made configurable somehow; preferably, the clients (at least the
  78  * ones using slave transfers) should be able to give us a hint.
  79  */
  80 #define NR_DESCS_PER_CHANNEL    64
  81
  82 /*----------------------------------------------------------------------*/
  83
  84 static struct device *chan2dev(struct dma_chan *chan)
  85 {
  86         return &chan->dev->device;
  87 }
  88 static struct device *chan2parent(struct dma_chan *chan)
  89 {
  90         return chan->dev->device.parent;
  91 }
  92
  93 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
  94 {
  95         return to_dw_desc(dwc->active_list.next);
  96 }
  97
  98 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
  99 {
 100         struct dw_desc *desc, *_desc;
 101         struct dw_desc *ret = NULL;
 102         unsigned int i = 0;
 103         unsigned long flags;
 104
 105         spin_lock_irqsave(&dwc->lock, flags);
 106         list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
 107                 i++;
 108                 if (async_tx_test_ack(&desc->txd)) {
 109                         list_del(&desc->desc_node);
 110                         ret = desc;
 111                         break;
 112                 }
 113                 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
 114         }
 115         spin_unlock_irqrestore(&dwc->lock, flags);
 116
 117         dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
 118
 119         return ret;
 120 }
 121
 122 /*
 123  * Move a descriptor, including any children, to the free list.
 124  * `desc' must not be on any lists.
 125  */
 126 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
 127 {
 128         unsigned long flags;
 129
 130         if (desc) {
 131                 struct dw_desc *child;
 132
 133                 spin_lock_irqsave(&dwc->lock, flags);
 134                 list_for_each_entry(child, &desc->tx_list, desc_node)
 135                         dev_vdbg(chan2dev(&dwc->chan),
 136                                         "moving child desc %p to freelist\n",
 137                                         child);
 138                 list_splice_init(&desc->tx_list, &dwc->free_list);
 139                 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
 140                 list_add(&desc->desc_node, &dwc->free_list);
 141                 spin_unlock_irqrestore(&dwc->lock, flags);
 142         }
 143 }
 144
 145 static void dwc_initialize(struct dw_dma_chan *dwc)
 146 {
 147         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 148         struct dw_dma_slave *dws = dwc->chan.private;
 149         u32 cfghi = DWC_CFGH_FIFO_MODE;
 150         u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
 151
 152         if (dwc->initialized == true)
 153                 return;
 154
 155         if (dws) {
 156                 /*
 157                  * We need controller-specific data to set up slave
 158                  * transfers.
 159                  */
 160                 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
 161
 162                 cfghi = dws->cfg_hi;
 163                 cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
 164         } else {
 165                 if (dwc->direction == DMA_MEM_TO_DEV)
 166                         cfghi = DWC_CFGH_DST_PER(dwc->request_line);
 167                 else if (dwc->direction == DMA_DEV_TO_MEM)
 168                         cfghi = DWC_CFGH_SRC_PER(dwc->request_line);
 169         }
 170
 171         channel_writel(dwc, CFG_LO, cfglo);
 172         channel_writel(dwc, CFG_HI, cfghi);
 173
 174         /* Enable interrupts */
 175         channel_set_bit(dw, MASK.XFER, dwc->mask);
 176         channel_set_bit(dw, MASK.ERROR, dwc->mask);
 177
 178         dwc->initialized = true;
 179 }
 180
 181 /*----------------------------------------------------------------------*/
 182
 183 static inline unsigned int dwc_fast_fls(unsigned long long v)
 184 {
 185         /*
 186          * We can be a lot more clever here, but this should take care
 187          * of the most common optimization.
 188          */
 189         if (!(v & 7))
 190                 return 3;
 191         else if (!(v & 3))
 192                 return 2;
 193         else if (!(v & 1))
 194                 return 1;
 195         return 0;
 196 }
 197
 198 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
 199 {
 200         dev_err(chan2dev(&dwc->chan),
 201                 "  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 202                 channel_readl(dwc, SAR),
 203                 channel_readl(dwc, DAR),
 204                 channel_readl(dwc, LLP),
 205                 channel_readl(dwc, CTL_HI),
 206                 channel_readl(dwc, CTL_LO));
 207 }
 208
 209 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
 210 {
 211         channel_clear_bit(dw, CH_EN, dwc->mask);
 212         while (dma_readl(dw, CH_EN) & dwc->mask)
 213                 cpu_relax();
 214 }
 215
 216 /*----------------------------------------------------------------------*/
 217
 218 /* Perform single block transfer */
 219 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
 220                                        struct dw_desc *desc)
 221 {
 222         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 223         u32             ctllo;
 224
 225         /* Software emulation of LLP mode relies on interrupts to continue
 226          * multi block transfer. */
 227         ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;
 228
 229         channel_writel(dwc, SAR, desc->lli.sar);
 230         channel_writel(dwc, DAR, desc->lli.dar);
 231         channel_writel(dwc, CTL_LO, ctllo);
 232         channel_writel(dwc, CTL_HI, desc->lli.ctlhi);
 233         channel_set_bit(dw, CH_EN, dwc->mask);
 234
 235         /* Move pointer to next descriptor */
 236         dwc->tx_node_active = dwc->tx_node_active->next;
 237 }
 238
 239 /* Called with dwc->lock held and bh disabled */
 240 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 241 {
 242         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 243         unsigned long   was_soft_llp;
 244
 245         /* ASSERT:  channel is idle */
 246         if (dma_readl(dw, CH_EN) & dwc->mask) {
 247                 dev_err(chan2dev(&dwc->chan),
 248                         "BUG: Attempted to start non-idle channel\n");
 249                 dwc_dump_chan_regs(dwc);
 250
 251                 /* The tasklet will hopefully advance the queue... */
 252                 return;
 253         }
 254
 255         if (dwc->nollp) {
 256                 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
 257                                                 &dwc->flags);
 258                 if (was_soft_llp) {
 259                         dev_err(chan2dev(&dwc->chan),
 260                                 "BUG: Attempted to start new LLP transfer "
 261                                 "inside ongoing one\n");
 262                         return;
 263                 }
 264
 265                 dwc_initialize(dwc);
 266
 267                 dwc->residue = first->total_len;
 268                 dwc->tx_node_active = &first->tx_list;
 269
 270                 /* Submit first block */
 271                 dwc_do_single_block(dwc, first);
 272
 273                 return;
 274         }
 275
 276         dwc_initialize(dwc);
 277
 278         channel_writel(dwc, LLP, first->txd.phys);
 279         channel_writel(dwc, CTL_LO,
 280                         DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 281         channel_writel(dwc, CTL_HI, 0);
 282         channel_set_bit(dw, CH_EN, dwc->mask);
 283 }
 284
 285 /*----------------------------------------------------------------------*/
 286
 287 static void
 288 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 289                 bool callback_required)
 290 {
 291         dma_async_tx_callback           callback = NULL;
 292         void                            *param = NULL;
 293         struct dma_async_tx_descriptor  *txd = &desc->txd;
 294         struct dw_desc                  *child;
 295         unsigned long                   flags;
 296
 297         dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 298
 299         spin_lock_irqsave(&dwc->lock, flags);
 300         dma_cookie_complete(txd);
 301         if (callback_required) {
 302                 callback = txd->callback;
 303                 param = txd->callback_param;
 304         }
 305
 306         /* async_tx_ack */
 307         list_for_each_entry(child, &desc->tx_list, desc_node)
 308                 async_tx_ack(&child->txd);
 309         async_tx_ack(&desc->txd);
 310
 311         list_splice_init(&desc->tx_list, &dwc->free_list);
 312         list_move(&desc->desc_node, &dwc->free_list);
 313
 314         if (!is_slave_direction(dwc->direction)) {
 315                 struct device *parent = chan2parent(&dwc->chan);
 316                 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 317                         if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 318                                 dma_unmap_single(parent, desc->lli.dar,
 319                                         desc->total_len, DMA_FROM_DEVICE);
 320                         else
 321                                 dma_unmap_page(parent, desc->lli.dar,
 322                                         desc->total_len, DMA_FROM_DEVICE);
 323                 }
 324                 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 325                         if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 326                                 dma_unmap_single(parent, desc->lli.sar,
 327                                         desc->total_len, DMA_TO_DEVICE);
 328                         else
 329                                 dma_unmap_page(parent, desc->lli.sar,
 330                                         desc->total_len, DMA_TO_DEVICE);
 331                 }
 332         }
 333
 334         spin_unlock_irqrestore(&dwc->lock, flags);
 335
 336         if (callback)
 337                 callback(param);
 338 }
 339
 340 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 341 {
 342         struct dw_desc *desc, *_desc;
 343         LIST_HEAD(list);
 344         unsigned long flags;
 345
 346         spin_lock_irqsave(&dwc->lock, flags);
 347         if (dma_readl(dw, CH_EN) & dwc->mask) {
 348                 dev_err(chan2dev(&dwc->chan),
 349                         "BUG: XFER bit set, but channel not idle!\n");
 350
 351                 /* Try to continue after resetting the channel... */
 352                 dwc_chan_disable(dw, dwc);
 353         }
 354
 355         /*
 356          * Submit queued descriptors ASAP, i.e. before we go through
 357          * the completed ones.
 358          */
 359         list_splice_init(&dwc->active_list, &list);
 360         if (!list_empty(&dwc->queue)) {
 361                 list_move(dwc->queue.next, &dwc->active_list);
 362                 dwc_dostart(dwc, dwc_first_active(dwc));
 363         }
 364
 365         spin_unlock_irqrestore(&dwc->lock, flags);
 366
 367         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 368                 dwc_descriptor_complete(dwc, desc, true);
 369 }
 370
 371 /* Returns how many bytes were already received from source */
 372 static inline u32 dwc_get_sent(struct dw_dma_chan *dwc)
 373 {
 374         u32 ctlhi = channel_readl(dwc, CTL_HI);
 375         u32 ctllo = channel_readl(dwc, CTL_LO);
 376
 377         return (ctlhi & DWC_CTLH_BLOCK_TS_MASK) * (1 << (ctllo >> 4 & 7));
 378 }
 379
 380 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 381 {
 382         dma_addr_t llp;
 383         struct dw_desc *desc, *_desc;
 384         struct dw_desc *child;
 385         u32 status_xfer;
 386         unsigned long flags;
 387
 388         spin_lock_irqsave(&dwc->lock, flags);
 389         llp = channel_readl(dwc, LLP);
 390         status_xfer = dma_readl(dw, RAW.XFER);
 391
 392         if (status_xfer & dwc->mask) {
 393                 /* Everything we've submitted is done */
 394                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 395
 396                 if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 397                         struct list_head *head, *active = dwc->tx_node_active;
 398
 399                         /*
 400                          * We are inside first active descriptor.
 401                          * Otherwise something is really wrong.
 402                          */
 403                         desc = dwc_first_active(dwc);
 404
 405                         head = &desc->tx_list;
 406                         if (active != head) {
 407                                 /* Update desc to reflect last sent one */
 408                                 if (active != head->next)
 409                                         desc = to_dw_desc(active->prev);
 410
 411                                 dwc->residue -= desc->len;
 412
 413                                 child = to_dw_desc(active);
 414
 415                                 /* Submit next block */
 416                                 dwc_do_single_block(dwc, child);
 417
 418                                 spin_unlock_irqrestore(&dwc->lock, flags);
 419                                 return;
 420                         }
 421
 422                         /* We are done here */
 423                         clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 424                 }
 425
 426                 dwc->residue = 0;
 427
 428                 spin_unlock_irqrestore(&dwc->lock, flags);
 429
 430                 dwc_complete_all(dw, dwc);
 431                 return;
 432         }
 433
 434         if (list_empty(&dwc->active_list)) {
 435                 dwc->residue = 0;
 436                 spin_unlock_irqrestore(&dwc->lock, flags);
 437                 return;
 438         }
 439
 440         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 441                 dev_vdbg(chan2dev(&dwc->chan), "%s: soft LLP mode\n", __func__);
 442                 spin_unlock_irqrestore(&dwc->lock, flags);
 443                 return;
 444         }
 445
 446         dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__,
 447                         (unsigned long long)llp);
 448
 449         list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 450                 /* Initial residue value */
 451                 dwc->residue = desc->total_len;
 452
 453                 /* Check first descriptors addr */
 454                 if (desc->txd.phys == llp) {
 455                         spin_unlock_irqrestore(&dwc->lock, flags);
 456                         return;
 457                 }
 458
 459                 /* Check first descriptors llp */
 460                 if (desc->lli.llp == llp) {
 461                         /* This one is currently in progress */
 462                         dwc->residue -= dwc_get_sent(dwc);
 463                         spin_unlock_irqrestore(&dwc->lock, flags);
 464                         return;
 465                 }
 466
 467                 dwc->residue -= desc->len;
 468                 list_for_each_entry(child, &desc->tx_list, desc_node) {
 469                         if (child->lli.llp == llp) {
 470                                 /* Currently in progress */
 471                                 dwc->residue -= dwc_get_sent(dwc);
 472                                 spin_unlock_irqrestore(&dwc->lock, flags);
 473                                 return;
 474                         }
 475                         dwc->residue -= child->len;
 476                 }
 477
 478                 /*
 479                  * No descriptors so far seem to be in progress, i.e.
 480                  * this one must be done.
 481                  */
 482                 spin_unlock_irqrestore(&dwc->lock, flags);
 483                 dwc_descriptor_complete(dwc, desc, true);
 484                 spin_lock_irqsave(&dwc->lock, flags);
 485         }
 486
 487         dev_err(chan2dev(&dwc->chan),
 488                 "BUG: All descriptors done, but channel not idle!\n");
 489
 490         /* Try to continue after resetting the channel... */
 491         dwc_chan_disable(dw, dwc);
 492
 493         if (!list_empty(&dwc->queue)) {
 494                 list_move(dwc->queue.next, &dwc->active_list);
 495                 dwc_dostart(dwc, dwc_first_active(dwc));
 496         }
 497         spin_unlock_irqrestore(&dwc->lock, flags);
 498 }
 499
 500 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
 501 {
 502         dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 503                  lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
 504 }
 505
 506 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 507 {
 508         struct dw_desc *bad_desc;
 509         struct dw_desc *child;
 510         unsigned long flags;
 511
 512         dwc_scan_descriptors(dw, dwc);
 513
 514         spin_lock_irqsave(&dwc->lock, flags);
 515
 516         /*
 517          * The descriptor currently at the head of the active list is
 518          * borked. Since we don't have any way to report errors, we'll
 519          * just have to scream loudly and try to carry on.
 520          */
 521         bad_desc = dwc_first_active(dwc);
 522         list_del_init(&bad_desc->desc_node);
 523         list_move(dwc->queue.next, dwc->active_list.prev);
 524
 525         /* Clear the error flag and try to restart the controller */
 526         dma_writel(dw, CLEAR.ERROR, dwc->mask);
 527         if (!list_empty(&dwc->active_list))
 528                 dwc_dostart(dwc, dwc_first_active(dwc));
 529
 530         /*
 531          * WARN may seem harsh, but since this only happens
 532          * when someone submits a bad physical address in a
 533          * descriptor, we should consider ourselves lucky that the
 534          * controller flagged an error instead of scribbling over
 535          * random memory locations.
 536          */
 537         dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
 538                                        "  cookie: %d\n", bad_desc->txd.cookie);
 539         dwc_dump_lli(dwc, &bad_desc->lli);
 540         list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 541                 dwc_dump_lli(dwc, &child->lli);
 542
 543         spin_unlock_irqrestore(&dwc->lock, flags);
 544
 545         /* Pretend the descriptor completed successfully */
 546         dwc_descriptor_complete(dwc, bad_desc, true);
 547 }
 548
 549 /* --------------------- Cyclic DMA API extensions -------------------- */
 550
 551 dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
 552 {
 553         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 554         return channel_readl(dwc, SAR);
 555 }
 556 EXPORT_SYMBOL(dw_dma_get_src_addr);
 557
 558 dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
 559 {
 560         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 561         return channel_readl(dwc, DAR);
 562 }
 563 EXPORT_SYMBOL(dw_dma_get_dst_addr);
 564
 565 /* Called with dwc->lock held and all DMAC interrupts disabled */
 566 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
 567                 u32 status_err, u32 status_xfer)
 568 {
 569         unsigned long flags;
 570
 571         if (dwc->mask) {
 572                 void (*callback)(void *param);
 573                 void *callback_param;
 574
 575                 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
 576                                 channel_readl(dwc, LLP));
 577
 578                 callback = dwc->cdesc->period_callback;
 579                 callback_param = dwc->cdesc->period_callback_param;
 580
 581                 if (callback)
 582                         callback(callback_param);
 583         }
 584
 585         /*
 586          * Error and transfer complete are highly unlikely, and will most
 587          * likely be due to a configuration error by the user.
 588          */
 589         if (unlikely(status_err & dwc->mask) ||
 590                         unlikely(status_xfer & dwc->mask)) {
 591                 int i;
 592
 593                 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
 594                                 "interrupt, stopping DMA transfer\n",
 595                                 status_xfer ? "xfer" : "error");
 596
 597                 spin_lock_irqsave(&dwc->lock, flags);
 598
 599                 dwc_dump_chan_regs(dwc);
 600
 601                 dwc_chan_disable(dw, dwc);
 602
 603                 /* Make sure DMA does not restart by loading a new list */
 604                 channel_writel(dwc, LLP, 0);
 605                 channel_writel(dwc, CTL_LO, 0);
 606                 channel_writel(dwc, CTL_HI, 0);
 607
 608                 dma_writel(dw, CLEAR.ERROR, dwc->mask);
 609                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 610
 611                 for (i = 0; i < dwc->cdesc->periods; i++)
 612                         dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
 613
 614                 spin_unlock_irqrestore(&dwc->lock, flags);
 615         }
 616 }
 617
 618 /* ------------------------------------------------------------------------- */
 619
 620 static void dw_dma_tasklet(unsigned long data)
 621 {
 622         struct dw_dma *dw = (struct dw_dma *)data;
 623         struct dw_dma_chan *dwc;
 624         u32 status_xfer;
 625         u32 status_err;
 626         int i;
 627
 628         status_xfer = dma_readl(dw, RAW.XFER);
 629         status_err = dma_readl(dw, RAW.ERROR);
 630
 631         dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
 632
 633         for (i = 0; i < dw->dma.chancnt; i++) {
 634                 dwc = &dw->chan[i];
 635                 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
 636                         dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
 637                 else if (status_err & (1 << i))
 638                         dwc_handle_error(dw, dwc);
 639                 else if (status_xfer & (1 << i))
 640                         dwc_scan_descriptors(dw, dwc);
 641         }
 642
 643         /*
 644          * Re-enable interrupts.
 645          */
 646         channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
 647         channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 648 }
 649
 650 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 651 {
 652         struct dw_dma *dw = dev_id;
 653         u32 status = dma_readl(dw, STATUS_INT);
 654
 655         dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);
 656
 657         /* Check if we have any interrupt from the DMAC */
 658         if (!status)
 659                 return IRQ_NONE;
 660
 661         /*
 662          * Just disable the interrupts. We'll turn them back on in the
 663          * softirq handler.
 664          */
 665         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 666         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 667
 668         status = dma_readl(dw, STATUS_INT);
 669         if (status) {
 670                 dev_err(dw->dma.dev,
 671                         "BUG: Unexpected interrupts pending: 0x%x\n",
 672                         status);
 673
 674                 /* Try to recover */
 675                 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
 676                 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 677                 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 678                 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
 679         }
 680
 681         tasklet_schedule(&dw->tasklet);
 682
 683         return IRQ_HANDLED;
 684 }
 685
 686 /*----------------------------------------------------------------------*/
 687
 688 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
 689 {
 690         struct dw_desc          *desc = txd_to_dw_desc(tx);
 691         struct dw_dma_chan      *dwc = to_dw_dma_chan(tx->chan);
 692         dma_cookie_t            cookie;
 693         unsigned long           flags;
 694
 695         spin_lock_irqsave(&dwc->lock, flags);
 696         cookie = dma_cookie_assign(tx);
 697
 698         /*
 699          * REVISIT: We should attempt to chain as many descriptors as
 700          * possible, perhaps even appending to those already submitted
 701          * for DMA. But this is hard to do in a race-free manner.
 702          */
 703         if (list_empty(&dwc->active_list)) {
 704                 dev_vdbg(chan2dev(tx->chan), "%s: started %u\n", __func__,
 705                                 desc->txd.cookie);
 706                 list_add_tail(&desc->desc_node, &dwc->active_list);
 707                 dwc_dostart(dwc, dwc_first_active(dwc));
 708         } else {
 709                 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__,
 710                                 desc->txd.cookie);
 711
 712                 list_add_tail(&desc->desc_node, &dwc->queue);
 713         }
 714
 715         spin_unlock_irqrestore(&dwc->lock, flags);
 716
 717         return cookie;
 718 }
 719
 720 static struct dma_async_tx_descriptor *
 721 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 722                 size_t len, unsigned long flags)
 723 {
 724         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 725         struct dw_dma           *dw = to_dw_dma(chan->device);
 726         struct dw_desc          *desc;
 727         struct dw_desc          *first;
 728         struct dw_desc          *prev;
 729         size_t                  xfer_count;
 730         size_t                  offset;
 731         unsigned int            src_width;
 732         unsigned int            dst_width;
 733         unsigned int            data_width;
 734         u32                     ctllo;
 735
 736         dev_vdbg(chan2dev(chan),
 737                         "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__,
 738                         (unsigned long long)dest, (unsigned long long)src,
 739                         len, flags);
 740
 741         if (unlikely(!len)) {
 742                 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
 743                 return NULL;
 744         }
 745
 746         dwc->direction = DMA_MEM_TO_MEM;
 747
 748         data_width = min_t(unsigned int, dw->data_width[dwc->src_master],
 749                            dw->data_width[dwc->dst_master]);
 750
 751         src_width = dst_width = min_t(unsigned int, data_width,
 752                                       dwc_fast_fls(src | dest | len));
 753
 754         ctllo = DWC_DEFAULT_CTLLO(chan)
 755                         | DWC_CTLL_DST_WIDTH(dst_width)
 756                         | DWC_CTLL_SRC_WIDTH(src_width)
 757                         | DWC_CTLL_DST_INC
 758                         | DWC_CTLL_SRC_INC
 759                         | DWC_CTLL_FC_M2M;
 760         prev = first = NULL;
 761
 762         for (offset = 0; offset < len; offset += xfer_count << src_width) {
 763                 xfer_count = min_t(size_t, (len - offset) >> src_width,
 764                                            dwc->block_size);
 765
 766                 desc = dwc_desc_get(dwc);
 767                 if (!desc)
 768                         goto err_desc_get;
 769
 770                 desc->lli.sar = src + offset;
 771                 desc->lli.dar = dest + offset;
 772                 desc->lli.ctllo = ctllo;
 773                 desc->lli.ctlhi = xfer_count;
 774                 desc->len = xfer_count << src_width;
 775
 776                 if (!first) {
 777                         first = desc;
 778                 } else {
 779                         prev->lli.llp = desc->txd.phys;
 780                         list_add_tail(&desc->desc_node,
 781                                         &first->tx_list);
 782                 }
 783                 prev = desc;
 784         }
 785
 786         if (flags & DMA_PREP_INTERRUPT)
 787                 /* Trigger interrupt after last block */
 788                 prev->lli.ctllo |= DWC_CTLL_INT_EN;
 789
 790         prev->lli.llp = 0;
 791         first->txd.flags = flags;
 792         first->total_len = len;
 793
 794         return &first->txd;
 795
 796 err_desc_get:
 797         dwc_desc_put(dwc, first);
 798         return NULL;
 799 }
 800
 801 static struct dma_async_tx_descriptor *
 802 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 803                 unsigned int sg_len, enum dma_transfer_direction direction,
 804                 unsigned long flags, void *context)
 805 {
 806         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 807         struct dw_dma           *dw = to_dw_dma(chan->device);
 808         struct dma_slave_config *sconfig = &dwc->dma_sconfig;
 809         struct dw_desc          *prev;
 810         struct dw_desc          *first;
 811         u32                     ctllo;
 812         dma_addr_t              reg;
 813         unsigned int            reg_width;
 814         unsigned int            mem_width;
 815         unsigned int            data_width;
 816         unsigned int            i;
 817         struct scatterlist      *sg;
 818         size_t                  total_len = 0;
 819
 820         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 821
 822         if (unlikely(!is_slave_direction(direction) || !sg_len))
 823                 return NULL;
 824
 825         dwc->direction = direction;
 826
 827         prev = first = NULL;
 828
 829         switch (direction) {
 830         case DMA_MEM_TO_DEV:
 831                 reg_width = __fls(sconfig->dst_addr_width);
 832                 reg = sconfig->dst_addr;
 833                 ctllo = (DWC_DEFAULT_CTLLO(chan)
 834                                 | DWC_CTLL_DST_WIDTH(reg_width)
 835                                 | DWC_CTLL_DST_FIX
 836                                 | DWC_CTLL_SRC_INC);
 837
 838                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
 839                         DWC_CTLL_FC(DW_DMA_FC_D_M2P);
 840
 841                 data_width = dw->data_width[dwc->src_master];
 842
 843                 for_each_sg(sgl, sg, sg_len, i) {
 844                         struct dw_desc  *desc;
 845                         u32             len, dlen, mem;
 846
 847                         mem = sg_dma_address(sg);
 848                         len = sg_dma_len(sg);
 849
 850                         mem_width = min_t(unsigned int,
 851                                           data_width, dwc_fast_fls(mem | len));
 852
 853 slave_sg_todev_fill_desc:
 854                         desc = dwc_desc_get(dwc);
 855                         if (!desc) {
 856                                 dev_err(chan2dev(chan),
 857                                         "not enough descriptors available\n");
 858                                 goto err_desc_get;
 859                         }
 860
 861                         desc->lli.sar = mem;
 862                         desc->lli.dar = reg;
 863                         desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
 864                         if ((len >> mem_width) > dwc->block_size) {
 865                                 dlen = dwc->block_size << mem_width;
 866                                 mem += dlen;
 867                                 len -= dlen;
 868                         } else {
 869                                 dlen = len;
 870                                 len = 0;
 871                         }
 872
 873                         desc->lli.ctlhi = dlen >> mem_width;
 874                         desc->len = dlen;
 875
 876                         if (!first) {
 877                                 first = desc;
 878                         } else {
 879                                 prev->lli.llp = desc->txd.phys;
 880                                 list_add_tail(&desc->desc_node,
 881                                                 &first->tx_list);
 882                         }
 883                         prev = desc;
 884                         total_len += dlen;
 885
 886                         if (len)
 887                                 goto slave_sg_todev_fill_desc;
 888                 }
 889                 break;
 890         case DMA_DEV_TO_MEM:
 891                 reg_width = __fls(sconfig->src_addr_width);
 892                 reg = sconfig->src_addr;
 893                 ctllo = (DWC_DEFAULT_CTLLO(chan)
 894                                 | DWC_CTLL_SRC_WIDTH(reg_width)
 895                                 | DWC_CTLL_DST_INC
 896                                 | DWC_CTLL_SRC_FIX);
 897
 898                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
 899                         DWC_CTLL_FC(DW_DMA_FC_D_P2M);
 900
 901                 data_width = dw->data_width[dwc->dst_master];
 902
 903                 for_each_sg(sgl, sg, sg_len, i) {
 904                         struct dw_desc  *desc;
 905                         u32             len, dlen, mem;
 906
 907                         mem = sg_dma_address(sg);
 908                         len = sg_dma_len(sg);
 909
 910                         mem_width = min_t(unsigned int,
 911                                           data_width, dwc_fast_fls(mem | len));
 912
 913 slave_sg_fromdev_fill_desc:
 914                         desc = dwc_desc_get(dwc);
 915                         if (!desc) {
 916                                 dev_err(chan2dev(chan),
 917                                                 "not enough descriptors available\n");
 918                                 goto err_desc_get;
 919                         }
 920
 921                         desc->lli.sar = reg;
 922                         desc->lli.dar = mem;
 923                         desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
 924                         if ((len >> reg_width) > dwc->block_size) {
 925                                 dlen = dwc->block_size << reg_width;
 926                                 mem += dlen;
 927                                 len -= dlen;
 928                         } else {
 929                                 dlen = len;
 930                                 len = 0;
 931                         }
 932                         desc->lli.ctlhi = dlen >> reg_width;
 933                         desc->len = dlen;
 934
 935                         if (!first) {
 936                                 first = desc;
 937                         } else {
 938                                 prev->lli.llp = desc->txd.phys;
 939                                 list_add_tail(&desc->desc_node,
 940                                                 &first->tx_list);
 941                         }
 942                         prev = desc;
 943                         total_len += dlen;
 944
 945                         if (len)
 946                                 goto slave_sg_fromdev_fill_desc;
 947                 }
 948                 break;
 949         default:
 950                 return NULL;
 951         }
 952
 953         if (flags & DMA_PREP_INTERRUPT)
 954                 /* Trigger interrupt after last block */
 955                 prev->lli.ctllo |= DWC_CTLL_INT_EN;
 956
 957         prev->lli.llp = 0;
 958         first->total_len = total_len;
 959
 960         return &first->txd;
 961
 962 err_desc_get:
 963         dwc_desc_put(dwc, first);
 964         return NULL;
 965 }
 966
 967 /*
 968  * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
 969  * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
 970  *
 971  * NOTE: burst size 2 is not supported by controller.
 972  *
 973  * This can be done by finding least significant bit set: n & (n - 1)
 974  */
 975 static inline void convert_burst(u32 *maxburst)
 976 {
 977         if (*maxburst > 1)
 978                 *maxburst = fls(*maxburst) - 2;
 979         else
 980                 *maxburst = 0;
 981 }
 982
 983 static int
 984 set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
 985 {
 986         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 987
 988         /* Check if chan will be configured for slave transfers */
 989         if (!is_slave_direction(sconfig->direction))
 990                 return -EINVAL;
 991
 992         memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
 993         dwc->direction = sconfig->direction;
 994
 995         /* Take the request line from slave_id member */
 996         if (is_request_line_unset(dwc))
 997                 dwc->request_line = sconfig->slave_id;
 998
 999         convert_burst(&dwc->dma_sconfig.src_maxburst);
1000         convert_burst(&dwc->dma_sconfig.dst_maxburst);
1001
1002         return 0;
1003 }
1004
1005 static inline void dwc_chan_pause(struct dw_dma_chan *dwc)
1006 {
1007         u32 cfglo = channel_readl(dwc, CFG_LO);
1008         unsigned int count = 20;        /* timeout iterations */
1009
1010         channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
1011         while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY) && count--)
1012                 udelay(2);
1013
1014         dwc->paused = true;
1015 }
1016
1017 static inline void dwc_chan_resume(struct dw_dma_chan *dwc)
1018 {
1019         u32 cfglo = channel_readl(dwc, CFG_LO);
1020
1021         channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
1022
1023         dwc->paused = false;
1024 }
1025
1026 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1027                        unsigned long arg)
1028 {
1029         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1030         struct dw_dma           *dw = to_dw_dma(chan->device);
1031         struct dw_desc          *desc, *_desc;
1032         unsigned long           flags;
1033         LIST_HEAD(list);
1034
1035         if (cmd == DMA_PAUSE) {
1036                 spin_lock_irqsave(&dwc->lock, flags);
1037
1038                 dwc_chan_pause(dwc);
1039
1040                 spin_unlock_irqrestore(&dwc->lock, flags);
1041         } else if (cmd == DMA_RESUME) {
1042                 if (!dwc->paused)
1043                         return 0;
1044
1045                 spin_lock_irqsave(&dwc->lock, flags);
1046
1047                 dwc_chan_resume(dwc);
1048
1049                 spin_unlock_irqrestore(&dwc->lock, flags);
1050         } else if (cmd == DMA_TERMINATE_ALL) {
1051                 spin_lock_irqsave(&dwc->lock, flags);
1052
1053                 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
1054
1055                 dwc_chan_disable(dw, dwc);
1056
1057                 dwc_chan_resume(dwc);
1058
1059                 /* active_list entries will end up before queued entries */
1060                 list_splice_init(&dwc->queue, &list);
1061                 list_splice_init(&dwc->active_list, &list);
1062
1063                 spin_unlock_irqrestore(&dwc->lock, flags);
1064
1065                 /* Flush all pending and queued descriptors */
1066                 list_for_each_entry_safe(desc, _desc, &list, desc_node)
1067                         dwc_descriptor_complete(dwc, desc, false);
1068         } else if (cmd == DMA_SLAVE_CONFIG) {
1069                 return set_runtime_config(chan, (struct dma_slave_config *)arg);
1070         } else {
1071                 return -ENXIO;
1072         }
1073
1074         return 0;
1075 }
1076
1077 static inline u32 dwc_get_residue(struct dw_dma_chan *dwc)
1078 {
1079         unsigned long flags;
1080         u32 residue;
1081
1082         spin_lock_irqsave(&dwc->lock, flags);
1083
1084         residue = dwc->residue;
1085         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue)
1086                 residue -= dwc_get_sent(dwc);
1087
1088         spin_unlock_irqrestore(&dwc->lock, flags);
1089         return residue;
1090 }
1091
1092 static enum dma_status
1093 dwc_tx_status(struct dma_chan *chan,
1094               dma_cookie_t cookie,
1095               struct dma_tx_state *txstate)
1096 {
1097         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1098         enum dma_status         ret;
1099
1100         ret = dma_cookie_status(chan, cookie, txstate);
1101         if (ret == DMA_SUCCESS)
1102                 return ret;
1103
1104         dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1105
1106         ret = dma_cookie_status(chan, cookie, txstate);
1107         if (ret != DMA_SUCCESS)
1108                 dma_set_residue(txstate, dwc_get_residue(dwc));
1109
1110         if (dwc->paused && ret == DMA_IN_PROGRESS)
1111                 return DMA_PAUSED;
1112
1113         return ret;
1114 }
1115
1116 static void dwc_issue_pending(struct dma_chan *chan)
1117 {
1118         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1119
1120         if (!list_empty(&dwc->queue))
1121                 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1122 }
1123
1124 static int dwc_alloc_chan_resources(struct dma_chan *chan)
1125 {
1126         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1127         struct dw_dma           *dw = to_dw_dma(chan->device);
1128         struct dw_desc          *desc;
1129         int                     i;
1130         unsigned long           flags;
1131
1132         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1133
1134         /* ASSERT:  channel is idle */
1135         if (dma_readl(dw, CH_EN) & dwc->mask) {
1136                 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
1137                 return -EIO;
1138         }
1139
1140         dma_cookie_init(chan);
1141
1142         /*
1143          * NOTE: some controllers may have additional features that we
1144          * need to initialize here, like "scatter-gather" (which
1145          * doesn't mean what you think it means), and status writeback.
1146          */
1147
1148         dwc_set_masters(dwc);
1149
1150         spin_lock_irqsave(&dwc->lock, flags);
1151         i = dwc->descs_allocated;
1152         while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
1153                 dma_addr_t phys;
1154
1155                 spin_unlock_irqrestore(&dwc->lock, flags);
1156
1157                 desc = dma_pool_alloc(dw->desc_pool, GFP_ATOMIC, &phys);
1158                 if (!desc)
1159                         goto err_desc_alloc;
1160
1161                 memset(desc, 0, sizeof(struct dw_desc));
1162
1163                 INIT_LIST_HEAD(&desc->tx_list);
1164                 dma_async_tx_descriptor_init(&desc->txd, chan);
1165                 desc->txd.tx_submit = dwc_tx_submit;
1166                 desc->txd.flags = DMA_CTRL_ACK;
1167                 desc->txd.phys = phys;
1168
1169                 dwc_desc_put(dwc, desc);
1170
1171                 spin_lock_irqsave(&dwc->lock, flags);
1172                 i = ++dwc->descs_allocated;
1173         }
1174
1175         spin_unlock_irqrestore(&dwc->lock, flags);
1176
1177         dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1178
1179         return i;
1180
1181 err_desc_alloc:
1182         dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);
1183
1184         return i;
1185 }
1186
1187 static void dwc_free_chan_resources(struct dma_chan *chan)
1188 {
1189         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1190         struct dw_dma           *dw = to_dw_dma(chan->device);
1191         struct dw_desc          *desc, *_desc;
1192         unsigned long           flags;
1193         LIST_HEAD(list);
1194
1195         dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1196                         dwc->descs_allocated);
1197
1198         /* ASSERT:  channel is idle */
1199         BUG_ON(!list_empty(&dwc->active_list));
1200         BUG_ON(!list_empty(&dwc->queue));
1201         BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1202
1203         spin_lock_irqsave(&dwc->lock, flags);
1204         list_splice_init(&dwc->free_list, &list);
1205         dwc->descs_allocated = 0;
1206         dwc->initialized = false;
1207         dwc->request_line = ~0;
1208
1209         /* Disable interrupts */
1210         channel_clear_bit(dw, MASK.XFER, dwc->mask);
1211         channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1212
1213         spin_unlock_irqrestore(&dwc->lock, flags);
1214
1215         list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1216                 dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
1217                 dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
1218         }
1219
1220         dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1221 }
1222
1223 /* --------------------- Cyclic DMA API extensions -------------------- */
1224
1225 /**
1226  * dw_dma_cyclic_start - start the cyclic DMA transfer
1227  * @chan: the DMA channel to start
1228  *
1229  * Must be called with soft interrupts disabled. Returns zero on success or
1230  * -errno on failure.
1231  */
1232 int dw_dma_cyclic_start(struct dma_chan *chan)
1233 {
1234         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1235         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1236         unsigned long           flags;
1237
1238         if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1239                 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1240                 return -ENODEV;
1241         }
1242
1243         spin_lock_irqsave(&dwc->lock, flags);
1244
1245         /* Assert channel is idle */
1246         if (dma_readl(dw, CH_EN) & dwc->mask) {
1247                 dev_err(chan2dev(&dwc->chan),
1248                         "BUG: Attempted to start non-idle channel\n");
1249                 dwc_dump_chan_regs(dwc);
1250                 spin_unlock_irqrestore(&dwc->lock, flags);
1251                 return -EBUSY;
1252         }
1253
1254         dma_writel(dw, CLEAR.ERROR, dwc->mask);
1255         dma_writel(dw, CLEAR.XFER, dwc->mask);
1256
1257         /* Setup DMAC channel registers */
1258         channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1259         channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1260         channel_writel(dwc, CTL_HI, 0);
1261
1262         channel_set_bit(dw, CH_EN, dwc->mask);
1263
1264         spin_unlock_irqrestore(&dwc->lock, flags);
1265
1266         return 0;
1267 }
1268 EXPORT_SYMBOL(dw_dma_cyclic_start);
1269
1270 /**
1271  * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1272  * @chan: the DMA channel to stop
1273  *
1274  * Must be called with soft interrupts disabled.
1275  */
1276 void dw_dma_cyclic_stop(struct dma_chan *chan)
1277 {
1278         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1279         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1280         unsigned long           flags;
1281
1282         spin_lock_irqsave(&dwc->lock, flags);
1283
1284         dwc_chan_disable(dw, dwc);
1285
1286         spin_unlock_irqrestore(&dwc->lock, flags);
1287 }
1288 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1289
1290 /**
1291  * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1292  * @chan: the DMA channel to prepare
1293  * @buf_addr: physical DMA address where the buffer starts
1294  * @buf_len: total number of bytes for the entire buffer
1295  * @period_len: number of bytes for each period
1296  * @direction: transfer direction, to or from device
1297  *
1298  * Must be called before trying to start the transfer. Returns a valid struct
1299  * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1300  */
1301 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1302                 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1303                 enum dma_transfer_direction direction)
1304 {
1305         struct dw_dma_chan              *dwc = to_dw_dma_chan(chan);
1306         struct dma_slave_config         *sconfig = &dwc->dma_sconfig;
1307         struct dw_cyclic_desc           *cdesc;
1308         struct dw_cyclic_desc           *retval = NULL;
1309         struct dw_desc                  *desc;
1310         struct dw_desc                  *last = NULL;
1311         unsigned long                   was_cyclic;
1312         unsigned int                    reg_width;
1313         unsigned int                    periods;
1314         unsigned int                    i;
1315         unsigned long                   flags;
1316
1317         spin_lock_irqsave(&dwc->lock, flags);
1318         if (dwc->nollp) {
1319                 spin_unlock_irqrestore(&dwc->lock, flags);
1320                 dev_dbg(chan2dev(&dwc->chan),
1321                                 "channel doesn't support LLP transfers\n");
1322                 return ERR_PTR(-EINVAL);
1323         }
1324
1325         if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1326                 spin_unlock_irqrestore(&dwc->lock, flags);
1327                 dev_dbg(chan2dev(&dwc->chan),
1328                                 "queue and/or active list are not empty\n");
1329                 return ERR_PTR(-EBUSY);
1330         }
1331
1332         was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1333         spin_unlock_irqrestore(&dwc->lock, flags);
1334         if (was_cyclic) {
1335                 dev_dbg(chan2dev(&dwc->chan),
1336                                 "channel already prepared for cyclic DMA\n");
1337                 return ERR_PTR(-EBUSY);
1338         }
1339
1340         retval = ERR_PTR(-EINVAL);
1341
1342         if (unlikely(!is_slave_direction(direction)))
1343                 goto out_err;
1344
1345         dwc->direction = direction;
1346
1347         if (direction == DMA_MEM_TO_DEV)
1348                 reg_width = __ffs(sconfig->dst_addr_width);
1349         else
1350                 reg_width = __ffs(sconfig->src_addr_width);
1351
1352         periods = buf_len / period_len;
1353
1354         /* Check for too big/unaligned periods and unaligned DMA buffer. */
1355         if (period_len > (dwc->block_size << reg_width))
1356                 goto out_err;
1357         if (unlikely(period_len & ((1 << reg_width) - 1)))
1358                 goto out_err;
1359         if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1360                 goto out_err;
1361
1362         retval = ERR_PTR(-ENOMEM);
1363
1364         if (periods > NR_DESCS_PER_CHANNEL)
1365                 goto out_err;
1366
1367         cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1368         if (!cdesc)
1369                 goto out_err;
1370
1371         cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1372         if (!cdesc->desc)
1373                 goto out_err_alloc;
1374
1375         for (i = 0; i < periods; i++) {
1376                 desc = dwc_desc_get(dwc);
1377                 if (!desc)
1378                         goto out_err_desc_get;
1379
1380                 switch (direction) {
1381                 case DMA_MEM_TO_DEV:
1382                         desc->lli.dar = sconfig->dst_addr;
1383                         desc->lli.sar = buf_addr + (period_len * i);
1384                         desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1385                                         | DWC_CTLL_DST_WIDTH(reg_width)
1386                                         | DWC_CTLL_SRC_WIDTH(reg_width)
1387                                         | DWC_CTLL_DST_FIX
1388                                         | DWC_CTLL_SRC_INC
1389                                         | DWC_CTLL_INT_EN);
1390
1391                         desc->lli.ctllo |= sconfig->device_fc ?
1392                                 DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
1393                                 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
1394
1395                         break;
1396                 case DMA_DEV_TO_MEM:
1397                         desc->lli.dar = buf_addr + (period_len * i);
1398                         desc->lli.sar = sconfig->src_addr;
1399                         desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1400                                         | DWC_CTLL_SRC_WIDTH(reg_width)
1401                                         | DWC_CTLL_DST_WIDTH(reg_width)
1402                                         | DWC_CTLL_DST_INC
1403                                         | DWC_CTLL_SRC_FIX
1404                                         | DWC_CTLL_INT_EN);
1405
1406                         desc->lli.ctllo |= sconfig->device_fc ?
1407                                 DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
1408                                 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
1409
1410                         break;
1411                 default:
1412                         break;
1413                 }
1414
1415                 desc->lli.ctlhi = (period_len >> reg_width);
1416                 cdesc->desc[i] = desc;
1417
1418                 if (last)
1419                         last->lli.llp = desc->txd.phys;
1420
1421                 last = desc;
1422         }
1423
1424         /* Let's make a cyclic list */
1425         last->lli.llp = cdesc->desc[0]->txd.phys;
1426
1427         dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
1428                         "period %zu periods %d\n", (unsigned long long)buf_addr,
1429                         buf_len, period_len, periods);
1430
1431         cdesc->periods = periods;
1432         dwc->cdesc = cdesc;
1433
1434         return cdesc;
1435
1436 out_err_desc_get:
1437         while (i--)
1438                 dwc_desc_put(dwc, cdesc->desc[i]);
1439 out_err_alloc:
1440         kfree(cdesc);
1441 out_err:
1442         clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1443         return (struct dw_cyclic_desc *)retval;
1444 }
1445 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1446
1447 /**
1448  * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1449  * @chan: the DMA channel to free
1450  */
1451 void dw_dma_cyclic_free(struct dma_chan *chan)
1452 {
1453         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1454         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1455         struct dw_cyclic_desc   *cdesc = dwc->cdesc;
1456         int                     i;
1457         unsigned long           flags;
1458
1459         dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
1460
1461         if (!cdesc)
1462                 return;
1463
1464         spin_lock_irqsave(&dwc->lock, flags);
1465
1466         dwc_chan_disable(dw, dwc);
1467
1468         dma_writel(dw, CLEAR.ERROR, dwc->mask);
1469         dma_writel(dw, CLEAR.XFER, dwc->mask);
1470
1471         spin_unlock_irqrestore(&dwc->lock, flags);
1472
1473         for (i = 0; i < cdesc->periods; i++)
1474                 dwc_desc_put(dwc, cdesc->desc[i]);
1475
1476         kfree(cdesc->desc);
1477         kfree(cdesc);
1478
1479         clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1480 }
1481 EXPORT_SYMBOL(dw_dma_cyclic_free);
1482
1483 /*----------------------------------------------------------------------*/
1484
1485 static void dw_dma_off(struct dw_dma *dw)
1486 {
1487         int i;
1488
1489         dma_writel(dw, CFG, 0);
1490
1491         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1492         channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1493         channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1494         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1495
1496         while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1497                 cpu_relax();
1498
1499         for (i = 0; i < dw->dma.chancnt; i++)
1500                 dw->chan[i].initialized = false;
1501 }
1502
1503 int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata)
1504 {
1505         struct dw_dma           *dw;
1506         size_t                  size;
1507         bool                    autocfg;
1508         unsigned int            dw_params;
1509         unsigned int            nr_channels;
1510         unsigned int            max_blk_size = 0;
1511         int                     err;
1512         int                     i;
1513
1514         dw_params = dma_read_byaddr(chip->regs, DW_PARAMS);
1515         autocfg = dw_params >> DW_PARAMS_EN & 0x1;
1516
1517         dev_dbg(chip->dev, "DW_PARAMS: 0x%08x\n", dw_params);
1518
1519         if (!pdata && autocfg) {
1520                 pdata = devm_kzalloc(chip->dev, sizeof(*pdata), GFP_KERNEL);
1521                 if (!pdata)
1522                         return -ENOMEM;
1523
1524                 /* Fill platform data with the default values */
1525                 pdata->is_private = true;
1526                 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1527                 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1528         } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1529                 return -EINVAL;
1530
1531         if (autocfg)
1532                 nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1;
1533         else
1534                 nr_channels = pdata->nr_channels;
1535
1536         size = sizeof(struct dw_dma) + nr_channels * sizeof(struct dw_dma_chan);
1537         dw = devm_kzalloc(chip->dev, size, GFP_KERNEL);
1538         if (!dw)
1539                 return -ENOMEM;
1540
1541         dw->clk = devm_clk_get(chip->dev, "hclk");
1542         if (IS_ERR(dw->clk))
1543                 return PTR_ERR(dw->clk);
1544         clk_prepare_enable(dw->clk);
1545
1546         dw->regs = chip->regs;
1547         chip->dw = dw;
1548
1549         /* Get hardware configuration parameters */
1550         if (autocfg) {
1551                 max_blk_size = dma_readl(dw, MAX_BLK_SIZE);
1552
1553                 dw->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1554                 for (i = 0; i < dw->nr_masters; i++) {
1555                         dw->data_width[i] =
1556                                 (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3) + 2;
1557                 }
1558         } else {
1559                 dw->nr_masters = pdata->nr_masters;
1560                 memcpy(dw->data_width, pdata->data_width, 4);
1561         }
1562
1563         /* Calculate all channel mask before DMA setup */
1564         dw->all_chan_mask = (1 << nr_channels) - 1;
1565
1566         /* Force dma off, just in case */
1567         dw_dma_off(dw);
1568
1569         /* Disable BLOCK interrupts as well */
1570         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1571
1572         err = devm_request_irq(chip->dev, chip->irq, dw_dma_interrupt,
1573                                IRQF_SHARED, "dw_dmac", dw);
1574         if (err)
1575                 return err;
1576
1577         /* Create a pool of consistent memory blocks for hardware descriptors */
1578         dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev,
1579                                          sizeof(struct dw_desc), 4, 0);
1580         if (!dw->desc_pool) {
1581                 dev_err(chip->dev, "No memory for descriptors dma pool\n");
1582                 return -ENOMEM;
1583         }
1584
1585         tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1586
1587         INIT_LIST_HEAD(&dw->dma.channels);
1588         for (i = 0; i < nr_channels; i++) {
1589                 struct dw_dma_chan      *dwc = &dw->chan[i];
1590                 int                     r = nr_channels - i - 1;
1591
1592                 dwc->chan.device = &dw->dma;
1593                 dma_cookie_init(&dwc->chan);
1594                 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1595                         list_add_tail(&dwc->chan.device_node,
1596                                         &dw->dma.channels);
1597                 else
1598                         list_add(&dwc->chan.device_node, &dw->dma.channels);
1599
1600                 /* 7 is highest priority & 0 is lowest. */
1601                 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1602                         dwc->priority = r;
1603                 else
1604                         dwc->priority = i;
1605
1606                 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1607                 spin_lock_init(&dwc->lock);
1608                 dwc->mask = 1 << i;
1609
1610                 INIT_LIST_HEAD(&dwc->active_list);
1611                 INIT_LIST_HEAD(&dwc->queue);
1612                 INIT_LIST_HEAD(&dwc->free_list);
1613
1614                 channel_clear_bit(dw, CH_EN, dwc->mask);
1615
1616                 dwc->direction = DMA_TRANS_NONE;
1617                 dwc->request_line = ~0;
1618
1619                 /* Hardware configuration */
1620                 if (autocfg) {
1621                         unsigned int dwc_params;
1622                         void __iomem *addr = chip->regs + r * sizeof(u32);
1623
1624                         dwc_params = dma_read_byaddr(addr, DWC_PARAMS);
1625
1626                         dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i,
1627                                            dwc_params);
1628
1629                         /* Decode maximum block size for given channel. The
1630                          * stored 4 bit value represents blocks from 0x00 for 3
1631                          * up to 0x0a for 4095. */
1632                         dwc->block_size =
1633                                 (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1;
1634                         dwc->nollp =
1635                                 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1636                 } else {
1637                         dwc->block_size = pdata->block_size;
1638
1639                         /* Check if channel supports multi block transfer */
1640                         channel_writel(dwc, LLP, 0xfffffffc);
1641                         dwc->nollp =
1642                                 (channel_readl(dwc, LLP) & 0xfffffffc) == 0;
1643                         channel_writel(dwc, LLP, 0);
1644                 }
1645         }
1646
1647         /* Clear all interrupts on all channels. */
1648         dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1649         dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1650         dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1651         dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1652         dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1653
1654         dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1655         dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1656         if (pdata->is_private)
1657                 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1658         dw->dma.dev = chip->dev;
1659         dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1660         dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1661
1662         dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1663
1664         dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1665         dw->dma.device_control = dwc_control;
1666
1667         dw->dma.device_tx_status = dwc_tx_status;
1668         dw->dma.device_issue_pending = dwc_issue_pending;
1669
1670         dma_writel(dw, CFG, DW_CFG_DMA_EN);
1671
1672         dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n",
1673                  nr_channels);
1674
1675         dma_async_device_register(&dw->dma);
1676
1677         return 0;
1678 }
1679 EXPORT_SYMBOL_GPL(dw_dma_probe);
1680
1681 int dw_dma_remove(struct dw_dma_chip *chip)
1682 {
1683         struct dw_dma           *dw = chip->dw;
1684         struct dw_dma_chan      *dwc, *_dwc;
1685
1686         dw_dma_off(dw);
1687         dma_async_device_unregister(&dw->dma);
1688
1689         tasklet_kill(&dw->tasklet);
1690
1691         list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1692                         chan.device_node) {
1693                 list_del(&dwc->chan.device_node);
1694                 channel_clear_bit(dw, CH_EN, dwc->mask);
1695         }
1696
1697         return 0;
1698 }
1699 EXPORT_SYMBOL_GPL(dw_dma_remove);
1700
1701 void dw_dma_shutdown(struct dw_dma_chip *chip)
1702 {
1703         struct dw_dma *dw = chip->dw;
1704
1705         dw_dma_off(dw);
1706         clk_disable_unprepare(dw->clk);
1707 }
1708 EXPORT_SYMBOL_GPL(dw_dma_shutdown);
1709
1710 #ifdef CONFIG_PM_SLEEP
1711
1712 int dw_dma_suspend(struct dw_dma_chip *chip)
1713 {
1714         struct dw_dma *dw = chip->dw;
1715
1716         dw_dma_off(dw);
1717         clk_disable_unprepare(dw->clk);
1718
1719         return 0;
1720 }
1721 EXPORT_SYMBOL_GPL(dw_dma_suspend);
1722
1723 int dw_dma_resume(struct dw_dma_chip *chip)
1724 {
1725         struct dw_dma *dw = chip->dw;
1726
1727         clk_prepare_enable(dw->clk);
1728         dma_writel(dw, CFG, DW_CFG_DMA_EN);
1729
1730         return 0;
1731 }
1732 EXPORT_SYMBOL_GPL(dw_dma_resume);
1733
1734 #endif /* CONFIG_PM_SLEEP */
1735
1736 MODULE_LICENSE("GPL v2");
1737 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller core driver");
1738 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1739 MODULE_AUTHOR("Viresh Kumar <viresh.linux@gmail.com>");