drivers/dma/ste_dma40.c

   1 /*
   2  * driver/dma/ste_dma40.c
   3  *
   4  * Copyright (C) ST-Ericsson 2007-2010
   5  * License terms: GNU General Public License (GPL) version 2
   6  * Author: Per Friden <per.friden@stericsson.com>
   7  * Author: Jonas Aaberg <jonas.aberg@stericsson.com>
   8  *
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/slab.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/platform_device.h>
  15 #include <linux/clk.h>
  16 #include <linux/delay.h>
  17
  18 #include <plat/ste_dma40.h>
  19
  20 #include "ste_dma40_ll.h"
  21
  22 #define D40_NAME "dma40"
  23
  24 #define D40_PHY_CHAN -1
  25
  26 /* For masking out/in 2 bit channel positions */
  27 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
  28 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  29
  30 /* Maximum iterations taken before giving up suspending a channel */
  31 #define D40_SUSPEND_MAX_IT 500
  32
  33 #define D40_ALLOC_FREE          (1 << 31)
  34 #define D40_ALLOC_PHY           (1 << 30)
  35 #define D40_ALLOC_LOG_FREE      0
  36
  37 /* The number of free d40_desc to keep in memory before starting
  38  * to kfree() them */
  39 #define D40_DESC_CACHE_SIZE 50
  40
  41 /* Hardware designer of the block */
  42 #define D40_PERIPHID2_DESIGNER 0x8
  43
  44 /**
  45  * enum 40_command - The different commands and/or statuses.
  46  *
  47  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  48  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  49  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  50  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  51  */
  52 enum d40_command {
  53         D40_DMA_STOP            = 0,
  54         D40_DMA_RUN             = 1,
  55         D40_DMA_SUSPEND_REQ     = 2,
  56         D40_DMA_SUSPENDED       = 3
  57 };
  58
  59 /**
  60  * struct d40_lli_pool - Structure for keeping LLIs in memory
  61  *
  62  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  63  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  64  * pre_alloc_lli is used.
  65  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  66  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  67  * one buffer to one buffer.
  68  */
  69 struct d40_lli_pool {
  70         void    *base;
  71         int     size;
  72         /* Space for dst and src, plus an extra for padding */
  73         u8      pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  74 };
  75
  76 /**
  77  * struct d40_desc - A descriptor is one DMA job.
  78  *
  79  * @lli_phy: LLI settings for physical channel. Both src and dst=
  80  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  81  * lli_len equals one.
  82  * @lli_log: Same as above but for logical channels.
  83  * @lli_pool: The pool with two entries pre-allocated.
  84  * @lli_len: Number of llis of current descriptor.
  85  * @lli_count: Number of transfered llis.
  86  * @lli_tx_len: Max number of LLIs per transfer, there can be
  87  * many transfer for one descriptor.
  88  * @txd: DMA engine struct. Used for among other things for communication
  89  * during a transfer.
  90  * @node: List entry.
  91  * @dir: The transfer direction of this job.
  92  * @is_in_client_list: true if the client owns this descriptor.
  93  *
  94  * This descriptor is used for both logical and physical transfers.
  95  */
  96
  97 struct d40_desc {
  98         /* LLI physical */
  99         struct d40_phy_lli_bidir         lli_phy;
 100         /* LLI logical */
 101         struct d40_log_lli_bidir         lli_log;
 102
 103         struct d40_lli_pool              lli_pool;
 104         int                              lli_len;
 105         int                              lli_count;
 106         u32                              lli_tx_len;
 107
 108         struct dma_async_tx_descriptor   txd;
 109         struct list_head                 node;
 110
 111         enum dma_data_direction          dir;
 112         bool                             is_in_client_list;
 113 };
 114
 115 /**
 116  * struct d40_lcla_pool - LCLA pool settings and data.
 117  *
 118  * @base: The virtual address of LCLA.
 119  * @phy: Physical base address of LCLA.
 120  * @base_size: size of lcla.
 121  * @lock: Lock to protect the content in this struct.
 122  * @alloc_map: Mapping between physical channel and LCLA entries.
 123  * @num_blocks: The number of entries of alloc_map. Equals to the
 124  * number of physical channels.
 125  */
 126 struct d40_lcla_pool {
 127         void            *base;
 128         dma_addr_t       phy;
 129         resource_size_t  base_size;
 130         spinlock_t       lock;
 131         u32             *alloc_map;
 132         int              num_blocks;
 133 };
 134
 135 /**
 136  * struct d40_phy_res - struct for handling eventlines mapped to physical
 137  * channels.
 138  *
 139  * @lock: A lock protection this entity.
 140  * @num: The physical channel number of this entity.
 141  * @allocated_src: Bit mapped to show which src event line's are mapped to
 142  * this physical channel. Can also be free or physically allocated.
 143  * @allocated_dst: Same as for src but is dst.
 144  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
 145  * event line number. Both allocated_src and allocated_dst can not be
 146  * allocated to a physical channel, since the interrupt handler has then
 147  * no way of figure out which one the interrupt belongs to.
 148  */
 149 struct d40_phy_res {
 150         spinlock_t lock;
 151         int        num;
 152         u32        allocated_src;
 153         u32        allocated_dst;
 154 };
 155
 156 struct d40_base;
 157
 158 /**
 159  * struct d40_chan - Struct that describes a channel.
 160  *
 161  * @lock: A spinlock to protect this struct.
 162  * @log_num: The logical number, if any of this channel.
 163  * @completed: Starts with 1, after first interrupt it is set to dma engine's
 164  * current cookie.
 165  * @pending_tx: The number of pending transfers. Used between interrupt handler
 166  * and tasklet.
 167  * @busy: Set to true when transfer is ongoing on this channel.
 168  * @phy_chan: Pointer to physical channel which this instance runs on.
 169  * @chan: DMA engine handle.
 170  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
 171  * transfer and call client callback.
 172  * @client: Cliented owned descriptor list.
 173  * @active: Active descriptor.
 174  * @queue: Queued jobs.
 175  * @free: List of free descripts, ready to be reused.
 176  * @free_len: Number of descriptors in the free list.
 177  * @dma_cfg: The client configuration of this dma channel.
 178  * @base: Pointer to the device instance struct.
 179  * @src_def_cfg: Default cfg register setting for src.
 180  * @dst_def_cfg: Default cfg register setting for dst.
 181  * @log_def: Default logical channel settings.
 182  * @lcla: Space for one dst src pair for logical channel transfers.
 183  * @lcpa: Pointer to dst and src lcpa settings.
 184  *
 185  * This struct can either "be" a logical or a physical channel.
 186  */
 187 struct d40_chan {
 188         spinlock_t                       lock;
 189         int                              log_num;
 190         /* ID of the most recent completed transfer */
 191         int                              completed;
 192         int                              pending_tx;
 193         bool                             busy;
 194         struct d40_phy_res              *phy_chan;
 195         struct dma_chan                  chan;
 196         struct tasklet_struct            tasklet;
 197         struct list_head                 client;
 198         struct list_head                 active;
 199         struct list_head                 queue;
 200         struct list_head                 free;
 201         int                              free_len;
 202         struct stedma40_chan_cfg         dma_cfg;
 203         struct d40_base                 *base;
 204         /* Default register configurations */
 205         u32                              src_def_cfg;
 206         u32                              dst_def_cfg;
 207         struct d40_def_lcsp              log_def;
 208         struct d40_lcla_elem             lcla;
 209         struct d40_log_lli_full         *lcpa;
 210 };
 211
 212 /**
 213  * struct d40_base - The big global struct, one for each probe'd instance.
 214  *
 215  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
 216  * @execmd_lock: Lock for execute command usage since several channels share
 217  * the same physical register.
 218  * @dev: The device structure.
 219  * @virtbase: The virtual base address of the DMA's register.
 220  * @clk: Pointer to the DMA clock structure.
 221  * @phy_start: Physical memory start of the DMA registers.
 222  * @phy_size: Size of the DMA register map.
 223  * @irq: The IRQ number.
 224  * @num_phy_chans: The number of physical channels. Read from HW. This
 225  * is the number of available channels for this driver, not counting "Secure
 226  * mode" allocated physical channels.
 227  * @num_log_chans: The number of logical channels. Calculated from
 228  * num_phy_chans.
 229  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
 230  * @dma_slave: dma_device channels that can do only do slave transfers.
 231  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
 232  * @phy_chans: Room for all possible physical channels in system.
 233  * @log_chans: Room for all possible logical channels in system.
 234  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
 235  * to log_chans entries.
 236  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
 237  * to phy_chans entries.
 238  * @plat_data: Pointer to provided platform_data which is the driver
 239  * configuration.
 240  * @phy_res: Vector containing all physical channels.
 241  * @lcla_pool: lcla pool settings and data.
 242  * @lcpa_base: The virtual mapped address of LCPA.
 243  * @phy_lcpa: The physical address of the LCPA.
 244  * @lcpa_size: The size of the LCPA area.
 245  */
 246 struct d40_base {
 247         spinlock_t                       interrupt_lock;
 248         spinlock_t                       execmd_lock;
 249         struct device                    *dev;
 250         void __iomem                     *virtbase;
 251         struct clk                       *clk;
 252         phys_addr_t                       phy_start;
 253         resource_size_t                   phy_size;
 254         int                               irq;
 255         int                               num_phy_chans;
 256         int                               num_log_chans;
 257         struct dma_device                 dma_both;
 258         struct dma_device                 dma_slave;
 259         struct dma_device                 dma_memcpy;
 260         struct d40_chan                  *phy_chans;
 261         struct d40_chan                  *log_chans;
 262         struct d40_chan                 **lookup_log_chans;
 263         struct d40_chan                 **lookup_phy_chans;
 264         struct stedma40_platform_data    *plat_data;
 265         /* Physical half channels */
 266         struct d40_phy_res               *phy_res;
 267         struct d40_lcla_pool              lcla_pool;
 268         void                             *lcpa_base;
 269         dma_addr_t                        phy_lcpa;
 270         resource_size_t                   lcpa_size;
 271 };
 272
 273 /**
 274  * struct d40_interrupt_lookup - lookup table for interrupt handler
 275  *
 276  * @src: Interrupt mask register.
 277  * @clr: Interrupt clear register.
 278  * @is_error: true if this is an error interrupt.
 279  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
 280  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
 281  */
 282 struct d40_interrupt_lookup {
 283         u32 src;
 284         u32 clr;
 285         bool is_error;
 286         int offset;
 287 };
 288
 289 /**
 290  * struct d40_reg_val - simple lookup struct
 291  *
 292  * @reg: The register.
 293  * @val: The value that belongs to the register in reg.
 294  */
 295 struct d40_reg_val {
 296         unsigned int reg;
 297         unsigned int val;
 298 };
 299
 300 static int d40_pool_lli_alloc(struct d40_desc *d40d,
 301                               int lli_len, bool is_log)
 302 {
 303         u32 align;
 304         void *base;
 305
 306         if (is_log)
 307                 align = sizeof(struct d40_log_lli);
 308         else
 309                 align = sizeof(struct d40_phy_lli);
 310
 311         if (lli_len == 1) {
 312                 base = d40d->lli_pool.pre_alloc_lli;
 313                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 314                 d40d->lli_pool.base = NULL;
 315         } else {
 316                 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
 317
 318                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 319                 d40d->lli_pool.base = base;
 320
 321                 if (d40d->lli_pool.base == NULL)
 322                         return -ENOMEM;
 323         }
 324
 325         if (is_log) {
 326                 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
 327                                               align);
 328                 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
 329                                               align);
 330         } else {
 331                 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
 332                                               align);
 333                 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
 334                                               align);
 335
 336                 d40d->lli_phy.src_addr = virt_to_phys(d40d->lli_phy.src);
 337                 d40d->lli_phy.dst_addr = virt_to_phys(d40d->lli_phy.dst);
 338         }
 339
 340         return 0;
 341 }
 342
 343 static void d40_pool_lli_free(struct d40_desc *d40d)
 344 {
 345         kfree(d40d->lli_pool.base);
 346         d40d->lli_pool.base = NULL;
 347         d40d->lli_pool.size = 0;
 348         d40d->lli_log.src = NULL;
 349         d40d->lli_log.dst = NULL;
 350         d40d->lli_phy.src = NULL;
 351         d40d->lli_phy.dst = NULL;
 352         d40d->lli_phy.src_addr = 0;
 353         d40d->lli_phy.dst_addr = 0;
 354 }
 355
 356 static dma_cookie_t d40_assign_cookie(struct d40_chan *d40c,
 357                                       struct d40_desc *desc)
 358 {
 359         dma_cookie_t cookie = d40c->chan.cookie;
 360
 361         if (++cookie < 0)
 362                 cookie = 1;
 363
 364         d40c->chan.cookie = cookie;
 365         desc->txd.cookie = cookie;
 366
 367         return cookie;
 368 }
 369
 370 static void d40_desc_remove(struct d40_desc *d40d)
 371 {
 372         list_del(&d40d->node);
 373 }
 374
 375 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 376 {
 377         struct d40_desc *desc;
 378         struct d40_desc *d;
 379         struct d40_desc *_d;
 380
 381         if (!list_empty(&d40c->client)) {
 382                 list_for_each_entry_safe(d, _d, &d40c->client, node)
 383                         if (async_tx_test_ack(&d->txd)) {
 384                                 d40_pool_lli_free(d);
 385                                 d40_desc_remove(d);
 386                                 desc = d;
 387                                 goto out;
 388                         }
 389         }
 390
 391         if (list_empty(&d40c->free)) {
 392                 /* Alloc new desc because we're out of used ones */
 393                 desc = kzalloc(sizeof(struct d40_desc), GFP_NOWAIT);
 394                 if (desc == NULL)
 395                         goto out;
 396                 INIT_LIST_HEAD(&desc->node);
 397         } else {
 398                 /* Reuse an old desc. */
 399                 desc = list_first_entry(&d40c->free,
 400                                         struct d40_desc,
 401                                         node);
 402                 list_del(&desc->node);
 403                 d40c->free_len--;
 404         }
 405 out:
 406         return desc;
 407 }
 408
 409 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 410 {
 411         if (d40c->free_len < D40_DESC_CACHE_SIZE) {
 412                 list_add_tail(&d40d->node, &d40c->free);
 413                 d40c->free_len++;
 414         } else
 415                 kfree(d40d);
 416 }
 417
 418 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 419 {
 420         list_add_tail(&desc->node, &d40c->active);
 421 }
 422
 423 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 424 {
 425         struct d40_desc *d;
 426
 427         if (list_empty(&d40c->active))
 428                 return NULL;
 429
 430         d = list_first_entry(&d40c->active,
 431                              struct d40_desc,
 432                              node);
 433         return d;
 434 }
 435
 436 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 437 {
 438         list_add_tail(&desc->node, &d40c->queue);
 439 }
 440
 441 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 442 {
 443         struct d40_desc *d;
 444
 445         if (list_empty(&d40c->queue))
 446                 return NULL;
 447
 448         d = list_first_entry(&d40c->queue,
 449                              struct d40_desc,
 450                              node);
 451         return d;
 452 }
 453
 454 /* Support functions for logical channels */
 455
 456 static int d40_lcla_id_get(struct d40_chan *d40c,
 457                            struct d40_lcla_pool *pool)
 458 {
 459         int src_id = 0;
 460         int dst_id = 0;
 461         struct d40_log_lli *lcla_lidx_base =
 462                 pool->base + d40c->phy_chan->num * 1024;
 463         int i;
 464         int lli_per_log = d40c->base->plat_data->llis_per_log;
 465
 466         if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0)
 467                 return 0;
 468
 469         if (pool->num_blocks > 32)
 470                 return -EINVAL;
 471
 472         spin_lock(&pool->lock);
 473
 474         for (i = 0; i < pool->num_blocks; i++) {
 475                 if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) {
 476                         pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i);
 477                         break;
 478                 }
 479         }
 480         src_id = i;
 481         if (src_id >= pool->num_blocks)
 482                 goto err;
 483
 484         for (; i < pool->num_blocks; i++) {
 485                 if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) {
 486                         pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i);
 487                         break;
 488                 }
 489         }
 490
 491         dst_id = i;
 492         if (dst_id == src_id)
 493                 goto err;
 494
 495         d40c->lcla.src_id = src_id;
 496         d40c->lcla.dst_id = dst_id;
 497         d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1;
 498         d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1;
 499
 500
 501         spin_unlock(&pool->lock);
 502         return 0;
 503 err:
 504         spin_unlock(&pool->lock);
 505         return -EINVAL;
 506 }
 507
 508 static void d40_lcla_id_put(struct d40_chan *d40c,
 509                             struct d40_lcla_pool *pool,
 510                             int id)
 511 {
 512         if (id < 0)
 513                 return;
 514
 515         d40c->lcla.src_id = -1;
 516         d40c->lcla.dst_id = -1;
 517
 518         spin_lock(&pool->lock);
 519         pool->alloc_map[d40c->phy_chan->num] &= (~(0x1 << id));
 520         spin_unlock(&pool->lock);
 521 }
 522
 523 static int d40_channel_execute_command(struct d40_chan *d40c,
 524                                        enum d40_command command)
 525 {
 526         int status, i;
 527         void __iomem *active_reg;
 528         int ret = 0;
 529         unsigned long flags;
 530
 531         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 532
 533         if (d40c->phy_chan->num % 2 == 0)
 534                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 535         else
 536                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 537
 538         if (command == D40_DMA_SUSPEND_REQ) {
 539                 status = (readl(active_reg) &
 540                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 541                         D40_CHAN_POS(d40c->phy_chan->num);
 542
 543                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 544                         goto done;
 545         }
 546
 547         writel(command << D40_CHAN_POS(d40c->phy_chan->num), active_reg);
 548
 549         if (command == D40_DMA_SUSPEND_REQ) {
 550
 551                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 552                         status = (readl(active_reg) &
 553                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 554                                 D40_CHAN_POS(d40c->phy_chan->num);
 555
 556                         cpu_relax();
 557                         /*
 558                          * Reduce the number of bus accesses while
 559                          * waiting for the DMA to suspend.
 560                          */
 561                         udelay(3);
 562
 563                         if (status == D40_DMA_STOP ||
 564                             status == D40_DMA_SUSPENDED)
 565                                 break;
 566                 }
 567
 568                 if (i == D40_SUSPEND_MAX_IT) {
 569                         dev_err(&d40c->chan.dev->device,
 570                                 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
 571                                 __func__, d40c->phy_chan->num, d40c->log_num,
 572                                 status);
 573                         dump_stack();
 574                         ret = -EBUSY;
 575                 }
 576
 577         }
 578 done:
 579         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 580         return ret;
 581 }
 582
 583 static void d40_term_all(struct d40_chan *d40c)
 584 {
 585         struct d40_desc *d40d;
 586         struct d40_desc *d;
 587         struct d40_desc *_d;
 588
 589         /* Release active descriptors */
 590         while ((d40d = d40_first_active_get(d40c))) {
 591                 d40_desc_remove(d40d);
 592
 593                 /* Return desc to free-list */
 594                 d40_desc_free(d40c, d40d);
 595         }
 596
 597         /* Release queued descriptors waiting for transfer */
 598         while ((d40d = d40_first_queued(d40c))) {
 599                 d40_desc_remove(d40d);
 600
 601                 /* Return desc to free-list */
 602                 d40_desc_free(d40c, d40d);
 603         }
 604
 605         /* Release client owned descriptors */
 606         if (!list_empty(&d40c->client))
 607                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
 608                         d40_pool_lli_free(d);
 609                         d40_desc_remove(d);
 610                         /* Return desc to free-list */
 611                         d40_desc_free(d40c, d40d);
 612                 }
 613
 614         d40_lcla_id_put(d40c, &d40c->base->lcla_pool,
 615                         d40c->lcla.src_id);
 616         d40_lcla_id_put(d40c, &d40c->base->lcla_pool,
 617                         d40c->lcla.dst_id);
 618
 619         d40c->pending_tx = 0;
 620         d40c->busy = false;
 621 }
 622
 623 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 624 {
 625         u32 val;
 626         unsigned long flags;
 627
 628         if (do_enable)
 629                 val = D40_ACTIVATE_EVENTLINE;
 630         else
 631                 val = D40_DEACTIVATE_EVENTLINE;
 632
 633         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 634
 635         /* Enable event line connected to device (or memcpy) */
 636         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 637             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 638                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 639
 640                 writel((val << D40_EVENTLINE_POS(event)) |
 641                        ~D40_EVENTLINE_MASK(event),
 642                        d40c->base->virtbase + D40_DREG_PCBASE +
 643                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 644                        D40_CHAN_REG_SSLNK);
 645         }
 646         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 647                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 648
 649                 writel((val << D40_EVENTLINE_POS(event)) |
 650                        ~D40_EVENTLINE_MASK(event),
 651                        d40c->base->virtbase + D40_DREG_PCBASE +
 652                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 653                        D40_CHAN_REG_SDLNK);
 654         }
 655
 656         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 657 }
 658
 659 static u32 d40_chan_has_events(struct d40_chan *d40c)
 660 {
 661         u32 val = 0;
 662
 663         /* If SSLNK or SDLNK is zero all events are disabled */
 664         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 665             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
 666                 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 667                             d40c->phy_chan->num * D40_DREG_PCDELTA +
 668                             D40_CHAN_REG_SSLNK);
 669
 670         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM)
 671                 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 672                             d40c->phy_chan->num * D40_DREG_PCDELTA +
 673                             D40_CHAN_REG_SDLNK);
 674         return val;
 675 }
 676
 677 static void d40_config_enable_lidx(struct d40_chan *d40c)
 678 {
 679         /* Set LIDX for lcla */
 680         writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 681                D40_SREG_ELEM_LOG_LIDX_MASK,
 682                d40c->base->virtbase + D40_DREG_PCBASE +
 683                d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT);
 684
 685         writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 686                D40_SREG_ELEM_LOG_LIDX_MASK,
 687                d40c->base->virtbase + D40_DREG_PCBASE +
 688                d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT);
 689 }
 690
 691 static int d40_config_write(struct d40_chan *d40c)
 692 {
 693         u32 addr_base;
 694         u32 var;
 695         int res;
 696
 697         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 698         if (res)
 699                 return res;
 700
 701         /* Odd addresses are even addresses + 4 */
 702         addr_base = (d40c->phy_chan->num % 2) * 4;
 703         /* Setup channel mode to logical or physical */
 704         var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
 705                 D40_CHAN_POS(d40c->phy_chan->num);
 706         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 707
 708         /* Setup operational mode option register */
 709         var = ((d40c->dma_cfg.channel_type >> STEDMA40_INFO_CH_MODE_OPT_POS) &
 710                0x3) << D40_CHAN_POS(d40c->phy_chan->num);
 711
 712         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 713
 714         if (d40c->log_num != D40_PHY_CHAN) {
 715                 /* Set default config for CFG reg */
 716                 writel(d40c->src_def_cfg,
 717                        d40c->base->virtbase + D40_DREG_PCBASE +
 718                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 719                        D40_CHAN_REG_SSCFG);
 720                 writel(d40c->dst_def_cfg,
 721                        d40c->base->virtbase + D40_DREG_PCBASE +
 722                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 723                        D40_CHAN_REG_SDCFG);
 724
 725                 d40_config_enable_lidx(d40c);
 726         }
 727         return res;
 728 }
 729
 730 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 731 {
 732
 733         if (d40d->lli_phy.dst && d40d->lli_phy.src) {
 734                 d40_phy_lli_write(d40c->base->virtbase,
 735                                   d40c->phy_chan->num,
 736                                   d40d->lli_phy.dst,
 737                                   d40d->lli_phy.src);
 738         } else if (d40d->lli_log.dst && d40d->lli_log.src) {
 739                 struct d40_log_lli *src = d40d->lli_log.src;
 740                 struct d40_log_lli *dst = d40d->lli_log.dst;
 741
 742                 src += d40d->lli_count;
 743                 dst += d40d->lli_count;
 744                 d40_log_lli_write(d40c->lcpa, d40c->lcla.src,
 745                                   d40c->lcla.dst,
 746                                   dst, src,
 747                                   d40c->base->plat_data->llis_per_log);
 748         }
 749         d40d->lli_count += d40d->lli_tx_len;
 750 }
 751
 752 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
 753 {
 754         struct d40_chan *d40c = container_of(tx->chan,
 755                                              struct d40_chan,
 756                                              chan);
 757         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
 758         unsigned long flags;
 759
 760         spin_lock_irqsave(&d40c->lock, flags);
 761
 762         tx->cookie = d40_assign_cookie(d40c, d40d);
 763
 764         d40_desc_queue(d40c, d40d);
 765
 766         spin_unlock_irqrestore(&d40c->lock, flags);
 767
 768         return tx->cookie;
 769 }
 770
 771 static int d40_start(struct d40_chan *d40c)
 772 {
 773         int err;
 774
 775         if (d40c->log_num != D40_PHY_CHAN) {
 776                 err = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 777                 if (err)
 778                         return err;
 779                 d40_config_set_event(d40c, true);
 780         }
 781
 782         err = d40_channel_execute_command(d40c, D40_DMA_RUN);
 783
 784         return err;
 785 }
 786
 787 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
 788 {
 789         struct d40_desc *d40d;
 790         int err;
 791
 792         /* Start queued jobs, if any */
 793         d40d = d40_first_queued(d40c);
 794
 795         if (d40d != NULL) {
 796                 d40c->busy = true;
 797
 798                 /* Remove from queue */
 799                 d40_desc_remove(d40d);
 800
 801                 /* Add to active queue */
 802                 d40_desc_submit(d40c, d40d);
 803
 804                 /* Initiate DMA job */
 805                 d40_desc_load(d40c, d40d);
 806
 807                 /* Start dma job */
 808                 err = d40_start(d40c);
 809
 810                 if (err)
 811                         return NULL;
 812         }
 813
 814         return d40d;
 815 }
 816
 817 /* called from interrupt context */
 818 static void dma_tc_handle(struct d40_chan *d40c)
 819 {
 820         struct d40_desc *d40d;
 821
 822         if (!d40c->phy_chan)
 823                 return;
 824
 825         /* Get first active entry from list */
 826         d40d = d40_first_active_get(d40c);
 827
 828         if (d40d == NULL)
 829                 return;
 830
 831         if (d40d->lli_count < d40d->lli_len) {
 832
 833                 d40_desc_load(d40c, d40d);
 834                 /* Start dma job */
 835                 (void) d40_start(d40c);
 836                 return;
 837         }
 838
 839         if (d40_queue_start(d40c) == NULL)
 840                 d40c->busy = false;
 841
 842         d40c->pending_tx++;
 843         tasklet_schedule(&d40c->tasklet);
 844
 845 }
 846
 847 static void dma_tasklet(unsigned long data)
 848 {
 849         struct d40_chan *d40c = (struct d40_chan *) data;
 850         struct d40_desc *d40d_fin;
 851         unsigned long flags;
 852         dma_async_tx_callback callback;
 853         void *callback_param;
 854
 855         spin_lock_irqsave(&d40c->lock, flags);
 856
 857         /* Get first active entry from list */
 858         d40d_fin = d40_first_active_get(d40c);
 859
 860         if (d40d_fin == NULL)
 861                 goto err;
 862
 863         d40c->completed = d40d_fin->txd.cookie;
 864
 865         /*
 866          * If terminating a channel pending_tx is set to zero.
 867          * This prevents any finished active jobs to return to the client.
 868          */
 869         if (d40c->pending_tx == 0) {
 870                 spin_unlock_irqrestore(&d40c->lock, flags);
 871                 return;
 872         }
 873
 874         /* Callback to client */
 875         callback = d40d_fin->txd.callback;
 876         callback_param = d40d_fin->txd.callback_param;
 877
 878         if (async_tx_test_ack(&d40d_fin->txd)) {
 879                 d40_pool_lli_free(d40d_fin);
 880                 d40_desc_remove(d40d_fin);
 881                 /* Return desc to free-list */
 882                 d40_desc_free(d40c, d40d_fin);
 883         } else {
 884                 if (!d40d_fin->is_in_client_list) {
 885                         d40_desc_remove(d40d_fin);
 886                         list_add_tail(&d40d_fin->node, &d40c->client);
 887                         d40d_fin->is_in_client_list = true;
 888                 }
 889         }
 890
 891         d40c->pending_tx--;
 892
 893         if (d40c->pending_tx)
 894                 tasklet_schedule(&d40c->tasklet);
 895
 896         spin_unlock_irqrestore(&d40c->lock, flags);
 897
 898         if (callback)
 899                 callback(callback_param);
 900
 901         return;
 902
 903  err:
 904         /* Rescue manouver if receiving double interrupts */
 905         if (d40c->pending_tx > 0)
 906                 d40c->pending_tx--;
 907         spin_unlock_irqrestore(&d40c->lock, flags);
 908 }
 909
 910 static irqreturn_t d40_handle_interrupt(int irq, void *data)
 911 {
 912         static const struct d40_interrupt_lookup il[] = {
 913                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
 914                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
 915                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
 916                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
 917                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
 918                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
 919                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
 920                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
 921                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
 922                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
 923         };
 924
 925         int i;
 926         u32 regs[ARRAY_SIZE(il)];
 927         u32 tmp;
 928         u32 idx;
 929         u32 row;
 930         long chan = -1;
 931         struct d40_chan *d40c;
 932         unsigned long flags;
 933         struct d40_base *base = data;
 934
 935         spin_lock_irqsave(&base->interrupt_lock, flags);
 936
 937         /* Read interrupt status of both logical and physical channels */
 938         for (i = 0; i < ARRAY_SIZE(il); i++)
 939                 regs[i] = readl(base->virtbase + il[i].src);
 940
 941         for (;;) {
 942
 943                 chan = find_next_bit((unsigned long *)regs,
 944                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
 945
 946                 /* No more set bits found? */
 947                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
 948                         break;
 949
 950                 row = chan / BITS_PER_LONG;
 951                 idx = chan & (BITS_PER_LONG - 1);
 952
 953                 /* ACK interrupt */
 954                 tmp = readl(base->virtbase + il[row].clr);
 955                 tmp |= 1 << idx;
 956                 writel(tmp, base->virtbase + il[row].clr);
 957
 958                 if (il[row].offset == D40_PHY_CHAN)
 959                         d40c = base->lookup_phy_chans[idx];
 960                 else
 961                         d40c = base->lookup_log_chans[il[row].offset + idx];
 962                 spin_lock(&d40c->lock);
 963
 964                 if (!il[row].is_error)
 965                         dma_tc_handle(d40c);
 966                 else
 967                         dev_err(base->dev, "[%s] IRQ chan: %ld offset %d idx %d\n",
 968                                 __func__, chan, il[row].offset, idx);
 969
 970                 spin_unlock(&d40c->lock);
 971         }
 972
 973         spin_unlock_irqrestore(&base->interrupt_lock, flags);
 974
 975         return IRQ_HANDLED;
 976 }
 977
 978
 979 static int d40_validate_conf(struct d40_chan *d40c,
 980                              struct stedma40_chan_cfg *conf)
 981 {
 982         int res = 0;
 983         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
 984         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
 985         bool is_log = (conf->channel_type & STEDMA40_CHANNEL_IN_OPER_MODE)
 986                 == STEDMA40_CHANNEL_IN_LOG_MODE;
 987
 988         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH &&
 989             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
 990                 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
 991                         __func__);
 992                 res = -EINVAL;
 993         }
 994
 995         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM &&
 996             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
 997                 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
 998                         __func__);
 999                 res = -EINVAL;
1000         }
1001
1002         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1003             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1004                 dev_err(&d40c->chan.dev->device,
1005                         "[%s] No event line\n", __func__);
1006                 res = -EINVAL;
1007         }
1008
1009         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1010             (src_event_group != dst_event_group)) {
1011                 dev_err(&d40c->chan.dev->device,
1012                         "[%s] Invalid event group\n", __func__);
1013                 res = -EINVAL;
1014         }
1015
1016         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1017                 /*
1018                  * DMAC HW supports it. Will be added to this driver,
1019                  * in case any dma client requires it.
1020                  */
1021                 dev_err(&d40c->chan.dev->device,
1022                         "[%s] periph to periph not supported\n",
1023                         __func__);
1024                 res = -EINVAL;
1025         }
1026
1027         return res;
1028 }
1029
1030 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1031                                int log_event_line, bool is_log)
1032 {
1033         unsigned long flags;
1034         spin_lock_irqsave(&phy->lock, flags);
1035         if (!is_log) {
1036                 /* Physical interrupts are masked per physical full channel */
1037                 if (phy->allocated_src == D40_ALLOC_FREE &&
1038                     phy->allocated_dst == D40_ALLOC_FREE) {
1039                         phy->allocated_dst = D40_ALLOC_PHY;
1040                         phy->allocated_src = D40_ALLOC_PHY;
1041                         goto found;
1042                 } else
1043                         goto not_found;
1044         }
1045
1046         /* Logical channel */
1047         if (is_src) {
1048                 if (phy->allocated_src == D40_ALLOC_PHY)
1049                         goto not_found;
1050
1051                 if (phy->allocated_src == D40_ALLOC_FREE)
1052                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1053
1054                 if (!(phy->allocated_src & (1 << log_event_line))) {
1055                         phy->allocated_src |= 1 << log_event_line;
1056                         goto found;
1057                 } else
1058                         goto not_found;
1059         } else {
1060                 if (phy->allocated_dst == D40_ALLOC_PHY)
1061                         goto not_found;
1062
1063                 if (phy->allocated_dst == D40_ALLOC_FREE)
1064                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1065
1066                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1067                         phy->allocated_dst |= 1 << log_event_line;
1068                         goto found;
1069                 } else
1070                         goto not_found;
1071         }
1072
1073 not_found:
1074         spin_unlock_irqrestore(&phy->lock, flags);
1075         return false;
1076 found:
1077         spin_unlock_irqrestore(&phy->lock, flags);
1078         return true;
1079 }
1080
1081 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1082                                int log_event_line)
1083 {
1084         unsigned long flags;
1085         bool is_free = false;
1086
1087         spin_lock_irqsave(&phy->lock, flags);
1088         if (!log_event_line) {
1089                 /* Physical interrupts are masked per physical full channel */
1090                 phy->allocated_dst = D40_ALLOC_FREE;
1091                 phy->allocated_src = D40_ALLOC_FREE;
1092                 is_free = true;
1093                 goto out;
1094         }
1095
1096         /* Logical channel */
1097         if (is_src) {
1098                 phy->allocated_src &= ~(1 << log_event_line);
1099                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1100                         phy->allocated_src = D40_ALLOC_FREE;
1101         } else {
1102                 phy->allocated_dst &= ~(1 << log_event_line);
1103                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1104                         phy->allocated_dst = D40_ALLOC_FREE;
1105         }
1106
1107         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1108                    D40_ALLOC_FREE);
1109
1110 out:
1111         spin_unlock_irqrestore(&phy->lock, flags);
1112
1113         return is_free;
1114 }
1115
1116 static int d40_allocate_channel(struct d40_chan *d40c)
1117 {
1118         int dev_type;
1119         int event_group;
1120         int event_line;
1121         struct d40_phy_res *phys;
1122         int i;
1123         int j;
1124         int log_num;
1125         bool is_src;
1126         bool is_log = (d40c->dma_cfg.channel_type & STEDMA40_CHANNEL_IN_OPER_MODE)
1127                 == STEDMA40_CHANNEL_IN_LOG_MODE;
1128
1129
1130         phys = d40c->base->phy_res;
1131
1132         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1133                 dev_type = d40c->dma_cfg.src_dev_type;
1134                 log_num = 2 * dev_type;
1135                 is_src = true;
1136         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1137                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1138                 /* dst event lines are used for logical memcpy */
1139                 dev_type = d40c->dma_cfg.dst_dev_type;
1140                 log_num = 2 * dev_type + 1;
1141                 is_src = false;
1142         } else
1143                 return -EINVAL;
1144
1145         event_group = D40_TYPE_TO_GROUP(dev_type);
1146         event_line = D40_TYPE_TO_EVENT(dev_type);
1147
1148         if (!is_log) {
1149                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1150                         /* Find physical half channel */
1151                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1152
1153                                 if (d40_alloc_mask_set(&phys[i], is_src,
1154                                                        0, is_log))
1155                                         goto found_phy;
1156                         }
1157                 } else
1158                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1159                                 int phy_num = j  + event_group * 2;
1160                                 for (i = phy_num; i < phy_num + 2; i++) {
1161                                         if (d40_alloc_mask_set(&phys[i], is_src,
1162                                                                0, is_log))
1163                                                 goto found_phy;
1164                                 }
1165                         }
1166                 return -EINVAL;
1167 found_phy:
1168                 d40c->phy_chan = &phys[i];
1169                 d40c->log_num = D40_PHY_CHAN;
1170                 goto out;
1171         }
1172         if (dev_type == -1)
1173                 return -EINVAL;
1174
1175         /* Find logical channel */
1176         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1177                 int phy_num = j + event_group * 2;
1178                 /*
1179                  * Spread logical channels across all available physical rather
1180                  * than pack every logical channel at the first available phy
1181                  * channels.
1182                  */
1183                 if (is_src) {
1184                         for (i = phy_num; i < phy_num + 2; i++) {
1185                                 if (d40_alloc_mask_set(&phys[i], is_src,
1186                                                        event_line, is_log))
1187                                         goto found_log;
1188                         }
1189                 } else {
1190                         for (i = phy_num + 1; i >= phy_num; i--) {
1191                                 if (d40_alloc_mask_set(&phys[i], is_src,
1192                                                        event_line, is_log))
1193                                         goto found_log;
1194                         }
1195                 }
1196         }
1197         return -EINVAL;
1198
1199 found_log:
1200         d40c->phy_chan = &phys[i];
1201         d40c->log_num = log_num;
1202 out:
1203
1204         if (is_log)
1205                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1206         else
1207                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1208
1209         return 0;
1210
1211 }
1212
1213 static int d40_config_chan(struct d40_chan *d40c,
1214                            struct stedma40_chan_cfg *info)
1215 {
1216
1217         /* Fill in basic CFG register values */
1218         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1219                     &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1220
1221         if (d40c->log_num != D40_PHY_CHAN) {
1222                 d40_log_cfg(&d40c->dma_cfg,
1223                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1224
1225                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1226                         d40c->lcpa = d40c->base->lcpa_base +
1227                                 d40c->dma_cfg.src_dev_type * 32;
1228                 else
1229                         d40c->lcpa = d40c->base->lcpa_base +
1230                                 d40c->dma_cfg.dst_dev_type * 32 + 16;
1231         }
1232
1233         /* Write channel configuration to the DMA */
1234         return d40_config_write(d40c);
1235 }
1236
1237 static int d40_config_memcpy(struct d40_chan *d40c)
1238 {
1239         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1240
1241         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1242                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1243                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1244                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1245                         memcpy[d40c->chan.chan_id];
1246
1247         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1248                    dma_has_cap(DMA_SLAVE, cap)) {
1249                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1250         } else {
1251                 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
1252                         __func__);
1253                 return -EINVAL;
1254         }
1255
1256         return 0;
1257 }
1258
1259
1260 static int d40_free_dma(struct d40_chan *d40c)
1261 {
1262
1263         int res = 0;
1264         u32 event, dir;
1265         struct d40_phy_res *phy = d40c->phy_chan;
1266         bool is_src;
1267
1268         /* Terminate all queued and active transfers */
1269         d40_term_all(d40c);
1270
1271         if (phy == NULL) {
1272                 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1273                         __func__);
1274                 return -EINVAL;
1275         }
1276
1277         if (phy->allocated_src == D40_ALLOC_FREE &&
1278             phy->allocated_dst == D40_ALLOC_FREE) {
1279                 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
1280                         __func__);
1281                 return -EINVAL;
1282         }
1283
1284
1285         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1286         if (res) {
1287                 dev_err(&d40c->chan.dev->device, "[%s] suspend\n",
1288                         __func__);
1289                 return res;
1290         }
1291
1292         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1293             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1294                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1295                 dir = D40_CHAN_REG_SDLNK;
1296                 is_src = false;
1297         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1298                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1299                 dir = D40_CHAN_REG_SSLNK;
1300                 is_src = true;
1301         } else {
1302                 dev_err(&d40c->chan.dev->device,
1303                         "[%s] Unknown direction\n", __func__);
1304                 return -EINVAL;
1305         }
1306
1307         if (d40c->log_num != D40_PHY_CHAN) {
1308                 /*
1309                  * Release logical channel, deactivate the event line during
1310                  * the time physical res is suspended.
1311                  */
1312                 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) &
1313                        D40_EVENTLINE_MASK(event),
1314                        d40c->base->virtbase + D40_DREG_PCBASE +
1315                        phy->num * D40_DREG_PCDELTA + dir);
1316
1317                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1318
1319                 /*
1320                  * Check if there are more logical allocation
1321                  * on this phy channel.
1322                  */
1323                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1324                         /* Resume the other logical channels if any */
1325                         if (d40_chan_has_events(d40c)) {
1326                                 res = d40_channel_execute_command(d40c,
1327                                                                   D40_DMA_RUN);
1328                                 if (res) {
1329                                         dev_err(&d40c->chan.dev->device,
1330                                                 "[%s] Executing RUN command\n",
1331                                                 __func__);
1332                                         return res;
1333                                 }
1334                         }
1335                         return 0;
1336                 }
1337         } else
1338                 d40_alloc_mask_free(phy, is_src, 0);
1339
1340         /* Release physical channel */
1341         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1342         if (res) {
1343                 dev_err(&d40c->chan.dev->device,
1344                         "[%s] Failed to stop channel\n", __func__);
1345                 return res;
1346         }
1347         d40c->phy_chan = NULL;
1348         /* Invalidate channel type */
1349         d40c->dma_cfg.channel_type = 0;
1350         d40c->base->lookup_phy_chans[phy->num] = NULL;
1351
1352         return 0;
1353
1354
1355 }
1356
1357 static int d40_pause(struct dma_chan *chan)
1358 {
1359         struct d40_chan *d40c =
1360                 container_of(chan, struct d40_chan, chan);
1361         int res;
1362
1363         unsigned long flags;
1364
1365         spin_lock_irqsave(&d40c->lock, flags);
1366
1367         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1368         if (res == 0) {
1369                 if (d40c->log_num != D40_PHY_CHAN) {
1370                         d40_config_set_event(d40c, false);
1371                         /* Resume the other logical channels if any */
1372                         if (d40_chan_has_events(d40c))
1373                                 res = d40_channel_execute_command(d40c,
1374                                                                   D40_DMA_RUN);
1375                 }
1376         }
1377
1378         spin_unlock_irqrestore(&d40c->lock, flags);
1379         return res;
1380 }
1381
1382 static bool d40_is_paused(struct d40_chan *d40c)
1383 {
1384         bool is_paused = false;
1385         unsigned long flags;
1386         void __iomem *active_reg;
1387         u32 status;
1388         u32 event;
1389         int res;
1390
1391         spin_lock_irqsave(&d40c->lock, flags);
1392
1393         if (d40c->log_num == D40_PHY_CHAN) {
1394                 if (d40c->phy_chan->num % 2 == 0)
1395                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1396                 else
1397                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1398
1399                 status = (readl(active_reg) &
1400                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1401                         D40_CHAN_POS(d40c->phy_chan->num);
1402                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1403                         is_paused = true;
1404
1405                 goto _exit;
1406         }
1407
1408         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1409         if (res != 0)
1410                 goto _exit;
1411
1412         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1413             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
1414                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1415         else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1416                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1417         else {
1418                 dev_err(&d40c->chan.dev->device,
1419                         "[%s] Unknown direction\n", __func__);
1420                 goto _exit;
1421         }
1422         status = d40_chan_has_events(d40c);
1423         status = (status & D40_EVENTLINE_MASK(event)) >>
1424                 D40_EVENTLINE_POS(event);
1425
1426         if (status != D40_DMA_RUN)
1427                 is_paused = true;
1428
1429         /* Resume the other logical channels if any */
1430         if (d40_chan_has_events(d40c))
1431                 res = d40_channel_execute_command(d40c,
1432                                                   D40_DMA_RUN);
1433
1434 _exit:
1435         spin_unlock_irqrestore(&d40c->lock, flags);
1436         return is_paused;
1437
1438 }
1439
1440
1441 static bool d40_tx_is_linked(struct d40_chan *d40c)
1442 {
1443         bool is_link;
1444
1445         if (d40c->log_num != D40_PHY_CHAN)
1446                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
1447         else
1448                 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1449                                 d40c->phy_chan->num * D40_DREG_PCDELTA +
1450                                 D40_CHAN_REG_SDLNK) &
1451                         D40_SREG_LNK_PHYS_LNK_MASK;
1452         return is_link;
1453 }
1454
1455 static u32 d40_residue(struct d40_chan *d40c)
1456 {
1457         u32 num_elt;
1458
1459         if (d40c->log_num != D40_PHY_CHAN)
1460                 num_elt = (readl(&d40c->lcpa->lcsp2) &  D40_MEM_LCSP2_ECNT_MASK)
1461                         >> D40_MEM_LCSP2_ECNT_POS;
1462         else
1463                 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
1464                                  d40c->phy_chan->num * D40_DREG_PCDELTA +
1465                                  D40_CHAN_REG_SDELT) &
1466                            D40_SREG_ELEM_PHY_ECNT_MASK) >> D40_SREG_ELEM_PHY_ECNT_POS;
1467         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
1468 }
1469
1470 static int d40_resume(struct dma_chan *chan)
1471 {
1472         struct d40_chan *d40c =
1473                 container_of(chan, struct d40_chan, chan);
1474         int res = 0;
1475         unsigned long flags;
1476
1477         spin_lock_irqsave(&d40c->lock, flags);
1478
1479         if (d40c->log_num != D40_PHY_CHAN) {
1480                 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1481                 if (res)
1482                         goto out;
1483
1484                 /* If bytes left to transfer or linked tx resume job */
1485                 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
1486                         d40_config_set_event(d40c, true);
1487                         res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1488                 }
1489         } else if (d40_residue(d40c) || d40_tx_is_linked(d40c))
1490                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1491
1492 out:
1493         spin_unlock_irqrestore(&d40c->lock, flags);
1494         return res;
1495 }
1496
1497 static u32 stedma40_residue(struct dma_chan *chan)
1498 {
1499         struct d40_chan *d40c =
1500                 container_of(chan, struct d40_chan, chan);
1501         u32 bytes_left;
1502         unsigned long flags;
1503
1504         spin_lock_irqsave(&d40c->lock, flags);
1505         bytes_left = d40_residue(d40c);
1506         spin_unlock_irqrestore(&d40c->lock, flags);
1507
1508         return bytes_left;
1509 }
1510
1511 /* Public DMA functions in addition to the DMA engine framework */
1512
1513 int stedma40_set_psize(struct dma_chan *chan,
1514                        int src_psize,
1515                        int dst_psize)
1516 {
1517         struct d40_chan *d40c =
1518                 container_of(chan, struct d40_chan, chan);
1519         unsigned long flags;
1520
1521         spin_lock_irqsave(&d40c->lock, flags);
1522
1523         if (d40c->log_num != D40_PHY_CHAN) {
1524                 d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
1525                 d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
1526                 d40c->log_def.lcsp1 |= src_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS;
1527                 d40c->log_def.lcsp3 |= dst_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS;
1528                 goto out;
1529         }
1530
1531         if (src_psize == STEDMA40_PSIZE_PHY_1)
1532                 d40c->src_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
1533         else {
1534                 d40c->src_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
1535                 d40c->src_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
1536                                        D40_SREG_CFG_PSIZE_POS);
1537                 d40c->src_def_cfg |= src_psize << D40_SREG_CFG_PSIZE_POS;
1538         }
1539
1540         if (dst_psize == STEDMA40_PSIZE_PHY_1)
1541                 d40c->dst_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
1542         else {
1543                 d40c->dst_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
1544                 d40c->dst_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
1545                                        D40_SREG_CFG_PSIZE_POS);
1546                 d40c->dst_def_cfg |= dst_psize << D40_SREG_CFG_PSIZE_POS;
1547         }
1548 out:
1549         spin_unlock_irqrestore(&d40c->lock, flags);
1550         return 0;
1551 }
1552 EXPORT_SYMBOL(stedma40_set_psize);
1553
1554 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1555                                                    struct scatterlist *sgl_dst,
1556                                                    struct scatterlist *sgl_src,
1557                                                    unsigned int sgl_len,
1558                                                    unsigned long flags)
1559 {
1560         int res;
1561         struct d40_desc *d40d;
1562         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1563                                              chan);
1564         unsigned long flg;
1565
1566
1567         spin_lock_irqsave(&d40c->lock, flg);
1568         d40d = d40_desc_get(d40c);
1569
1570         if (d40d == NULL)
1571                 goto err;
1572
1573         memset(d40d, 0, sizeof(struct d40_desc));
1574         d40d->lli_len = sgl_len;
1575         d40d->lli_tx_len = d40d->lli_len;
1576         d40d->txd.flags = flags;
1577
1578         if (d40c->log_num != D40_PHY_CHAN) {
1579                 if (d40d->lli_len > d40c->base->plat_data->llis_per_log)
1580                         d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
1581
1582                 if (sgl_len > 1)
1583                         /*
1584                          * Check if there is space available in lcla. If not,
1585                          * split list into 1-length and run only in lcpa
1586                          * space.
1587                          */
1588                         if (d40_lcla_id_get(d40c,
1589                                             &d40c->base->lcla_pool) != 0)
1590                                 d40d->lli_tx_len = 1;
1591
1592                 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
1593                         dev_err(&d40c->chan.dev->device,
1594                                 "[%s] Out of memory\n", __func__);
1595                         goto err;
1596                 }
1597
1598                 (void) d40_log_sg_to_lli(d40c->lcla.src_id,
1599                                          sgl_src,
1600                                          sgl_len,
1601                                          d40d->lli_log.src,
1602                                          d40c->log_def.lcsp1,
1603                                          d40c->dma_cfg.src_info.data_width,
1604                                          flags & DMA_PREP_INTERRUPT,
1605                                          d40d->lli_tx_len,
1606                                          d40c->base->plat_data->llis_per_log);
1607
1608                 (void) d40_log_sg_to_lli(d40c->lcla.dst_id,
1609                                          sgl_dst,
1610                                          sgl_len,
1611                                          d40d->lli_log.dst,
1612                                          d40c->log_def.lcsp3,
1613                                          d40c->dma_cfg.dst_info.data_width,
1614                                          flags & DMA_PREP_INTERRUPT,
1615                                          d40d->lli_tx_len,
1616                                          d40c->base->plat_data->llis_per_log);
1617
1618
1619         } else {
1620                 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1621                         dev_err(&d40c->chan.dev->device,
1622                                 "[%s] Out of memory\n", __func__);
1623                         goto err;
1624                 }
1625
1626                 res = d40_phy_sg_to_lli(sgl_src,
1627                                         sgl_len,
1628                                         0,
1629                                         d40d->lli_phy.src,
1630                                         d40d->lli_phy.src_addr,
1631                                         d40c->src_def_cfg,
1632                                         d40c->dma_cfg.src_info.data_width,
1633                                         d40c->dma_cfg.src_info.psize,
1634                                         true);
1635
1636                 if (res < 0)
1637                         goto err;
1638
1639                 res = d40_phy_sg_to_lli(sgl_dst,
1640                                         sgl_len,
1641                                         0,
1642                                         d40d->lli_phy.dst,
1643                                         d40d->lli_phy.dst_addr,
1644                                         d40c->dst_def_cfg,
1645                                         d40c->dma_cfg.dst_info.data_width,
1646                                         d40c->dma_cfg.dst_info.psize,
1647                                         true);
1648
1649                 if (res < 0)
1650                         goto err;
1651
1652                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1653                                       d40d->lli_pool.size, DMA_TO_DEVICE);
1654         }
1655
1656         dma_async_tx_descriptor_init(&d40d->txd, chan);
1657
1658         d40d->txd.tx_submit = d40_tx_submit;
1659
1660         spin_unlock_irqrestore(&d40c->lock, flg);
1661
1662         return &d40d->txd;
1663 err:
1664         spin_unlock_irqrestore(&d40c->lock, flg);
1665         return NULL;
1666 }
1667 EXPORT_SYMBOL(stedma40_memcpy_sg);
1668
1669 bool stedma40_filter(struct dma_chan *chan, void *data)
1670 {
1671         struct stedma40_chan_cfg *info = data;
1672         struct d40_chan *d40c =
1673                 container_of(chan, struct d40_chan, chan);
1674         int err;
1675
1676         if (data) {
1677                 err = d40_validate_conf(d40c, info);
1678                 if (!err)
1679                         d40c->dma_cfg = *info;
1680         } else
1681                 err = d40_config_memcpy(d40c);
1682
1683         return err == 0;
1684 }
1685 EXPORT_SYMBOL(stedma40_filter);
1686
1687 /* DMA ENGINE functions */
1688 static int d40_alloc_chan_resources(struct dma_chan *chan)
1689 {
1690         int err;
1691         unsigned long flags;
1692         struct d40_chan *d40c =
1693                 container_of(chan, struct d40_chan, chan);
1694
1695         spin_lock_irqsave(&d40c->lock, flags);
1696
1697         d40c->completed = chan->cookie = 1;
1698
1699         /*
1700          * If no dma configuration is set (channel_type == 0)
1701          * use default configuration
1702          */
1703         if (d40c->dma_cfg.channel_type == 0) {
1704                 err = d40_config_memcpy(d40c);
1705                 if (err)
1706                         goto err_alloc;
1707         }
1708
1709         err = d40_allocate_channel(d40c);
1710         if (err) {
1711                 dev_err(&d40c->chan.dev->device,
1712                         "[%s] Failed to allocate channel\n", __func__);
1713                 goto err_alloc;
1714         }
1715
1716         err = d40_config_chan(d40c, &d40c->dma_cfg);
1717         if (err) {
1718                 dev_err(&d40c->chan.dev->device,
1719                         "[%s] Failed to configure channel\n",
1720                         __func__);
1721                 goto err_config;
1722         }
1723
1724         spin_unlock_irqrestore(&d40c->lock, flags);
1725         return 0;
1726
1727  err_config:
1728         (void) d40_free_dma(d40c);
1729  err_alloc:
1730         spin_unlock_irqrestore(&d40c->lock, flags);
1731         dev_err(&d40c->chan.dev->device,
1732                 "[%s] Channel allocation failed\n", __func__);
1733         return -EINVAL;
1734 }
1735
1736 static void d40_free_chan_resources(struct dma_chan *chan)
1737 {
1738         struct d40_chan *d40c =
1739                 container_of(chan, struct d40_chan, chan);
1740         int err;
1741         unsigned long flags;
1742
1743         spin_lock_irqsave(&d40c->lock, flags);
1744
1745         err = d40_free_dma(d40c);
1746
1747         if (err)
1748                 dev_err(&d40c->chan.dev->device,
1749                         "[%s] Failed to free channel\n", __func__);
1750         spin_unlock_irqrestore(&d40c->lock, flags);
1751 }
1752
1753 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1754                                                        dma_addr_t dst,
1755                                                        dma_addr_t src,
1756                                                        size_t size,
1757                                                        unsigned long flags)
1758 {
1759         struct d40_desc *d40d;
1760         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1761                                              chan);
1762         unsigned long flg;
1763         int err = 0;
1764
1765         spin_lock_irqsave(&d40c->lock, flg);
1766         d40d = d40_desc_get(d40c);
1767
1768         if (d40d == NULL) {
1769                 dev_err(&d40c->chan.dev->device,
1770                         "[%s] Descriptor is NULL\n", __func__);
1771                 goto err;
1772         }
1773
1774         memset(d40d, 0, sizeof(struct d40_desc));
1775
1776         d40d->txd.flags = flags;
1777
1778         dma_async_tx_descriptor_init(&d40d->txd, chan);
1779
1780         d40d->txd.tx_submit = d40_tx_submit;
1781
1782         if (d40c->log_num != D40_PHY_CHAN) {
1783
1784                 if (d40_pool_lli_alloc(d40d, 1, true) < 0) {
1785                         dev_err(&d40c->chan.dev->device,
1786                                 "[%s] Out of memory\n", __func__);
1787                         goto err;
1788                 }
1789                 d40d->lli_len = 1;
1790                 d40d->lli_tx_len = 1;
1791
1792                 d40_log_fill_lli(d40d->lli_log.src,
1793                                  src,
1794                                  size,
1795                                  0,
1796                                  d40c->log_def.lcsp1,
1797                                  d40c->dma_cfg.src_info.data_width,
1798                                  true, true);
1799
1800                 d40_log_fill_lli(d40d->lli_log.dst,
1801                                  dst,
1802                                  size,
1803                                  0,
1804                                  d40c->log_def.lcsp3,
1805                                  d40c->dma_cfg.dst_info.data_width,
1806                                  true, true);
1807
1808         } else {
1809
1810                 if (d40_pool_lli_alloc(d40d, 1, false) < 0) {
1811                         dev_err(&d40c->chan.dev->device,
1812                                 "[%s] Out of memory\n", __func__);
1813                         goto err;
1814                 }
1815
1816                 err = d40_phy_fill_lli(d40d->lli_phy.src,
1817                                        src,
1818                                        size,
1819                                        d40c->dma_cfg.src_info.psize,
1820                                        0,
1821                                        d40c->src_def_cfg,
1822                                        true,
1823                                        d40c->dma_cfg.src_info.data_width,
1824                                        false);
1825                 if (err)
1826                         goto err_fill_lli;
1827
1828                 err = d40_phy_fill_lli(d40d->lli_phy.dst,
1829                                        dst,
1830                                        size,
1831                                        d40c->dma_cfg.dst_info.psize,
1832                                        0,
1833                                        d40c->dst_def_cfg,
1834                                        true,
1835                                        d40c->dma_cfg.dst_info.data_width,
1836                                        false);
1837
1838                 if (err)
1839                         goto err_fill_lli;
1840
1841                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1842                                       d40d->lli_pool.size, DMA_TO_DEVICE);
1843         }
1844
1845         spin_unlock_irqrestore(&d40c->lock, flg);
1846         return &d40d->txd;
1847
1848 err_fill_lli:
1849         dev_err(&d40c->chan.dev->device,
1850                 "[%s] Failed filling in PHY LLI\n", __func__);
1851         d40_pool_lli_free(d40d);
1852 err:
1853         spin_unlock_irqrestore(&d40c->lock, flg);
1854         return NULL;
1855 }
1856
1857 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1858                                  struct d40_chan *d40c,
1859                                  struct scatterlist *sgl,
1860                                  unsigned int sg_len,
1861                                  enum dma_data_direction direction,
1862                                  unsigned long flags)
1863 {
1864         dma_addr_t dev_addr = 0;
1865         int total_size;
1866
1867         if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
1868                 dev_err(&d40c->chan.dev->device,
1869                         "[%s] Out of memory\n", __func__);
1870                 return -ENOMEM;
1871         }
1872
1873         d40d->lli_len = sg_len;
1874         if (d40d->lli_len <= d40c->base->plat_data->llis_per_log)
1875                 d40d->lli_tx_len = d40d->lli_len;
1876         else
1877                 d40d->lli_tx_len = d40c->base->plat_data->llis_per_log;
1878
1879         if (sg_len > 1)
1880                 /*
1881                  * Check if there is space available in lcla.
1882                  * If not, split list into 1-length and run only
1883                  * in lcpa space.
1884                  */
1885                 if (d40_lcla_id_get(d40c, &d40c->base->lcla_pool) != 0)
1886                         d40d->lli_tx_len = 1;
1887
1888         if (direction == DMA_FROM_DEVICE) {
1889                 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1890                 total_size = d40_log_sg_to_dev(&d40c->lcla,
1891                                                sgl, sg_len,
1892                                                &d40d->lli_log,
1893                                                &d40c->log_def,
1894                                                d40c->dma_cfg.src_info.data_width,
1895                                                d40c->dma_cfg.dst_info.data_width,
1896                                                direction,
1897                                                flags & DMA_PREP_INTERRUPT,
1898                                                dev_addr, d40d->lli_tx_len,
1899                                                d40c->base->plat_data->llis_per_log);
1900         } else if (direction == DMA_TO_DEVICE) {
1901                 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1902                 total_size = d40_log_sg_to_dev(&d40c->lcla,
1903                                                sgl, sg_len,
1904                                                &d40d->lli_log,
1905                                                &d40c->log_def,
1906                                                d40c->dma_cfg.src_info.data_width,
1907                                                d40c->dma_cfg.dst_info.data_width,
1908                                                direction,
1909                                                flags & DMA_PREP_INTERRUPT,
1910                                                dev_addr, d40d->lli_tx_len,
1911                                                d40c->base->plat_data->llis_per_log);
1912         } else
1913                 return -EINVAL;
1914         if (total_size < 0)
1915                 return -EINVAL;
1916
1917         return 0;
1918 }
1919
1920 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1921                                  struct d40_chan *d40c,
1922                                  struct scatterlist *sgl,
1923                                  unsigned int sgl_len,
1924                                  enum dma_data_direction direction,
1925                                  unsigned long flags)
1926 {
1927         dma_addr_t src_dev_addr;
1928         dma_addr_t dst_dev_addr;
1929         int res;
1930
1931         if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
1932                 dev_err(&d40c->chan.dev->device,
1933                         "[%s] Out of memory\n", __func__);
1934                 return -ENOMEM;
1935         }
1936
1937         d40d->lli_len = sgl_len;
1938         d40d->lli_tx_len = sgl_len;
1939
1940         if (direction == DMA_FROM_DEVICE) {
1941                 dst_dev_addr = 0;
1942                 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
1943         } else if (direction == DMA_TO_DEVICE) {
1944                 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
1945                 src_dev_addr = 0;
1946         } else
1947                 return -EINVAL;
1948
1949         res = d40_phy_sg_to_lli(sgl,
1950                                 sgl_len,
1951                                 src_dev_addr,
1952                                 d40d->lli_phy.src,
1953                                 d40d->lli_phy.src_addr,
1954                                 d40c->src_def_cfg,
1955                                 d40c->dma_cfg.src_info.data_width,
1956                                 d40c->dma_cfg.src_info.psize,
1957                                 true);
1958         if (res < 0)
1959                 return res;
1960
1961         res = d40_phy_sg_to_lli(sgl,
1962                                 sgl_len,
1963                                 dst_dev_addr,
1964                                 d40d->lli_phy.dst,
1965                                 d40d->lli_phy.dst_addr,
1966                                 d40c->dst_def_cfg,
1967                                 d40c->dma_cfg.dst_info.data_width,
1968                                 d40c->dma_cfg.dst_info.psize,
1969                                  true);
1970         if (res < 0)
1971                 return res;
1972
1973         (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1974                               d40d->lli_pool.size, DMA_TO_DEVICE);
1975         return 0;
1976 }
1977
1978 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
1979                                                          struct scatterlist *sgl,
1980                                                          unsigned int sg_len,
1981                                                          enum dma_data_direction direction,
1982                                                          unsigned long flags)
1983 {
1984         struct d40_desc *d40d;
1985         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1986                                              chan);
1987         unsigned long flg;
1988         int err;
1989
1990         if (d40c->dma_cfg.pre_transfer)
1991                 d40c->dma_cfg.pre_transfer(chan,
1992                                            d40c->dma_cfg.pre_transfer_data,
1993                                            sg_dma_len(sgl));
1994
1995         spin_lock_irqsave(&d40c->lock, flg);
1996         d40d = d40_desc_get(d40c);
1997         spin_unlock_irqrestore(&d40c->lock, flg);
1998
1999         if (d40d == NULL)
2000                 return NULL;
2001
2002         memset(d40d, 0, sizeof(struct d40_desc));
2003
2004         if (d40c->log_num != D40_PHY_CHAN)
2005                 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2006                                             direction, flags);
2007         else
2008                 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2009                                             direction, flags);
2010         if (err) {
2011                 dev_err(&d40c->chan.dev->device,
2012                         "[%s] Failed to prepare %s slave sg job: %d\n",
2013                         __func__,
2014                         d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2015                 return NULL;
2016         }
2017
2018         d40d->txd.flags = flags;
2019
2020         dma_async_tx_descriptor_init(&d40d->txd, chan);
2021
2022         d40d->txd.tx_submit = d40_tx_submit;
2023
2024         return &d40d->txd;
2025 }
2026
2027 static enum dma_status d40_tx_status(struct dma_chan *chan,
2028                                      dma_cookie_t cookie,
2029                                      struct dma_tx_state *txstate)
2030 {
2031         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2032         dma_cookie_t last_used;
2033         dma_cookie_t last_complete;
2034         int ret;
2035
2036         last_complete = d40c->completed;
2037         last_used = chan->cookie;
2038
2039         if (d40_is_paused(d40c))
2040                 ret = DMA_PAUSED;
2041         else
2042                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2043
2044         dma_set_tx_state(txstate, last_complete, last_used,
2045                          stedma40_residue(chan));
2046
2047         return ret;
2048 }
2049
2050 static void d40_issue_pending(struct dma_chan *chan)
2051 {
2052         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2053         unsigned long flags;
2054
2055         spin_lock_irqsave(&d40c->lock, flags);
2056
2057         /* Busy means that pending jobs are already being processed */
2058         if (!d40c->busy)
2059                 (void) d40_queue_start(d40c);
2060
2061         spin_unlock_irqrestore(&d40c->lock, flags);
2062 }
2063
2064 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2065                        unsigned long arg)
2066 {
2067         unsigned long flags;
2068         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2069
2070         switch (cmd) {
2071         case DMA_TERMINATE_ALL:
2072                 spin_lock_irqsave(&d40c->lock, flags);
2073                 d40_term_all(d40c);
2074                 spin_unlock_irqrestore(&d40c->lock, flags);
2075                 return 0;
2076         case DMA_PAUSE:
2077                 return d40_pause(chan);
2078         case DMA_RESUME:
2079                 return d40_resume(chan);
2080         }
2081
2082         /* Other commands are unimplemented */
2083         return -ENXIO;
2084 }
2085
2086 /* Initialization functions */
2087
2088 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2089                                  struct d40_chan *chans, int offset,
2090                                  int num_chans)
2091 {
2092         int i = 0;
2093         struct d40_chan *d40c;
2094
2095         INIT_LIST_HEAD(&dma->channels);
2096
2097         for (i = offset; i < offset + num_chans; i++) {
2098                 d40c = &chans[i];
2099                 d40c->base = base;
2100                 d40c->chan.device = dma;
2101
2102                 /* Invalidate lcla element */
2103                 d40c->lcla.src_id = -1;
2104                 d40c->lcla.dst_id = -1;
2105
2106                 spin_lock_init(&d40c->lock);
2107
2108                 d40c->log_num = D40_PHY_CHAN;
2109
2110                 INIT_LIST_HEAD(&d40c->free);
2111                 INIT_LIST_HEAD(&d40c->active);
2112                 INIT_LIST_HEAD(&d40c->queue);
2113                 INIT_LIST_HEAD(&d40c->client);
2114
2115                 d40c->free_len = 0;
2116
2117                 tasklet_init(&d40c->tasklet, dma_tasklet,
2118                              (unsigned long) d40c);
2119
2120                 list_add_tail(&d40c->chan.device_node,
2121                               &dma->channels);
2122         }
2123 }
2124
2125 static int __init d40_dmaengine_init(struct d40_base *base,
2126                                      int num_reserved_chans)
2127 {
2128         int err ;
2129
2130         d40_chan_init(base, &base->dma_slave, base->log_chans,
2131                       0, base->num_log_chans);
2132
2133         dma_cap_zero(base->dma_slave.cap_mask);
2134         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2135
2136         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2137         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2138         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2139         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2140         base->dma_slave.device_tx_status = d40_tx_status;
2141         base->dma_slave.device_issue_pending = d40_issue_pending;
2142         base->dma_slave.device_control = d40_control;
2143         base->dma_slave.dev = base->dev;
2144
2145         err = dma_async_device_register(&base->dma_slave);
2146
2147         if (err) {
2148                 dev_err(base->dev,
2149                         "[%s] Failed to register slave channels\n",
2150                         __func__);
2151                 goto failure1;
2152         }
2153
2154         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2155                       base->num_log_chans, base->plat_data->memcpy_len);
2156
2157         dma_cap_zero(base->dma_memcpy.cap_mask);
2158         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2159
2160         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2161         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2162         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2163         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2164         base->dma_memcpy.device_tx_status = d40_tx_status;
2165         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2166         base->dma_memcpy.device_control = d40_control;
2167         base->dma_memcpy.dev = base->dev;
2168         /*
2169          * This controller can only access address at even
2170          * 32bit boundaries, i.e. 2^2
2171          */
2172         base->dma_memcpy.copy_align = 2;
2173
2174         err = dma_async_device_register(&base->dma_memcpy);
2175
2176         if (err) {
2177                 dev_err(base->dev,
2178                         "[%s] Failed to regsiter memcpy only channels\n",
2179                         __func__);
2180                 goto failure2;
2181         }
2182
2183         d40_chan_init(base, &base->dma_both, base->phy_chans,
2184                       0, num_reserved_chans);
2185
2186         dma_cap_zero(base->dma_both.cap_mask);
2187         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2188         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2189
2190         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2191         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2192         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2193         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2194         base->dma_both.device_tx_status = d40_tx_status;
2195         base->dma_both.device_issue_pending = d40_issue_pending;
2196         base->dma_both.device_control = d40_control;
2197         base->dma_both.dev = base->dev;
2198         base->dma_both.copy_align = 2;
2199         err = dma_async_device_register(&base->dma_both);
2200
2201         if (err) {
2202                 dev_err(base->dev,
2203                         "[%s] Failed to register logical and physical capable channels\n",
2204                         __func__);
2205                 goto failure3;
2206         }
2207         return 0;
2208 failure3:
2209         dma_async_device_unregister(&base->dma_memcpy);
2210 failure2:
2211         dma_async_device_unregister(&base->dma_slave);
2212 failure1:
2213         return err;
2214 }
2215
2216 /* Initialization functions. */
2217
2218 static int __init d40_phy_res_init(struct d40_base *base)
2219 {
2220         int i;
2221         int num_phy_chans_avail = 0;
2222         u32 val[2];
2223         int odd_even_bit = -2;
2224
2225         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2226         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2227
2228         for (i = 0; i < base->num_phy_chans; i++) {
2229                 base->phy_res[i].num = i;
2230                 odd_even_bit += 2 * ((i % 2) == 0);
2231                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2232                         /* Mark security only channels as occupied */
2233                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2234                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2235                 } else {
2236                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2237                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2238                         num_phy_chans_avail++;
2239                 }
2240                 spin_lock_init(&base->phy_res[i].lock);
2241         }
2242         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2243                  num_phy_chans_avail, base->num_phy_chans);
2244
2245         /* Verify settings extended vs standard */
2246         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2247
2248         for (i = 0; i < base->num_phy_chans; i++) {
2249
2250                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2251                     (val[0] & 0x3) != 1)
2252                         dev_info(base->dev,
2253                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2254                                  __func__, i, val[0] & 0x3);
2255
2256                 val[0] = val[0] >> 2;
2257         }
2258
2259         return num_phy_chans_avail;
2260 }
2261
2262 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2263 {
2264         static const struct d40_reg_val dma_id_regs[] = {
2265                 /* Peripheral Id */
2266                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2267                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2268                 /*
2269                  * D40_DREG_PERIPHID2 Depends on HW revision:
2270                  *  MOP500/HREF ED has 0x0008,
2271                  *  ? has 0x0018,
2272                  *  HREF V1 has 0x0028
2273                  */
2274                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2275
2276                 /* PCell Id */
2277                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2278                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2279                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2280                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2281         };
2282         struct stedma40_platform_data *plat_data;
2283         struct clk *clk = NULL;
2284         void __iomem *virtbase = NULL;
2285         struct resource *res = NULL;
2286         struct d40_base *base = NULL;
2287         int num_log_chans = 0;
2288         int num_phy_chans;
2289         int i;
2290
2291         clk = clk_get(&pdev->dev, NULL);
2292
2293         if (IS_ERR(clk)) {
2294                 dev_err(&pdev->dev, "[%s] No matching clock found\n",
2295                         __func__);
2296                 goto failure;
2297         }
2298
2299         clk_enable(clk);
2300
2301         /* Get IO for DMAC base address */
2302         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2303         if (!res)
2304                 goto failure;
2305
2306         if (request_mem_region(res->start, resource_size(res),
2307                                D40_NAME " I/O base") == NULL)
2308                 goto failure;
2309
2310         virtbase = ioremap(res->start, resource_size(res));
2311         if (!virtbase)
2312                 goto failure;
2313
2314         /* HW version check */
2315         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2316                 if (dma_id_regs[i].val !=
2317                     readl(virtbase + dma_id_regs[i].reg)) {
2318                         dev_err(&pdev->dev,
2319                                 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2320                                 __func__,
2321                                 dma_id_regs[i].val,
2322                                 dma_id_regs[i].reg,
2323                                 readl(virtbase + dma_id_regs[i].reg));
2324                         goto failure;
2325                 }
2326         }
2327
2328         i = readl(virtbase + D40_DREG_PERIPHID2);
2329
2330         if ((i & 0xf) != D40_PERIPHID2_DESIGNER) {
2331                 dev_err(&pdev->dev,
2332                         "[%s] Unknown designer! Got %x wanted %x\n",
2333                         __func__, i & 0xf, D40_PERIPHID2_DESIGNER);
2334                 goto failure;
2335         }
2336
2337         /* The number of physical channels on this HW */
2338         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2339
2340         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2341                  (i >> 4) & 0xf, res->start);
2342
2343         plat_data = pdev->dev.platform_data;
2344
2345         /* Count the number of logical channels in use */
2346         for (i = 0; i < plat_data->dev_len; i++)
2347                 if (plat_data->dev_rx[i] != 0)
2348                         num_log_chans++;
2349
2350         for (i = 0; i < plat_data->dev_len; i++)
2351                 if (plat_data->dev_tx[i] != 0)
2352                         num_log_chans++;
2353
2354         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2355                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2356                        sizeof(struct d40_chan), GFP_KERNEL);
2357
2358         if (base == NULL) {
2359                 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
2360                 goto failure;
2361         }
2362
2363         base->clk = clk;
2364         base->num_phy_chans = num_phy_chans;
2365         base->num_log_chans = num_log_chans;
2366         base->phy_start = res->start;
2367         base->phy_size = resource_size(res);
2368         base->virtbase = virtbase;
2369         base->plat_data = plat_data;
2370         base->dev = &pdev->dev;
2371         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2372         base->log_chans = &base->phy_chans[num_phy_chans];
2373
2374         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2375                                 GFP_KERNEL);
2376         if (!base->phy_res)
2377                 goto failure;
2378
2379         base->lookup_phy_chans = kzalloc(num_phy_chans *
2380                                          sizeof(struct d40_chan *),
2381                                          GFP_KERNEL);
2382         if (!base->lookup_phy_chans)
2383                 goto failure;
2384
2385         if (num_log_chans + plat_data->memcpy_len) {
2386                 /*
2387                  * The max number of logical channels are event lines for all
2388                  * src devices and dst devices
2389                  */
2390                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2391                                                  sizeof(struct d40_chan *),
2392                                                  GFP_KERNEL);
2393                 if (!base->lookup_log_chans)
2394                         goto failure;
2395         }
2396         base->lcla_pool.alloc_map = kzalloc(num_phy_chans * sizeof(u32),
2397                                             GFP_KERNEL);
2398         if (!base->lcla_pool.alloc_map)
2399                 goto failure;
2400
2401         return base;
2402
2403 failure:
2404         if (clk) {
2405                 clk_disable(clk);
2406                 clk_put(clk);
2407         }
2408         if (virtbase)
2409                 iounmap(virtbase);
2410         if (res)
2411                 release_mem_region(res->start,
2412                                    resource_size(res));
2413         if (virtbase)
2414                 iounmap(virtbase);
2415
2416         if (base) {
2417                 kfree(base->lcla_pool.alloc_map);
2418                 kfree(base->lookup_log_chans);
2419                 kfree(base->lookup_phy_chans);
2420                 kfree(base->phy_res);
2421                 kfree(base);
2422         }
2423
2424         return NULL;
2425 }
2426
2427 static void __init d40_hw_init(struct d40_base *base)
2428 {
2429
2430         static const struct d40_reg_val dma_init_reg[] = {
2431                 /* Clock every part of the DMA block from start */
2432                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2433
2434                 /* Interrupts on all logical channels */
2435                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2436                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2437                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2438                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2439                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2440                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2441                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2442                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2443                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2444                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2445                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2446                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2447         };
2448         int i;
2449         u32 prmseo[2] = {0, 0};
2450         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2451         u32 pcmis = 0;
2452         u32 pcicr = 0;
2453
2454         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2455                 writel(dma_init_reg[i].val,
2456                        base->virtbase + dma_init_reg[i].reg);
2457
2458         /* Configure all our dma channels to default settings */
2459         for (i = 0; i < base->num_phy_chans; i++) {
2460
2461                 activeo[i % 2] = activeo[i % 2] << 2;
2462
2463                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2464                     == D40_ALLOC_PHY) {
2465                         activeo[i % 2] |= 3;
2466                         continue;
2467                 }
2468
2469                 /* Enable interrupt # */
2470                 pcmis = (pcmis << 1) | 1;
2471
2472                 /* Clear interrupt # */
2473                 pcicr = (pcicr << 1) | 1;
2474
2475                 /* Set channel to physical mode */
2476                 prmseo[i % 2] = prmseo[i % 2] << 2;
2477                 prmseo[i % 2] |= 1;
2478
2479         }
2480
2481         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2482         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2483         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2484         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2485
2486         /* Write which interrupt to enable */
2487         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2488
2489         /* Write which interrupt to clear */
2490         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2491
2492 }
2493
2494 static int __init d40_probe(struct platform_device *pdev)
2495 {
2496         int err;
2497         int ret = -ENOENT;
2498         struct d40_base *base;
2499         struct resource *res = NULL;
2500         int num_reserved_chans;
2501         u32 val;
2502
2503         base = d40_hw_detect_init(pdev);
2504
2505         if (!base)
2506                 goto failure;
2507
2508         num_reserved_chans = d40_phy_res_init(base);
2509
2510         platform_set_drvdata(pdev, base);
2511
2512         spin_lock_init(&base->interrupt_lock);
2513         spin_lock_init(&base->execmd_lock);
2514
2515         /* Get IO for logical channel parameter address */
2516         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2517         if (!res) {
2518                 ret = -ENOENT;
2519                 dev_err(&pdev->dev,
2520                         "[%s] No \"lcpa\" memory resource\n",
2521                         __func__);
2522                 goto failure;
2523         }
2524         base->lcpa_size = resource_size(res);
2525         base->phy_lcpa = res->start;
2526
2527         if (request_mem_region(res->start, resource_size(res),
2528                                D40_NAME " I/O lcpa") == NULL) {
2529                 ret = -EBUSY;
2530                 dev_err(&pdev->dev,
2531                         "[%s] Failed to request LCPA region 0x%x-0x%x\n",
2532                         __func__, res->start, res->end);
2533                 goto failure;
2534         }
2535
2536         /* We make use of ESRAM memory for this. */
2537         val = readl(base->virtbase + D40_DREG_LCPA);
2538         if (res->start != val && val != 0) {
2539                 dev_warn(&pdev->dev,
2540                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2541                          __func__, val, res->start);
2542         } else
2543                 writel(res->start, base->virtbase + D40_DREG_LCPA);
2544
2545         base->lcpa_base = ioremap(res->start, resource_size(res));
2546         if (!base->lcpa_base) {
2547                 ret = -ENOMEM;
2548                 dev_err(&pdev->dev,
2549                         "[%s] Failed to ioremap LCPA region\n",
2550                         __func__);
2551                 goto failure;
2552         }
2553         /* Get IO for logical channel link address */
2554         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcla");
2555         if (!res) {
2556                 ret = -ENOENT;
2557                 dev_err(&pdev->dev,
2558                         "[%s] No \"lcla\" resource defined\n",
2559                         __func__);
2560                 goto failure;
2561         }
2562
2563         base->lcla_pool.base_size = resource_size(res);
2564         base->lcla_pool.phy = res->start;
2565
2566         if (request_mem_region(res->start, resource_size(res),
2567                                D40_NAME " I/O lcla") == NULL) {
2568                 ret = -EBUSY;
2569                 dev_err(&pdev->dev,
2570                         "[%s] Failed to request LCLA region 0x%x-0x%x\n",
2571                         __func__, res->start, res->end);
2572                 goto failure;
2573         }
2574         val = readl(base->virtbase + D40_DREG_LCLA);
2575         if (res->start != val && val != 0) {
2576                 dev_warn(&pdev->dev,
2577                          "[%s] Mismatch LCLA dma 0x%x, def 0x%x\n",
2578                          __func__, val, res->start);
2579         } else
2580                 writel(res->start, base->virtbase + D40_DREG_LCLA);
2581
2582         base->lcla_pool.base = ioremap(res->start, resource_size(res));
2583         if (!base->lcla_pool.base) {
2584                 ret = -ENOMEM;
2585                 dev_err(&pdev->dev,
2586                         "[%s] Failed to ioremap LCLA 0x%x-0x%x\n",
2587                         __func__, res->start, res->end);
2588                 goto failure;
2589         }
2590
2591         spin_lock_init(&base->lcla_pool.lock);
2592
2593         base->lcla_pool.num_blocks = base->num_phy_chans;
2594
2595         base->irq = platform_get_irq(pdev, 0);
2596
2597         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
2598
2599         if (ret) {
2600                 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
2601                 goto failure;
2602         }
2603
2604         err = d40_dmaengine_init(base, num_reserved_chans);
2605         if (err)
2606                 goto failure;
2607
2608         d40_hw_init(base);
2609
2610         dev_info(base->dev, "initialized\n");
2611         return 0;
2612
2613 failure:
2614         if (base) {
2615                 if (base->virtbase)
2616                         iounmap(base->virtbase);
2617                 if (base->lcla_pool.phy)
2618                         release_mem_region(base->lcla_pool.phy,
2619                                            base->lcla_pool.base_size);
2620                 if (base->phy_lcpa)
2621                         release_mem_region(base->phy_lcpa,
2622                                            base->lcpa_size);
2623                 if (base->phy_start)
2624                         release_mem_region(base->phy_start,
2625                                            base->phy_size);
2626                 if (base->clk) {
2627                         clk_disable(base->clk);
2628                         clk_put(base->clk);
2629                 }
2630
2631                 kfree(base->lcla_pool.alloc_map);
2632                 kfree(base->lookup_log_chans);
2633                 kfree(base->lookup_phy_chans);
2634                 kfree(base->phy_res);
2635                 kfree(base);
2636         }
2637
2638         dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
2639         return ret;
2640 }
2641
2642 static struct platform_driver d40_driver = {
2643         .driver = {
2644                 .owner = THIS_MODULE,
2645                 .name  = D40_NAME,
2646         },
2647 };
2648
2649 int __init stedma40_init(void)
2650 {
2651         return platform_driver_probe(&d40_driver, d40_probe);
2652 }
2653 arch_initcall(stedma40_init);