drivers/dma/iop-adma.c

   1 /*
   2  * offload engine driver for the Intel Xscale series of i/o processors
   3  * Copyright © 2006, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  */
  19
  20 /*
  21  * This driver supports the asynchrounous DMA copy and RAID engines available
  22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  23  */
  24
  25 #include <linux/init.h>
  26 #include <linux/module.h>
  27 #include <linux/delay.h>
  28 #include <linux/dma-mapping.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/memory.h>
  33 #include <linux/ioport.h>
  34 #include <linux/raid/pq.h>
  35 #include <linux/slab.h>
  36
  37 #include <mach/adma.h>
  38
  39 #include "dmaengine.h"
  40
  41 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
  42 #define to_iop_adma_device(dev) \
  43         container_of(dev, struct iop_adma_device, common)
  44 #define tx_to_iop_adma_slot(tx) \
  45         container_of(tx, struct iop_adma_desc_slot, async_tx)
  46
  47 /**
  48  * iop_adma_free_slots - flags descriptor slots for reuse
  49  * @slot: Slot to free
  50  * Caller must hold &iop_chan->lock while calling this function
  51  */
  52 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
  53 {
  54         int stride = slot->slots_per_op;
  55
  56         while (stride--) {
  57                 slot->slots_per_op = 0;
  58                 slot = list_entry(slot->slot_node.next,
  59                                 struct iop_adma_desc_slot,
  60                                 slot_node);
  61         }
  62 }
  63
  64 static void
  65 iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
  66 {
  67         struct dma_async_tx_descriptor *tx = &desc->async_tx;
  68         struct iop_adma_desc_slot *unmap = desc->group_head;
  69         struct device *dev = &iop_chan->device->pdev->dev;
  70         u32 len = unmap->unmap_len;
  71         enum dma_ctrl_flags flags = tx->flags;
  72         u32 src_cnt;
  73         dma_addr_t addr;
  74         dma_addr_t dest;
  75
  76         src_cnt = unmap->unmap_src_cnt;
  77         dest = iop_desc_get_dest_addr(unmap, iop_chan);
  78         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
  79                 enum dma_data_direction dir;
  80
  81                 if (src_cnt > 1) /* is xor? */
  82                         dir = DMA_BIDIRECTIONAL;
  83                 else
  84                         dir = DMA_FROM_DEVICE;
  85
  86                 dma_unmap_page(dev, dest, len, dir);
  87         }
  88
  89         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
  90                 while (src_cnt--) {
  91                         addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
  92                         if (addr == dest)
  93                                 continue;
  94                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
  95                 }
  96         }
  97         desc->group_head = NULL;
  98 }
  99
 100 static void
 101 iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
 102 {
 103         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 104         struct iop_adma_desc_slot *unmap = desc->group_head;
 105         struct device *dev = &iop_chan->device->pdev->dev;
 106         u32 len = unmap->unmap_len;
 107         enum dma_ctrl_flags flags = tx->flags;
 108         u32 src_cnt = unmap->unmap_src_cnt;
 109         dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
 110         dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
 111         int i;
 112
 113         if (tx->flags & DMA_PREP_CONTINUE)
 114                 src_cnt -= 3;
 115
 116         if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
 117                 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
 118                 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
 119         }
 120
 121         if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 122                 dma_addr_t addr;
 123
 124                 for (i = 0; i < src_cnt; i++) {
 125                         addr = iop_desc_get_src_addr(unmap, iop_chan, i);
 126                         dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
 127                 }
 128                 if (desc->pq_check_result) {
 129                         dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
 130                         dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
 131                 }
 132         }
 133
 134         desc->group_head = NULL;
 135 }
 136
 137
 138 static dma_cookie_t
 139 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 140         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 141 {
 142         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 143
 144         BUG_ON(tx->cookie < 0);
 145         if (tx->cookie > 0) {
 146                 cookie = tx->cookie;
 147                 tx->cookie = 0;
 148
 149                 /* call the callback (must not sleep or submit new
 150                  * operations to this channel)
 151                  */
 152                 if (tx->callback)
 153                         tx->callback(tx->callback_param);
 154
 155                 /* unmap dma addresses
 156                  * (unmap_single vs unmap_page?)
 157                  */
 158                 if (desc->group_head && desc->unmap_len) {
 159                         if (iop_desc_is_pq(desc))
 160                                 iop_desc_unmap_pq(iop_chan, desc);
 161                         else
 162                                 iop_desc_unmap(iop_chan, desc);
 163                 }
 164         }
 165
 166         /* run dependent operations */
 167         dma_run_dependencies(tx);
 168
 169         return cookie;
 170 }
 171
 172 static int
 173 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 174         struct iop_adma_chan *iop_chan)
 175 {
 176         /* the client is allowed to attach dependent operations
 177          * until 'ack' is set
 178          */
 179         if (!async_tx_test_ack(&desc->async_tx))
 180                 return 0;
 181
 182         /* leave the last descriptor in the chain
 183          * so we can append to it
 184          */
 185         if (desc->chain_node.next == &iop_chan->chain)
 186                 return 1;
 187
 188         dev_dbg(iop_chan->device->common.dev,
 189                 "\tfree slot: %d slots_per_op: %d\n",
 190                 desc->idx, desc->slots_per_op);
 191
 192         list_del(&desc->chain_node);
 193         iop_adma_free_slots(desc);
 194
 195         return 0;
 196 }
 197
 198 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 199 {
 200         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
 201         dma_cookie_t cookie = 0;
 202         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
 203         int busy = iop_chan_is_busy(iop_chan);
 204         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
 205
 206         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 207         /* free completed slots from the chain starting with
 208          * the oldest descriptor
 209          */
 210         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 211                                         chain_node) {
 212                 pr_debug("\tcookie: %d slot: %d busy: %d "
 213                         "this_desc: %#x next_desc: %#x ack: %d\n",
 214                         iter->async_tx.cookie, iter->idx, busy,
 215                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
 216                         async_tx_test_ack(&iter->async_tx));
 217                 prefetch(_iter);
 218                 prefetch(&_iter->async_tx);
 219
 220                 /* do not advance past the current descriptor loaded into the
 221                  * hardware channel, subsequent descriptors are either in
 222                  * process or have not been submitted
 223                  */
 224                 if (seen_current)
 225                         break;
 226
 227                 /* stop the search if we reach the current descriptor and the
 228                  * channel is busy, or if it appears that the current descriptor
 229                  * needs to be re-read (i.e. has been appended to)
 230                  */
 231                 if (iter->async_tx.phys == current_desc) {
 232                         BUG_ON(seen_current++);
 233                         if (busy || iop_desc_get_next_desc(iter))
 234                                 break;
 235                 }
 236
 237                 /* detect the start of a group transaction */
 238                 if (!slot_cnt && !slots_per_op) {
 239                         slot_cnt = iter->slot_cnt;
 240                         slots_per_op = iter->slots_per_op;
 241                         if (slot_cnt <= slots_per_op) {
 242                                 slot_cnt = 0;
 243                                 slots_per_op = 0;
 244                         }
 245                 }
 246
 247                 if (slot_cnt) {
 248                         pr_debug("\tgroup++\n");
 249                         if (!grp_start)
 250                                 grp_start = iter;
 251                         slot_cnt -= slots_per_op;
 252                 }
 253
 254                 /* all the members of a group are complete */
 255                 if (slots_per_op != 0 && slot_cnt == 0) {
 256                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
 257                         int end_of_chain = 0;
 258                         pr_debug("\tgroup end\n");
 259
 260                         /* collect the total results */
 261                         if (grp_start->xor_check_result) {
 262                                 u32 zero_sum_result = 0;
 263                                 slot_cnt = grp_start->slot_cnt;
 264                                 grp_iter = grp_start;
 265
 266                                 list_for_each_entry_from(grp_iter,
 267                                         &iop_chan->chain, chain_node) {
 268                                         zero_sum_result |=
 269                                             iop_desc_get_zero_result(grp_iter);
 270                                             pr_debug("\titer%d result: %d\n",
 271                                             grp_iter->idx, zero_sum_result);
 272                                         slot_cnt -= slots_per_op;
 273                                         if (slot_cnt == 0)
 274                                                 break;
 275                                 }
 276                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
 277                                         grp_start->xor_check_result);
 278                                 *grp_start->xor_check_result = zero_sum_result;
 279                         }
 280
 281                         /* clean up the group */
 282                         slot_cnt = grp_start->slot_cnt;
 283                         grp_iter = grp_start;
 284                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
 285                                 &iop_chan->chain, chain_node) {
 286                                 cookie = iop_adma_run_tx_complete_actions(
 287                                         grp_iter, iop_chan, cookie);
 288
 289                                 slot_cnt -= slots_per_op;
 290                                 end_of_chain = iop_adma_clean_slot(grp_iter,
 291                                         iop_chan);
 292
 293                                 if (slot_cnt == 0 || end_of_chain)
 294                                         break;
 295                         }
 296
 297                         /* the group should be complete at this point */
 298                         BUG_ON(slot_cnt);
 299
 300                         slots_per_op = 0;
 301                         grp_start = NULL;
 302                         if (end_of_chain)
 303                                 break;
 304                         else
 305                                 continue;
 306                 } else if (slots_per_op) /* wait for group completion */
 307                         continue;
 308
 309                 /* write back zero sum results (single descriptor case) */
 310                 if (iter->xor_check_result && iter->async_tx.cookie)
 311                         *iter->xor_check_result =
 312                                 iop_desc_get_zero_result(iter);
 313
 314                 cookie = iop_adma_run_tx_complete_actions(
 315                                         iter, iop_chan, cookie);
 316
 317                 if (iop_adma_clean_slot(iter, iop_chan))
 318                         break;
 319         }
 320
 321         if (cookie > 0) {
 322                 iop_chan->common.completed_cookie = cookie;
 323                 pr_debug("\tcompleted cookie %d\n", cookie);
 324         }
 325 }
 326
 327 static void
 328 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
 329 {
 330         spin_lock_bh(&iop_chan->lock);
 331         __iop_adma_slot_cleanup(iop_chan);
 332         spin_unlock_bh(&iop_chan->lock);
 333 }
 334
 335 static void iop_adma_tasklet(unsigned long data)
 336 {
 337         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
 338
 339         /* lockdep will flag depedency submissions as potentially
 340          * recursive locking, this is not the case as a dependency
 341          * submission will never recurse a channels submit routine.
 342          * There are checks in async_tx.c to prevent this.
 343          */
 344         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
 345         __iop_adma_slot_cleanup(iop_chan);
 346         spin_unlock(&iop_chan->lock);
 347 }
 348
 349 static struct iop_adma_desc_slot *
 350 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
 351                         int slots_per_op)
 352 {
 353         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
 354         LIST_HEAD(chain);
 355         int slots_found, retry = 0;
 356
 357         /* start search from the last allocated descrtiptor
 358          * if a contiguous allocation can not be found start searching
 359          * from the beginning of the list
 360          */
 361 retry:
 362         slots_found = 0;
 363         if (retry == 0)
 364                 iter = iop_chan->last_used;
 365         else
 366                 iter = list_entry(&iop_chan->all_slots,
 367                         struct iop_adma_desc_slot,
 368                         slot_node);
 369
 370         list_for_each_entry_safe_continue(
 371                 iter, _iter, &iop_chan->all_slots, slot_node) {
 372                 prefetch(_iter);
 373                 prefetch(&_iter->async_tx);
 374                 if (iter->slots_per_op) {
 375                         /* give up after finding the first busy slot
 376                          * on the second pass through the list
 377                          */
 378                         if (retry)
 379                                 break;
 380
 381                         slots_found = 0;
 382                         continue;
 383                 }
 384
 385                 /* start the allocation if the slot is correctly aligned */
 386                 if (!slots_found++) {
 387                         if (iop_desc_is_aligned(iter, slots_per_op))
 388                                 alloc_start = iter;
 389                         else {
 390                                 slots_found = 0;
 391                                 continue;
 392                         }
 393                 }
 394
 395                 if (slots_found == num_slots) {
 396                         struct iop_adma_desc_slot *alloc_tail = NULL;
 397                         struct iop_adma_desc_slot *last_used = NULL;
 398                         iter = alloc_start;
 399                         while (num_slots) {
 400                                 int i;
 401                                 dev_dbg(iop_chan->device->common.dev,
 402                                         "allocated slot: %d "
 403                                         "(desc %p phys: %#x) slots_per_op %d\n",
 404                                         iter->idx, iter->hw_desc,
 405                                         iter->async_tx.phys, slots_per_op);
 406
 407                                 /* pre-ack all but the last descriptor */
 408                                 if (num_slots != slots_per_op)
 409                                         async_tx_ack(&iter->async_tx);
 410
 411                                 list_add_tail(&iter->chain_node, &chain);
 412                                 alloc_tail = iter;
 413                                 iter->async_tx.cookie = 0;
 414                                 iter->slot_cnt = num_slots;
 415                                 iter->xor_check_result = NULL;
 416                                 for (i = 0; i < slots_per_op; i++) {
 417                                         iter->slots_per_op = slots_per_op - i;
 418                                         last_used = iter;
 419                                         iter = list_entry(iter->slot_node.next,
 420                                                 struct iop_adma_desc_slot,
 421                                                 slot_node);
 422                                 }
 423                                 num_slots -= slots_per_op;
 424                         }
 425                         alloc_tail->group_head = alloc_start;
 426                         alloc_tail->async_tx.cookie = -EBUSY;
 427                         list_splice(&chain, &alloc_tail->tx_list);
 428                         iop_chan->last_used = last_used;
 429                         iop_desc_clear_next_desc(alloc_start);
 430                         iop_desc_clear_next_desc(alloc_tail);
 431                         return alloc_tail;
 432                 }
 433         }
 434         if (!retry++)
 435                 goto retry;
 436
 437         /* perform direct reclaim if the allocation fails */
 438         __iop_adma_slot_cleanup(iop_chan);
 439
 440         return NULL;
 441 }
 442
 443 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
 444 {
 445         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
 446                 iop_chan->pending);
 447
 448         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
 449                 iop_chan->pending = 0;
 450                 iop_chan_append(iop_chan);
 451         }
 452 }
 453
 454 static dma_cookie_t
 455 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
 456 {
 457         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
 458         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
 459         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
 460         int slot_cnt;
 461         int slots_per_op;
 462         dma_cookie_t cookie;
 463         dma_addr_t next_dma;
 464
 465         grp_start = sw_desc->group_head;
 466         slot_cnt = grp_start->slot_cnt;
 467         slots_per_op = grp_start->slots_per_op;
 468
 469         spin_lock_bh(&iop_chan->lock);
 470         cookie = dma_cookie_assign(tx);
 471
 472         old_chain_tail = list_entry(iop_chan->chain.prev,
 473                 struct iop_adma_desc_slot, chain_node);
 474         list_splice_init(&sw_desc->tx_list,
 475                          &old_chain_tail->chain_node);
 476
 477         /* fix up the hardware chain */
 478         next_dma = grp_start->async_tx.phys;
 479         iop_desc_set_next_desc(old_chain_tail, next_dma);
 480         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
 481
 482         /* check for pre-chained descriptors */
 483         iop_paranoia(iop_desc_get_next_desc(sw_desc));
 484
 485         /* increment the pending count by the number of slots
 486          * memcpy operations have a 1:1 (slot:operation) relation
 487          * other operations are heavier and will pop the threshold
 488          * more often.
 489          */
 490         iop_chan->pending += slot_cnt;
 491         iop_adma_check_threshold(iop_chan);
 492         spin_unlock_bh(&iop_chan->lock);
 493
 494         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
 495                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
 496
 497         return cookie;
 498 }
 499
 500 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
 501 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
 502
 503 /**
 504  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
 505  * @chan - allocate descriptor resources for this channel
 506  * @client - current client requesting the channel be ready for requests
 507  *
 508  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
 509  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
 510  * greater than 2x the number slots needed to satisfy a device->max_xor
 511  * request.
 512  * */
 513 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
 514 {
 515         char *hw_desc;
 516         int idx;
 517         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 518         struct iop_adma_desc_slot *slot = NULL;
 519         int init = iop_chan->slots_allocated ? 0 : 1;
 520         struct iop_adma_platform_data *plat_data =
 521                 dev_get_platdata(&iop_chan->device->pdev->dev);
 522         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
 523
 524         /* Allocate descriptor slots */
 525         do {
 526                 idx = iop_chan->slots_allocated;
 527                 if (idx == num_descs_in_pool)
 528                         break;
 529
 530                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
 531                 if (!slot) {
 532                         printk(KERN_INFO "IOP ADMA Channel only initialized"
 533                                 " %d descriptor slots", idx);
 534                         break;
 535                 }
 536                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
 537                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 538
 539                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
 540                 slot->async_tx.tx_submit = iop_adma_tx_submit;
 541                 INIT_LIST_HEAD(&slot->tx_list);
 542                 INIT_LIST_HEAD(&slot->chain_node);
 543                 INIT_LIST_HEAD(&slot->slot_node);
 544                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
 545                 slot->async_tx.phys =
 546                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
 547                 slot->idx = idx;
 548
 549                 spin_lock_bh(&iop_chan->lock);
 550                 iop_chan->slots_allocated++;
 551                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
 552                 spin_unlock_bh(&iop_chan->lock);
 553         } while (iop_chan->slots_allocated < num_descs_in_pool);
 554
 555         if (idx && !iop_chan->last_used)
 556                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
 557                                         struct iop_adma_desc_slot,
 558                                         slot_node);
 559
 560         dev_dbg(iop_chan->device->common.dev,
 561                 "allocated %d descriptor slots last_used: %p\n",
 562                 iop_chan->slots_allocated, iop_chan->last_used);
 563
 564         /* initialize the channel and the chain with a null operation */
 565         if (init) {
 566                 if (dma_has_cap(DMA_MEMCPY,
 567                         iop_chan->device->common.cap_mask))
 568                         iop_chan_start_null_memcpy(iop_chan);
 569                 else if (dma_has_cap(DMA_XOR,
 570                         iop_chan->device->common.cap_mask))
 571                         iop_chan_start_null_xor(iop_chan);
 572                 else
 573                         BUG();
 574         }
 575
 576         return (idx > 0) ? idx : -ENOMEM;
 577 }
 578
 579 static struct dma_async_tx_descriptor *
 580 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
 581 {
 582         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 583         struct iop_adma_desc_slot *sw_desc, *grp_start;
 584         int slot_cnt, slots_per_op;
 585
 586         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
 587
 588         spin_lock_bh(&iop_chan->lock);
 589         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
 590         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 591         if (sw_desc) {
 592                 grp_start = sw_desc->group_head;
 593                 iop_desc_init_interrupt(grp_start, iop_chan);
 594                 grp_start->unmap_len = 0;
 595                 sw_desc->async_tx.flags = flags;
 596         }
 597         spin_unlock_bh(&iop_chan->lock);
 598
 599         return sw_desc ? &sw_desc->async_tx : NULL;
 600 }
 601
 602 static struct dma_async_tx_descriptor *
 603 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 604                          dma_addr_t dma_src, size_t len, unsigned long flags)
 605 {
 606         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 607         struct iop_adma_desc_slot *sw_desc, *grp_start;
 608         int slot_cnt, slots_per_op;
 609
 610         if (unlikely(!len))
 611                 return NULL;
 612         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
 613
 614         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
 615                 __func__, len);
 616
 617         spin_lock_bh(&iop_chan->lock);
 618         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
 619         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 620         if (sw_desc) {
 621                 grp_start = sw_desc->group_head;
 622                 iop_desc_init_memcpy(grp_start, flags);
 623                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 624                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 625                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
 626                 sw_desc->unmap_src_cnt = 1;
 627                 sw_desc->unmap_len = len;
 628                 sw_desc->async_tx.flags = flags;
 629         }
 630         spin_unlock_bh(&iop_chan->lock);
 631
 632         return sw_desc ? &sw_desc->async_tx : NULL;
 633 }
 634
 635 static struct dma_async_tx_descriptor *
 636 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
 637                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
 638                       unsigned long flags)
 639 {
 640         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 641         struct iop_adma_desc_slot *sw_desc, *grp_start;
 642         int slot_cnt, slots_per_op;
 643
 644         if (unlikely(!len))
 645                 return NULL;
 646         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 647
 648         dev_dbg(iop_chan->device->common.dev,
 649                 "%s src_cnt: %d len: %u flags: %lx\n",
 650                 __func__, src_cnt, len, flags);
 651
 652         spin_lock_bh(&iop_chan->lock);
 653         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
 654         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 655         if (sw_desc) {
 656                 grp_start = sw_desc->group_head;
 657                 iop_desc_init_xor(grp_start, src_cnt, flags);
 658                 iop_desc_set_byte_count(grp_start, iop_chan, len);
 659                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
 660                 sw_desc->unmap_src_cnt = src_cnt;
 661                 sw_desc->unmap_len = len;
 662                 sw_desc->async_tx.flags = flags;
 663                 while (src_cnt--)
 664                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
 665                                                   dma_src[src_cnt]);
 666         }
 667         spin_unlock_bh(&iop_chan->lock);
 668
 669         return sw_desc ? &sw_desc->async_tx : NULL;
 670 }
 671
 672 static struct dma_async_tx_descriptor *
 673 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
 674                           unsigned int src_cnt, size_t len, u32 *result,
 675                           unsigned long flags)
 676 {
 677         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 678         struct iop_adma_desc_slot *sw_desc, *grp_start;
 679         int slot_cnt, slots_per_op;
 680
 681         if (unlikely(!len))
 682                 return NULL;
 683
 684         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 685                 __func__, src_cnt, len);
 686
 687         spin_lock_bh(&iop_chan->lock);
 688         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
 689         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 690         if (sw_desc) {
 691                 grp_start = sw_desc->group_head;
 692                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
 693                 iop_desc_set_zero_sum_byte_count(grp_start, len);
 694                 grp_start->xor_check_result = result;
 695                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
 696                         __func__, grp_start->xor_check_result);
 697                 sw_desc->unmap_src_cnt = src_cnt;
 698                 sw_desc->unmap_len = len;
 699                 sw_desc->async_tx.flags = flags;
 700                 while (src_cnt--)
 701                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
 702                                                        dma_src[src_cnt]);
 703         }
 704         spin_unlock_bh(&iop_chan->lock);
 705
 706         return sw_desc ? &sw_desc->async_tx : NULL;
 707 }
 708
 709 static struct dma_async_tx_descriptor *
 710 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 711                      unsigned int src_cnt, const unsigned char *scf, size_t len,
 712                      unsigned long flags)
 713 {
 714         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 715         struct iop_adma_desc_slot *sw_desc, *g;
 716         int slot_cnt, slots_per_op;
 717         int continue_srcs;
 718
 719         if (unlikely(!len))
 720                 return NULL;
 721         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 722
 723         dev_dbg(iop_chan->device->common.dev,
 724                 "%s src_cnt: %d len: %u flags: %lx\n",
 725                 __func__, src_cnt, len, flags);
 726
 727         if (dmaf_p_disabled_continue(flags))
 728                 continue_srcs = 1+src_cnt;
 729         else if (dmaf_continue(flags))
 730                 continue_srcs = 3+src_cnt;
 731         else
 732                 continue_srcs = 0+src_cnt;
 733
 734         spin_lock_bh(&iop_chan->lock);
 735         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
 736         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 737         if (sw_desc) {
 738                 int i;
 739
 740                 g = sw_desc->group_head;
 741                 iop_desc_set_byte_count(g, iop_chan, len);
 742
 743                 /* even if P is disabled its destination address (bits
 744                  * [3:0]) must match Q.  It is ok if P points to an
 745                  * invalid address, it won't be written.
 746                  */
 747                 if (flags & DMA_PREP_PQ_DISABLE_P)
 748                         dst[0] = dst[1] & 0x7;
 749
 750                 iop_desc_set_pq_addr(g, dst);
 751                 sw_desc->unmap_src_cnt = src_cnt;
 752                 sw_desc->unmap_len = len;
 753                 sw_desc->async_tx.flags = flags;
 754                 for (i = 0; i < src_cnt; i++)
 755                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
 756
 757                 /* if we are continuing a previous operation factor in
 758                  * the old p and q values, see the comment for dma_maxpq
 759                  * in include/linux/dmaengine.h
 760                  */
 761                 if (dmaf_p_disabled_continue(flags))
 762                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 763                 else if (dmaf_continue(flags)) {
 764                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
 765                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
 766                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
 767                 }
 768                 iop_desc_init_pq(g, i, flags);
 769         }
 770         spin_unlock_bh(&iop_chan->lock);
 771
 772         return sw_desc ? &sw_desc->async_tx : NULL;
 773 }
 774
 775 static struct dma_async_tx_descriptor *
 776 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 777                          unsigned int src_cnt, const unsigned char *scf,
 778                          size_t len, enum sum_check_flags *pqres,
 779                          unsigned long flags)
 780 {
 781         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 782         struct iop_adma_desc_slot *sw_desc, *g;
 783         int slot_cnt, slots_per_op;
 784
 785         if (unlikely(!len))
 786                 return NULL;
 787         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
 788
 789         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
 790                 __func__, src_cnt, len);
 791
 792         spin_lock_bh(&iop_chan->lock);
 793         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
 794         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
 795         if (sw_desc) {
 796                 /* for validate operations p and q are tagged onto the
 797                  * end of the source list
 798                  */
 799                 int pq_idx = src_cnt;
 800
 801                 g = sw_desc->group_head;
 802                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
 803                 iop_desc_set_pq_zero_sum_byte_count(g, len);
 804                 g->pq_check_result = pqres;
 805                 pr_debug("\t%s: g->pq_check_result: %p\n",
 806                         __func__, g->pq_check_result);
 807                 sw_desc->unmap_src_cnt = src_cnt+2;
 808                 sw_desc->unmap_len = len;
 809                 sw_desc->async_tx.flags = flags;
 810                 while (src_cnt--)
 811                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
 812                                                           src[src_cnt],
 813                                                           scf[src_cnt]);
 814                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
 815         }
 816         spin_unlock_bh(&iop_chan->lock);
 817
 818         return sw_desc ? &sw_desc->async_tx : NULL;
 819 }
 820
 821 static void iop_adma_free_chan_resources(struct dma_chan *chan)
 822 {
 823         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 824         struct iop_adma_desc_slot *iter, *_iter;
 825         int in_use_descs = 0;
 826
 827         iop_adma_slot_cleanup(iop_chan);
 828
 829         spin_lock_bh(&iop_chan->lock);
 830         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
 831                                         chain_node) {
 832                 in_use_descs++;
 833                 list_del(&iter->chain_node);
 834         }
 835         list_for_each_entry_safe_reverse(
 836                 iter, _iter, &iop_chan->all_slots, slot_node) {
 837                 list_del(&iter->slot_node);
 838                 kfree(iter);
 839                 iop_chan->slots_allocated--;
 840         }
 841         iop_chan->last_used = NULL;
 842
 843         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
 844                 __func__, iop_chan->slots_allocated);
 845         spin_unlock_bh(&iop_chan->lock);
 846
 847         /* one is ok since we left it on there on purpose */
 848         if (in_use_descs > 1)
 849                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
 850                         in_use_descs - 1);
 851 }
 852
 853 /**
 854  * iop_adma_status - poll the status of an ADMA transaction
 855  * @chan: ADMA channel handle
 856  * @cookie: ADMA transaction identifier
 857  * @txstate: a holder for the current state of the channel or NULL
 858  */
 859 static enum dma_status iop_adma_status(struct dma_chan *chan,
 860                                         dma_cookie_t cookie,
 861                                         struct dma_tx_state *txstate)
 862 {
 863         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 864         int ret;
 865
 866         ret = dma_cookie_status(chan, cookie, txstate);
 867         if (ret == DMA_COMPLETE)
 868                 return ret;
 869
 870         iop_adma_slot_cleanup(iop_chan);
 871
 872         return dma_cookie_status(chan, cookie, txstate);
 873 }
 874
 875 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
 876 {
 877         struct iop_adma_chan *chan = data;
 878
 879         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 880
 881         tasklet_schedule(&chan->irq_tasklet);
 882
 883         iop_adma_device_clear_eot_status(chan);
 884
 885         return IRQ_HANDLED;
 886 }
 887
 888 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
 889 {
 890         struct iop_adma_chan *chan = data;
 891
 892         dev_dbg(chan->device->common.dev, "%s\n", __func__);
 893
 894         tasklet_schedule(&chan->irq_tasklet);
 895
 896         iop_adma_device_clear_eoc_status(chan);
 897
 898         return IRQ_HANDLED;
 899 }
 900
 901 static irqreturn_t iop_adma_err_handler(int irq, void *data)
 902 {
 903         struct iop_adma_chan *chan = data;
 904         unsigned long status = iop_chan_get_status(chan);
 905
 906         dev_err(chan->device->common.dev,
 907                 "error ( %s%s%s%s%s%s%s)\n",
 908                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
 909                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
 910                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
 911                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
 912                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
 913                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
 914                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
 915
 916         iop_adma_device_clear_err_status(chan);
 917
 918         BUG();
 919
 920         return IRQ_HANDLED;
 921 }
 922
 923 static void iop_adma_issue_pending(struct dma_chan *chan)
 924 {
 925         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
 926
 927         if (iop_chan->pending) {
 928                 iop_chan->pending = 0;
 929                 iop_chan_append(iop_chan);
 930         }
 931 }
 932
 933 /*
 934  * Perform a transaction to verify the HW works.
 935  */
 936 #define IOP_ADMA_TEST_SIZE 2000
 937
 938 static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
 939 {
 940         int i;
 941         void *src, *dest;
 942         dma_addr_t src_dma, dest_dma;
 943         struct dma_chan *dma_chan;
 944         dma_cookie_t cookie;
 945         struct dma_async_tx_descriptor *tx;
 946         int err = 0;
 947         struct iop_adma_chan *iop_chan;
 948
 949         dev_dbg(device->common.dev, "%s\n", __func__);
 950
 951         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 952         if (!src)
 953                 return -ENOMEM;
 954         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
 955         if (!dest) {
 956                 kfree(src);
 957                 return -ENOMEM;
 958         }
 959
 960         /* Fill in src buffer */
 961         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
 962                 ((u8 *) src)[i] = (u8)i;
 963
 964         /* Start copy, using first DMA channel */
 965         dma_chan = container_of(device->common.channels.next,
 966                                 struct dma_chan,
 967                                 device_node);
 968         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
 969                 err = -ENODEV;
 970                 goto out;
 971         }
 972
 973         dest_dma = dma_map_single(dma_chan->device->dev, dest,
 974                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 975         src_dma = dma_map_single(dma_chan->device->dev, src,
 976                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
 977         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
 978                                       IOP_ADMA_TEST_SIZE,
 979                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 980
 981         cookie = iop_adma_tx_submit(tx);
 982         iop_adma_issue_pending(dma_chan);
 983         msleep(1);
 984
 985         if (iop_adma_status(dma_chan, cookie, NULL) !=
 986                         DMA_COMPLETE) {
 987                 dev_err(dma_chan->device->dev,
 988                         "Self-test copy timed out, disabling\n");
 989                 err = -ENODEV;
 990                 goto free_resources;
 991         }
 992
 993         iop_chan = to_iop_adma_chan(dma_chan);
 994         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
 995                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
 996         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
 997                 dev_err(dma_chan->device->dev,
 998                         "Self-test copy failed compare, disabling\n");
 999                 err = -ENODEV;
1000                 goto free_resources;
1001         }
1002
1003 free_resources:
1004         iop_adma_free_chan_resources(dma_chan);
1005 out:
1006         kfree(src);
1007         kfree(dest);
1008         return err;
1009 }
1010
1011 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
1012 static int
1013 iop_adma_xor_val_self_test(struct iop_adma_device *device)
1014 {
1015         int i, src_idx;
1016         struct page *dest;
1017         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
1018         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1019         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
1020         dma_addr_t dest_dma;
1021         struct dma_async_tx_descriptor *tx;
1022         struct dma_chan *dma_chan;
1023         dma_cookie_t cookie;
1024         u8 cmp_byte = 0;
1025         u32 cmp_word;
1026         u32 zero_sum_result;
1027         int err = 0;
1028         struct iop_adma_chan *iop_chan;
1029
1030         dev_dbg(device->common.dev, "%s\n", __func__);
1031
1032         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1033                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1034                 if (!xor_srcs[src_idx]) {
1035                         while (src_idx--)
1036                                 __free_page(xor_srcs[src_idx]);
1037                         return -ENOMEM;
1038                 }
1039         }
1040
1041         dest = alloc_page(GFP_KERNEL);
1042         if (!dest) {
1043                 while (src_idx--)
1044                         __free_page(xor_srcs[src_idx]);
1045                 return -ENOMEM;
1046         }
1047
1048         /* Fill in src buffers */
1049         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
1050                 u8 *ptr = page_address(xor_srcs[src_idx]);
1051                 for (i = 0; i < PAGE_SIZE; i++)
1052                         ptr[i] = (1 << src_idx);
1053         }
1054
1055         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
1056                 cmp_byte ^= (u8) (1 << src_idx);
1057
1058         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1059                         (cmp_byte << 8) | cmp_byte;
1060
1061         memset(page_address(dest), 0, PAGE_SIZE);
1062
1063         dma_chan = container_of(device->common.channels.next,
1064                                 struct dma_chan,
1065                                 device_node);
1066         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1067                 err = -ENODEV;
1068                 goto out;
1069         }
1070
1071         /* test xor */
1072         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
1073                                 PAGE_SIZE, DMA_FROM_DEVICE);
1074         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1075                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
1076                                            0, PAGE_SIZE, DMA_TO_DEVICE);
1077         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1078                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
1079                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1080
1081         cookie = iop_adma_tx_submit(tx);
1082         iop_adma_issue_pending(dma_chan);
1083         msleep(8);
1084
1085         if (iop_adma_status(dma_chan, cookie, NULL) !=
1086                 DMA_COMPLETE) {
1087                 dev_err(dma_chan->device->dev,
1088                         "Self-test xor timed out, disabling\n");
1089                 err = -ENODEV;
1090                 goto free_resources;
1091         }
1092
1093         iop_chan = to_iop_adma_chan(dma_chan);
1094         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1095                 PAGE_SIZE, DMA_FROM_DEVICE);
1096         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1097                 u32 *ptr = page_address(dest);
1098                 if (ptr[i] != cmp_word) {
1099                         dev_err(dma_chan->device->dev,
1100                                 "Self-test xor failed compare, disabling\n");
1101                         err = -ENODEV;
1102                         goto free_resources;
1103                 }
1104         }
1105         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1106                 PAGE_SIZE, DMA_TO_DEVICE);
1107
1108         /* skip zero sum if the capability is not present */
1109         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1110                 goto free_resources;
1111
1112         /* zero sum the sources with the destintation page */
1113         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1114                 zero_sum_srcs[i] = xor_srcs[i];
1115         zero_sum_srcs[i] = dest;
1116
1117         zero_sum_result = 1;
1118
1119         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1120                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1121                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1122                                            DMA_TO_DEVICE);
1123         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1124                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1125                                        &zero_sum_result,
1126                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1127
1128         cookie = iop_adma_tx_submit(tx);
1129         iop_adma_issue_pending(dma_chan);
1130         msleep(8);
1131
1132         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1133                 dev_err(dma_chan->device->dev,
1134                         "Self-test zero sum timed out, disabling\n");
1135                 err = -ENODEV;
1136                 goto free_resources;
1137         }
1138
1139         if (zero_sum_result != 0) {
1140                 dev_err(dma_chan->device->dev,
1141                         "Self-test zero sum failed compare, disabling\n");
1142                 err = -ENODEV;
1143                 goto free_resources;
1144         }
1145
1146         /* test for non-zero parity sum */
1147         zero_sum_result = 0;
1148         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1149                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1150                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1151                                            DMA_TO_DEVICE);
1152         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1153                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1154                                        &zero_sum_result,
1155                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1156
1157         cookie = iop_adma_tx_submit(tx);
1158         iop_adma_issue_pending(dma_chan);
1159         msleep(8);
1160
1161         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1162                 dev_err(dma_chan->device->dev,
1163                         "Self-test non-zero sum timed out, disabling\n");
1164                 err = -ENODEV;
1165                 goto free_resources;
1166         }
1167
1168         if (zero_sum_result != 1) {
1169                 dev_err(dma_chan->device->dev,
1170                         "Self-test non-zero sum failed compare, disabling\n");
1171                 err = -ENODEV;
1172                 goto free_resources;
1173         }
1174
1175 free_resources:
1176         iop_adma_free_chan_resources(dma_chan);
1177 out:
1178         src_idx = IOP_ADMA_NUM_SRC_TEST;
1179         while (src_idx--)
1180                 __free_page(xor_srcs[src_idx]);
1181         __free_page(dest);
1182         return err;
1183 }
1184
1185 #ifdef CONFIG_RAID6_PQ
1186 static int
1187 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1188 {
1189         /* combined sources, software pq results, and extra hw pq results */
1190         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1191         /* ptr to the extra hw pq buffers defined above */
1192         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1193         /* address conversion buffers (dma_map / page_address) */
1194         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1195         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1196         dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1197
1198         int i;
1199         struct dma_async_tx_descriptor *tx;
1200         struct dma_chan *dma_chan;
1201         dma_cookie_t cookie;
1202         u32 zero_sum_result;
1203         int err = 0;
1204         struct device *dev;
1205
1206         dev_dbg(device->common.dev, "%s\n", __func__);
1207
1208         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1209                 pq[i] = alloc_page(GFP_KERNEL);
1210                 if (!pq[i]) {
1211                         while (i--)
1212                                 __free_page(pq[i]);
1213                         return -ENOMEM;
1214                 }
1215         }
1216
1217         /* Fill in src buffers */
1218         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1219                 pq_sw[i] = page_address(pq[i]);
1220                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1221         }
1222         pq_sw[i] = page_address(pq[i]);
1223         pq_sw[i+1] = page_address(pq[i+1]);
1224
1225         dma_chan = container_of(device->common.channels.next,
1226                                 struct dma_chan,
1227                                 device_node);
1228         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1229                 err = -ENODEV;
1230                 goto out;
1231         }
1232
1233         dev = dma_chan->device->dev;
1234
1235         /* initialize the dests */
1236         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1237         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1238
1239         /* test pq */
1240         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1241         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1242         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1243                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1244                                          DMA_TO_DEVICE);
1245
1246         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1247                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1248                                   PAGE_SIZE,
1249                                   DMA_PREP_INTERRUPT |
1250                                   DMA_CTRL_ACK);
1251
1252         cookie = iop_adma_tx_submit(tx);
1253         iop_adma_issue_pending(dma_chan);
1254         msleep(8);
1255
1256         if (iop_adma_status(dma_chan, cookie, NULL) !=
1257                 DMA_COMPLETE) {
1258                 dev_err(dev, "Self-test pq timed out, disabling\n");
1259                 err = -ENODEV;
1260                 goto free_resources;
1261         }
1262
1263         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1264
1265         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1266                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1267                 dev_err(dev, "Self-test p failed compare, disabling\n");
1268                 err = -ENODEV;
1269                 goto free_resources;
1270         }
1271         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1272                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1273                 dev_err(dev, "Self-test q failed compare, disabling\n");
1274                 err = -ENODEV;
1275                 goto free_resources;
1276         }
1277
1278         /* test correct zero sum using the software generated pq values */
1279         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1280                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1281                                          DMA_TO_DEVICE);
1282
1283         zero_sum_result = ~0;
1284         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1285                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1286                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1287                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1288
1289         cookie = iop_adma_tx_submit(tx);
1290         iop_adma_issue_pending(dma_chan);
1291         msleep(8);
1292
1293         if (iop_adma_status(dma_chan, cookie, NULL) !=
1294                 DMA_COMPLETE) {
1295                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1296                 err = -ENODEV;
1297                 goto free_resources;
1298         }
1299
1300         if (zero_sum_result != 0) {
1301                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1302                         zero_sum_result);
1303                 err = -ENODEV;
1304                 goto free_resources;
1305         }
1306
1307         /* test incorrect zero sum */
1308         i = IOP_ADMA_NUM_SRC_TEST;
1309         memset(pq_sw[i] + 100, 0, 100);
1310         memset(pq_sw[i+1] + 200, 0, 200);
1311         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1312                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1313                                          DMA_TO_DEVICE);
1314
1315         zero_sum_result = 0;
1316         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1317                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1318                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1319                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1320
1321         cookie = iop_adma_tx_submit(tx);
1322         iop_adma_issue_pending(dma_chan);
1323         msleep(8);
1324
1325         if (iop_adma_status(dma_chan, cookie, NULL) !=
1326                 DMA_COMPLETE) {
1327                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1328                 err = -ENODEV;
1329                 goto free_resources;
1330         }
1331
1332         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1333                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1334                         zero_sum_result);
1335                 err = -ENODEV;
1336                 goto free_resources;
1337         }
1338
1339 free_resources:
1340         iop_adma_free_chan_resources(dma_chan);
1341 out:
1342         i = ARRAY_SIZE(pq);
1343         while (i--)
1344                 __free_page(pq[i]);
1345         return err;
1346 }
1347 #endif
1348
1349 static int iop_adma_remove(struct platform_device *dev)
1350 {
1351         struct iop_adma_device *device = platform_get_drvdata(dev);
1352         struct dma_chan *chan, *_chan;
1353         struct iop_adma_chan *iop_chan;
1354         struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev);
1355
1356         dma_async_device_unregister(&device->common);
1357
1358         dma_free_coherent(&dev->dev, plat_data->pool_size,
1359                         device->dma_desc_pool_virt, device->dma_desc_pool);
1360
1361         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1362                                 device_node) {
1363                 iop_chan = to_iop_adma_chan(chan);
1364                 list_del(&chan->device_node);
1365                 kfree(iop_chan);
1366         }
1367         kfree(device);
1368
1369         return 0;
1370 }
1371
1372 static int iop_adma_probe(struct platform_device *pdev)
1373 {
1374         struct resource *res;
1375         int ret = 0, i;
1376         struct iop_adma_device *adev;
1377         struct iop_adma_chan *iop_chan;
1378         struct dma_device *dma_dev;
1379         struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev);
1380
1381         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1382         if (!res)
1383                 return -ENODEV;
1384
1385         if (!devm_request_mem_region(&pdev->dev, res->start,
1386                                 resource_size(res), pdev->name))
1387                 return -EBUSY;
1388
1389         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1390         if (!adev)
1391                 return -ENOMEM;
1392         dma_dev = &adev->common;
1393
1394         /* allocate coherent memory for hardware descriptors
1395          * note: writecombine gives slightly better performance, but
1396          * requires that we explicitly flush the writes
1397          */
1398         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1399                                         plat_data->pool_size,
1400                                         &adev->dma_desc_pool,
1401                                         GFP_KERNEL)) == NULL) {
1402                 ret = -ENOMEM;
1403                 goto err_free_adev;
1404         }
1405
1406         dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1407                 __func__, adev->dma_desc_pool_virt,
1408                 (void *) adev->dma_desc_pool);
1409
1410         adev->id = plat_data->hw_id;
1411
1412         /* discover transaction capabilites from the platform data */
1413         dma_dev->cap_mask = plat_data->cap_mask;
1414
1415         adev->pdev = pdev;
1416         platform_set_drvdata(pdev, adev);
1417
1418         INIT_LIST_HEAD(&dma_dev->channels);
1419
1420         /* set base routines */
1421         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1422         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1423         dma_dev->device_tx_status = iop_adma_status;
1424         dma_dev->device_issue_pending = iop_adma_issue_pending;
1425         dma_dev->dev = &pdev->dev;
1426
1427         /* set prep routines based on capability */
1428         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1429                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1430         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1431                 dma_dev->max_xor = iop_adma_get_max_xor();
1432                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1433         }
1434         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1435                 dma_dev->device_prep_dma_xor_val =
1436                         iop_adma_prep_dma_xor_val;
1437         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1438                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1439                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1440         }
1441         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1442                 dma_dev->device_prep_dma_pq_val =
1443                         iop_adma_prep_dma_pq_val;
1444         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1445                 dma_dev->device_prep_dma_interrupt =
1446                         iop_adma_prep_dma_interrupt;
1447
1448         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1449         if (!iop_chan) {
1450                 ret = -ENOMEM;
1451                 goto err_free_dma;
1452         }
1453         iop_chan->device = adev;
1454
1455         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1456                                         resource_size(res));
1457         if (!iop_chan->mmr_base) {
1458                 ret = -ENOMEM;
1459                 goto err_free_iop_chan;
1460         }
1461         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1462                 iop_chan);
1463
1464         /* clear errors before enabling interrupts */
1465         iop_adma_device_clear_err_status(iop_chan);
1466
1467         for (i = 0; i < 3; i++) {
1468                 irq_handler_t handler[] = { iop_adma_eot_handler,
1469                                         iop_adma_eoc_handler,
1470                                         iop_adma_err_handler };
1471                 int irq = platform_get_irq(pdev, i);
1472                 if (irq < 0) {
1473                         ret = -ENXIO;
1474                         goto err_free_iop_chan;
1475                 } else {
1476                         ret = devm_request_irq(&pdev->dev, irq,
1477                                         handler[i], 0, pdev->name, iop_chan);
1478                         if (ret)
1479                                 goto err_free_iop_chan;
1480                 }
1481         }
1482
1483         spin_lock_init(&iop_chan->lock);
1484         INIT_LIST_HEAD(&iop_chan->chain);
1485         INIT_LIST_HEAD(&iop_chan->all_slots);
1486         iop_chan->common.device = dma_dev;
1487         dma_cookie_init(&iop_chan->common);
1488         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1489
1490         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1491                 ret = iop_adma_memcpy_self_test(adev);
1492                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1493                 if (ret)
1494                         goto err_free_iop_chan;
1495         }
1496
1497         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1498                 ret = iop_adma_xor_val_self_test(adev);
1499                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1500                 if (ret)
1501                         goto err_free_iop_chan;
1502         }
1503
1504         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1505             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1506                 #ifdef CONFIG_RAID6_PQ
1507                 ret = iop_adma_pq_zero_sum_self_test(adev);
1508                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1509                 #else
1510                 /* can not test raid6, so do not publish capability */
1511                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1512                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1513                 ret = 0;
1514                 #endif
1515                 if (ret)
1516                         goto err_free_iop_chan;
1517         }
1518
1519         dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n",
1520                  dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1521                  dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1522                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1523                  dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1524                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1525                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1526
1527         dma_async_device_register(dma_dev);
1528         goto out;
1529
1530  err_free_iop_chan:
1531         kfree(iop_chan);
1532  err_free_dma:
1533         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1534                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1535  err_free_adev:
1536         kfree(adev);
1537  out:
1538         return ret;
1539 }
1540
1541 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1542 {
1543         struct iop_adma_desc_slot *sw_desc, *grp_start;
1544         dma_cookie_t cookie;
1545         int slot_cnt, slots_per_op;
1546
1547         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1548
1549         spin_lock_bh(&iop_chan->lock);
1550         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1551         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1552         if (sw_desc) {
1553                 grp_start = sw_desc->group_head;
1554
1555                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1556                 async_tx_ack(&sw_desc->async_tx);
1557                 iop_desc_init_memcpy(grp_start, 0);
1558                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1559                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1560                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1561
1562                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1563
1564                 /* initialize the completed cookie to be less than
1565                  * the most recently used cookie
1566                  */
1567                 iop_chan->common.completed_cookie = cookie - 1;
1568
1569                 /* channel should not be busy */
1570                 BUG_ON(iop_chan_is_busy(iop_chan));
1571
1572                 /* clear any prior error-status bits */
1573                 iop_adma_device_clear_err_status(iop_chan);
1574
1575                 /* disable operation */
1576                 iop_chan_disable(iop_chan);
1577
1578                 /* set the descriptor address */
1579                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1580
1581                 /* 1/ don't add pre-chained descriptors
1582                  * 2/ dummy read to flush next_desc write
1583                  */
1584                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1585
1586                 /* run the descriptor */
1587                 iop_chan_enable(iop_chan);
1588         } else
1589                 dev_err(iop_chan->device->common.dev,
1590                         "failed to allocate null descriptor\n");
1591         spin_unlock_bh(&iop_chan->lock);
1592 }
1593
1594 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1595 {
1596         struct iop_adma_desc_slot *sw_desc, *grp_start;
1597         dma_cookie_t cookie;
1598         int slot_cnt, slots_per_op;
1599
1600         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1601
1602         spin_lock_bh(&iop_chan->lock);
1603         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1604         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1605         if (sw_desc) {
1606                 grp_start = sw_desc->group_head;
1607                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1608                 async_tx_ack(&sw_desc->async_tx);
1609                 iop_desc_init_null_xor(grp_start, 2, 0);
1610                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1611                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1612                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1613                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1614
1615                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1616
1617                 /* initialize the completed cookie to be less than
1618                  * the most recently used cookie
1619                  */
1620                 iop_chan->common.completed_cookie = cookie - 1;
1621
1622                 /* channel should not be busy */
1623                 BUG_ON(iop_chan_is_busy(iop_chan));
1624
1625                 /* clear any prior error-status bits */
1626                 iop_adma_device_clear_err_status(iop_chan);
1627
1628                 /* disable operation */
1629                 iop_chan_disable(iop_chan);
1630
1631                 /* set the descriptor address */
1632                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1633
1634                 /* 1/ don't add pre-chained descriptors
1635                  * 2/ dummy read to flush next_desc write
1636                  */
1637                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1638
1639                 /* run the descriptor */
1640                 iop_chan_enable(iop_chan);
1641         } else
1642                 dev_err(iop_chan->device->common.dev,
1643                         "failed to allocate null descriptor\n");
1644         spin_unlock_bh(&iop_chan->lock);
1645 }
1646
1647 static struct platform_driver iop_adma_driver = {
1648         .probe          = iop_adma_probe,
1649         .remove         = iop_adma_remove,
1650         .driver         = {
1651                 .owner  = THIS_MODULE,
1652                 .name   = "iop-adma",
1653         },
1654 };
1655
1656 module_platform_driver(iop_adma_driver);
1657
1658 MODULE_AUTHOR("Intel Corporation");
1659 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1660 MODULE_LICENSE("GPL");
1661 MODULE_ALIAS("platform:iop-adma");