drivers/block/rsxx/dma.c

   1 /*
   2 * Filename: dma.c
   3 *
   4 *
   5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
   6 *       Philip Kelleher <pjk1939@linux.vnet.ibm.com>
   7 *
   8 * (C) Copyright 2013 IBM Corporation
   9 *
  10 * This program is free software; you can redistribute it and/or
  11 * modify it under the terms of the GNU General Public License as
  12 * published by the Free Software Foundation; either version 2 of the
  13 * License, or (at your option) any later version.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18 * General Public License for more details.
  19 *
  20 * You should have received a copy of the GNU General Public License
  21 * along with this program; if not, write to the Free Software Foundation,
  22 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23 */
  24
  25 #include <linux/slab.h>
  26 #include "rsxx_priv.h"
  27
  28 struct rsxx_dma {
  29         struct list_head         list;
  30         u8                       cmd;
  31         unsigned int             laddr;     /* Logical address on the ramsan */
  32         struct {
  33                 u32              off;
  34                 u32              cnt;
  35         } sub_page;
  36         dma_addr_t               dma_addr;
  37         struct page              *page;
  38         unsigned int             pg_off;    /* Page Offset */
  39         rsxx_dma_cb              cb;
  40         void                     *cb_data;
  41 };
  42
  43 /* This timeout is used to detect a stalled DMA channel */
  44 #define DMA_ACTIVITY_TIMEOUT    msecs_to_jiffies(10000)
  45
  46 struct hw_status {
  47         u8      status;
  48         u8      tag;
  49         __le16  count;
  50         __le32  _rsvd2;
  51         __le64  _rsvd3;
  52 } __packed;
  53
  54 enum rsxx_dma_status {
  55         DMA_SW_ERR    = 0x1,
  56         DMA_HW_FAULT  = 0x2,
  57         DMA_CANCELLED = 0x4,
  58 };
  59
  60 struct hw_cmd {
  61         u8      command;
  62         u8      tag;
  63         u8      _rsvd;
  64         u8      sub_page; /* Bit[0:2]: 512byte offset */
  65                           /* Bit[4:6]: 512byte count */
  66         __le32  device_addr;
  67         __le64  host_addr;
  68 } __packed;
  69
  70 enum rsxx_hw_cmd {
  71         HW_CMD_BLK_DISCARD      = 0x70,
  72         HW_CMD_BLK_WRITE        = 0x80,
  73         HW_CMD_BLK_READ         = 0xC0,
  74         HW_CMD_BLK_RECON_READ   = 0xE0,
  75 };
  76
  77 enum rsxx_hw_status {
  78         HW_STATUS_CRC           = 0x01,
  79         HW_STATUS_HARD_ERR      = 0x02,
  80         HW_STATUS_SOFT_ERR      = 0x04,
  81         HW_STATUS_FAULT         = 0x08,
  82 };
  83
  84 #define STATUS_BUFFER_SIZE8     4096
  85 #define COMMAND_BUFFER_SIZE8    4096
  86
  87 static struct kmem_cache *rsxx_dma_pool;
  88
  89 struct dma_tracker {
  90         int                     next_tag;
  91         struct rsxx_dma *dma;
  92 };
  93
  94 #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
  95                 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
  96
  97 struct dma_tracker_list {
  98         spinlock_t              lock;
  99         int                     head;
 100         struct dma_tracker      list[0];
 101 };
 102
 103
 104 /*----------------- Misc Utility Functions -------------------*/
 105 static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
 106 {
 107         unsigned long long tgt_addr8;
 108
 109         tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
 110                       card->_stripe.upper_mask) |
 111                     ((addr8) & card->_stripe.lower_mask);
 112         do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
 113         return tgt_addr8;
 114 }
 115
 116 static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
 117 {
 118         unsigned int tgt;
 119
 120         tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
 121
 122         return tgt;
 123 }
 124
 125 static void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
 126 {
 127         /* Reset all DMA Command/Status Queues */
 128         iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
 129 }
 130
 131 static unsigned int get_dma_size(struct rsxx_dma *dma)
 132 {
 133         if (dma->sub_page.cnt)
 134                 return dma->sub_page.cnt << 9;
 135         else
 136                 return RSXX_HW_BLK_SIZE;
 137 }
 138
 139
 140 /*----------------- DMA Tracker -------------------*/
 141 static void set_tracker_dma(struct dma_tracker_list *trackers,
 142                             int tag,
 143                             struct rsxx_dma *dma)
 144 {
 145         trackers->list[tag].dma = dma;
 146 }
 147
 148 static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
 149                                             int tag)
 150 {
 151         return trackers->list[tag].dma;
 152 }
 153
 154 static int pop_tracker(struct dma_tracker_list *trackers)
 155 {
 156         int tag;
 157
 158         spin_lock(&trackers->lock);
 159         tag = trackers->head;
 160         if (tag != -1) {
 161                 trackers->head = trackers->list[tag].next_tag;
 162                 trackers->list[tag].next_tag = -1;
 163         }
 164         spin_unlock(&trackers->lock);
 165
 166         return tag;
 167 }
 168
 169 static void push_tracker(struct dma_tracker_list *trackers, int tag)
 170 {
 171         spin_lock(&trackers->lock);
 172         trackers->list[tag].next_tag = trackers->head;
 173         trackers->head = tag;
 174         trackers->list[tag].dma = NULL;
 175         spin_unlock(&trackers->lock);
 176 }
 177
 178
 179 /*----------------- Interrupt Coalescing -------------*/
 180 /*
 181  * Interrupt Coalescing Register Format:
 182  * Interrupt Timer (64ns units) [15:0]
 183  * Interrupt Count [24:16]
 184  * Reserved [31:25]
 185 */
 186 #define INTR_COAL_LATENCY_MASK       (0x0000ffff)
 187
 188 #define INTR_COAL_COUNT_SHIFT        16
 189 #define INTR_COAL_COUNT_BITS         9
 190 #define INTR_COAL_COUNT_MASK         (((1 << INTR_COAL_COUNT_BITS) - 1) << \
 191                                         INTR_COAL_COUNT_SHIFT)
 192 #define INTR_COAL_LATENCY_UNITS_NS   64
 193
 194
 195 static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
 196 {
 197         u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
 198
 199         if (mode == RSXX_INTR_COAL_DISABLED)
 200                 return 0;
 201
 202         return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
 203                         (latency_units & INTR_COAL_LATENCY_MASK);
 204
 205 }
 206
 207 static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
 208 {
 209         int i;
 210         u32 q_depth = 0;
 211         u32 intr_coal;
 212
 213         if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE)
 214                 return;
 215
 216         for (i = 0; i < card->n_targets; i++)
 217                 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
 218
 219         intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 220                                       q_depth / 2,
 221                                       card->config.data.intr_coal.latency);
 222         iowrite32(intr_coal, card->regmap + INTR_COAL);
 223 }
 224
 225 /*----------------- RSXX DMA Handling -------------------*/
 226 static void rsxx_complete_dma(struct rsxx_cardinfo *card,
 227                                   struct rsxx_dma *dma,
 228                                   unsigned int status)
 229 {
 230         if (status & DMA_SW_ERR)
 231                 printk_ratelimited(KERN_ERR
 232                                    "SW Error in DMA(cmd x%02x, laddr x%08x)\n",
 233                                    dma->cmd, dma->laddr);
 234         if (status & DMA_HW_FAULT)
 235                 printk_ratelimited(KERN_ERR
 236                                    "HW Fault in DMA(cmd x%02x, laddr x%08x)\n",
 237                                    dma->cmd, dma->laddr);
 238         if (status & DMA_CANCELLED)
 239                 printk_ratelimited(KERN_ERR
 240                                    "DMA Cancelled(cmd x%02x, laddr x%08x)\n",
 241                                    dma->cmd, dma->laddr);
 242
 243         if (dma->dma_addr)
 244                 pci_unmap_page(card->dev, dma->dma_addr, get_dma_size(dma),
 245                                dma->cmd == HW_CMD_BLK_WRITE ?
 246                                            PCI_DMA_TODEVICE :
 247                                            PCI_DMA_FROMDEVICE);
 248
 249         if (dma->cb)
 250                 dma->cb(card, dma->cb_data, status ? 1 : 0);
 251
 252         kmem_cache_free(rsxx_dma_pool, dma);
 253 }
 254
 255 static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
 256                                  struct rsxx_dma *dma)
 257 {
 258         /*
 259          * Requeued DMAs go to the front of the queue so they are issued
 260          * first.
 261          */
 262         spin_lock(&ctrl->queue_lock);
 263         list_add(&dma->list, &ctrl->queue);
 264         spin_unlock(&ctrl->queue_lock);
 265 }
 266
 267 static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
 268                                       struct rsxx_dma *dma,
 269                                       u8 hw_st)
 270 {
 271         unsigned int status = 0;
 272         int requeue_cmd = 0;
 273
 274         dev_dbg(CARD_TO_DEV(ctrl->card),
 275                 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
 276                 dma->cmd, dma->laddr, hw_st);
 277
 278         if (hw_st & HW_STATUS_CRC)
 279                 ctrl->stats.crc_errors++;
 280         if (hw_st & HW_STATUS_HARD_ERR)
 281                 ctrl->stats.hard_errors++;
 282         if (hw_st & HW_STATUS_SOFT_ERR)
 283                 ctrl->stats.soft_errors++;
 284
 285         switch (dma->cmd) {
 286         case HW_CMD_BLK_READ:
 287                 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 288                         if (ctrl->card->scrub_hard) {
 289                                 dma->cmd = HW_CMD_BLK_RECON_READ;
 290                                 requeue_cmd = 1;
 291                                 ctrl->stats.reads_retried++;
 292                         } else {
 293                                 status |= DMA_HW_FAULT;
 294                                 ctrl->stats.reads_failed++;
 295                         }
 296                 } else if (hw_st & HW_STATUS_FAULT) {
 297                         status |= DMA_HW_FAULT;
 298                         ctrl->stats.reads_failed++;
 299                 }
 300
 301                 break;
 302         case HW_CMD_BLK_RECON_READ:
 303                 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
 304                         /* Data could not be reconstructed. */
 305                         status |= DMA_HW_FAULT;
 306                         ctrl->stats.reads_failed++;
 307                 }
 308
 309                 break;
 310         case HW_CMD_BLK_WRITE:
 311                 status |= DMA_HW_FAULT;
 312                 ctrl->stats.writes_failed++;
 313
 314                 break;
 315         case HW_CMD_BLK_DISCARD:
 316                 status |= DMA_HW_FAULT;
 317                 ctrl->stats.discards_failed++;
 318
 319                 break;
 320         default:
 321                 dev_err(CARD_TO_DEV(ctrl->card),
 322                         "Unknown command in DMA!(cmd: x%02x "
 323                            "laddr x%08x st: x%02x\n",
 324                            dma->cmd, dma->laddr, hw_st);
 325                 status |= DMA_SW_ERR;
 326
 327                 break;
 328         }
 329
 330         if (requeue_cmd)
 331                 rsxx_requeue_dma(ctrl, dma);
 332         else
 333                 rsxx_complete_dma(ctrl->card, dma, status);
 334 }
 335
 336 static void dma_engine_stalled(unsigned long data)
 337 {
 338         struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
 339
 340         if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
 341                 return;
 342
 343         if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
 344                 /*
 345                  * The dma engine was stalled because the SW_CMD_IDX write
 346                  * was lost. Issue it again to recover.
 347                  */
 348                 dev_warn(CARD_TO_DEV(ctrl->card),
 349                         "SW_CMD_IDX write was lost, re-writing...\n");
 350                 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 351                 mod_timer(&ctrl->activity_timer,
 352                           jiffies + DMA_ACTIVITY_TIMEOUT);
 353         } else {
 354                 dev_warn(CARD_TO_DEV(ctrl->card),
 355                         "DMA channel %d has stalled, faulting interface.\n",
 356                         ctrl->id);
 357                 ctrl->card->dma_fault = 1;
 358         }
 359 }
 360
 361 static void rsxx_issue_dmas(struct work_struct *work)
 362 {
 363         struct rsxx_dma_ctrl *ctrl;
 364         struct rsxx_dma *dma;
 365         int tag;
 366         int cmds_pending = 0;
 367         struct hw_cmd *hw_cmd_buf;
 368
 369         ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
 370         hw_cmd_buf = ctrl->cmd.buf;
 371
 372         if (unlikely(ctrl->card->halt))
 373                 return;
 374
 375         while (1) {
 376                 spin_lock(&ctrl->queue_lock);
 377                 if (list_empty(&ctrl->queue)) {
 378                         spin_unlock(&ctrl->queue_lock);
 379                         break;
 380                 }
 381                 spin_unlock(&ctrl->queue_lock);
 382
 383                 tag = pop_tracker(ctrl->trackers);
 384                 if (tag == -1)
 385                         break;
 386
 387                 spin_lock(&ctrl->queue_lock);
 388                 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
 389                 list_del(&dma->list);
 390                 ctrl->stats.sw_q_depth--;
 391                 spin_unlock(&ctrl->queue_lock);
 392
 393                 /*
 394                  * This will catch any DMAs that slipped in right before the
 395                  * fault, but was queued after all the other DMAs were
 396                  * cancelled.
 397                  */
 398                 if (unlikely(ctrl->card->dma_fault)) {
 399                         push_tracker(ctrl->trackers, tag);
 400                         rsxx_complete_dma(ctrl->card, dma, DMA_CANCELLED);
 401                         continue;
 402                 }
 403
 404                 set_tracker_dma(ctrl->trackers, tag, dma);
 405                 hw_cmd_buf[ctrl->cmd.idx].command  = dma->cmd;
 406                 hw_cmd_buf[ctrl->cmd.idx].tag      = tag;
 407                 hw_cmd_buf[ctrl->cmd.idx]._rsvd    = 0;
 408                 hw_cmd_buf[ctrl->cmd.idx].sub_page =
 409                                         ((dma->sub_page.cnt & 0x7) << 4) |
 410                                          (dma->sub_page.off & 0x7);
 411
 412                 hw_cmd_buf[ctrl->cmd.idx].device_addr =
 413                                         cpu_to_le32(dma->laddr);
 414
 415                 hw_cmd_buf[ctrl->cmd.idx].host_addr =
 416                                         cpu_to_le64(dma->dma_addr);
 417
 418                 dev_dbg(CARD_TO_DEV(ctrl->card),
 419                         "Issue DMA%d(laddr %d tag %d) to idx %d\n",
 420                         ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
 421
 422                 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
 423                 cmds_pending++;
 424
 425                 if (dma->cmd == HW_CMD_BLK_WRITE)
 426                         ctrl->stats.writes_issued++;
 427                 else if (dma->cmd == HW_CMD_BLK_DISCARD)
 428                         ctrl->stats.discards_issued++;
 429                 else
 430                         ctrl->stats.reads_issued++;
 431         }
 432
 433         /* Let HW know we've queued commands. */
 434         if (cmds_pending) {
 435                 /*
 436                  * We must guarantee that the CPU writes to 'ctrl->cmd.buf'
 437                  * (which is in PCI-consistent system-memory) from the loop
 438                  * above make it into the coherency domain before the
 439                  * following PIO "trigger" updating the cmd.idx.  A WMB is
 440                  * sufficient. We need not explicitly CPU cache-flush since
 441                  * the memory is a PCI-consistent (ie; coherent) mapping.
 442                  */
 443                 wmb();
 444
 445                 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
 446                 mod_timer(&ctrl->activity_timer,
 447                           jiffies + DMA_ACTIVITY_TIMEOUT);
 448                 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 449         }
 450 }
 451
 452 static void rsxx_dma_done(struct work_struct *work)
 453 {
 454         struct rsxx_dma_ctrl *ctrl;
 455         struct rsxx_dma *dma;
 456         unsigned long flags;
 457         u16 count;
 458         u8 status;
 459         u8 tag;
 460         struct hw_status *hw_st_buf;
 461
 462         ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
 463         hw_st_buf = ctrl->status.buf;
 464
 465         if (unlikely(ctrl->card->halt) ||
 466             unlikely(ctrl->card->dma_fault))
 467                 return;
 468
 469         count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 470
 471         while (count == ctrl->e_cnt) {
 472                 /*
 473                  * The read memory-barrier is necessary to keep aggressive
 474                  * processors/optimizers (such as the PPC Apple G5) from
 475                  * reordering the following status-buffer tag & status read
 476                  * *before* the count read on subsequent iterations of the
 477                  * loop!
 478                  */
 479                 rmb();
 480
 481                 status = hw_st_buf[ctrl->status.idx].status;
 482                 tag    = hw_st_buf[ctrl->status.idx].tag;
 483
 484                 dma = get_tracker_dma(ctrl->trackers, tag);
 485                 if (dma == NULL) {
 486                         spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 487                         rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
 488                         spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 489
 490                         dev_err(CARD_TO_DEV(ctrl->card),
 491                                 "No tracker for tag %d "
 492                                 "(idx %d id %d)\n",
 493                                 tag, ctrl->status.idx, ctrl->id);
 494                         return;
 495                 }
 496
 497                 dev_dbg(CARD_TO_DEV(ctrl->card),
 498                         "Completing DMA%d"
 499                         "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
 500                         ctrl->id, dma->laddr, tag, status, count,
 501                         ctrl->status.idx);
 502
 503                 atomic_dec(&ctrl->stats.hw_q_depth);
 504
 505                 mod_timer(&ctrl->activity_timer,
 506                           jiffies + DMA_ACTIVITY_TIMEOUT);
 507
 508                 if (status)
 509                         rsxx_handle_dma_error(ctrl, dma, status);
 510                 else
 511                         rsxx_complete_dma(ctrl->card, dma, 0);
 512
 513                 push_tracker(ctrl->trackers, tag);
 514
 515                 ctrl->status.idx = (ctrl->status.idx + 1) &
 516                                    RSXX_CS_IDX_MASK;
 517                 ctrl->e_cnt++;
 518
 519                 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
 520         }
 521
 522         dma_intr_coal_auto_tune(ctrl->card);
 523
 524         if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
 525                 del_timer_sync(&ctrl->activity_timer);
 526
 527         spin_lock_irqsave(&ctrl->card->irq_lock, flags);
 528         rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
 529         spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
 530
 531         spin_lock(&ctrl->queue_lock);
 532         if (ctrl->stats.sw_q_depth)
 533                 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
 534         spin_unlock(&ctrl->queue_lock);
 535 }
 536
 537 static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card,
 538                                       struct list_head *q)
 539 {
 540         struct rsxx_dma *dma;
 541         struct rsxx_dma *tmp;
 542         int cnt = 0;
 543
 544         list_for_each_entry_safe(dma, tmp, q, list) {
 545                 list_del(&dma->list);
 546
 547                 if (dma->dma_addr)
 548                         pci_unmap_page(card->dev, dma->dma_addr,
 549                                        get_dma_size(dma),
 550                                        (dma->cmd == HW_CMD_BLK_WRITE) ?
 551                                        PCI_DMA_TODEVICE :
 552                                        PCI_DMA_FROMDEVICE);
 553                 kmem_cache_free(rsxx_dma_pool, dma);
 554                 cnt++;
 555         }
 556
 557         return cnt;
 558 }
 559
 560 static int rsxx_queue_discard(struct rsxx_cardinfo *card,
 561                                   struct list_head *q,
 562                                   unsigned int laddr,
 563                                   rsxx_dma_cb cb,
 564                                   void *cb_data)
 565 {
 566         struct rsxx_dma *dma;
 567
 568         dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 569         if (!dma)
 570                 return -ENOMEM;
 571
 572         dma->cmd          = HW_CMD_BLK_DISCARD;
 573         dma->laddr        = laddr;
 574         dma->dma_addr     = 0;
 575         dma->sub_page.off = 0;
 576         dma->sub_page.cnt = 0;
 577         dma->page         = NULL;
 578         dma->pg_off       = 0;
 579         dma->cb           = cb;
 580         dma->cb_data      = cb_data;
 581
 582         dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
 583
 584         list_add_tail(&dma->list, q);
 585
 586         return 0;
 587 }
 588
 589 static int rsxx_queue_dma(struct rsxx_cardinfo *card,
 590                               struct list_head *q,
 591                               int dir,
 592                               unsigned int dma_off,
 593                               unsigned int dma_len,
 594                               unsigned int laddr,
 595                               struct page *page,
 596                               unsigned int pg_off,
 597                               rsxx_dma_cb cb,
 598                               void *cb_data)
 599 {
 600         struct rsxx_dma *dma;
 601
 602         dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
 603         if (!dma)
 604                 return -ENOMEM;
 605
 606         dma->dma_addr = pci_map_page(card->dev, page, pg_off, dma_len,
 607                                      dir ? PCI_DMA_TODEVICE :
 608                                      PCI_DMA_FROMDEVICE);
 609         if (!dma->dma_addr) {
 610                 kmem_cache_free(rsxx_dma_pool, dma);
 611                 return -ENOMEM;
 612         }
 613
 614         dma->cmd          = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
 615         dma->laddr        = laddr;
 616         dma->sub_page.off = (dma_off >> 9);
 617         dma->sub_page.cnt = (dma_len >> 9);
 618         dma->page         = page;
 619         dma->pg_off       = pg_off;
 620         dma->cb           = cb;
 621         dma->cb_data      = cb_data;
 622
 623         dev_dbg(CARD_TO_DEV(card),
 624                 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
 625                 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
 626                 dma->sub_page.cnt, dma->page, dma->pg_off);
 627
 628         /* Queue the DMA */
 629         list_add_tail(&dma->list, q);
 630
 631         return 0;
 632 }
 633
 634 int rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
 635                            struct bio *bio,
 636                            atomic_t *n_dmas,
 637                            rsxx_dma_cb cb,
 638                            void *cb_data)
 639 {
 640         struct list_head dma_list[RSXX_MAX_TARGETS];
 641         struct bio_vec *bvec;
 642         unsigned long long addr8;
 643         unsigned int laddr;
 644         unsigned int bv_len;
 645         unsigned int bv_off;
 646         unsigned int dma_off;
 647         unsigned int dma_len;
 648         int dma_cnt[RSXX_MAX_TARGETS];
 649         int tgt;
 650         int st;
 651         int i;
 652
 653         addr8 = bio->bi_sector << 9; /* sectors are 512 bytes */
 654         atomic_set(n_dmas, 0);
 655
 656         for (i = 0; i < card->n_targets; i++) {
 657                 INIT_LIST_HEAD(&dma_list[i]);
 658                 dma_cnt[i] = 0;
 659         }
 660
 661         if (bio->bi_rw & REQ_DISCARD) {
 662                 bv_len = bio->bi_size;
 663
 664                 while (bv_len > 0) {
 665                         tgt   = rsxx_get_dma_tgt(card, addr8);
 666                         laddr = rsxx_addr8_to_laddr(addr8, card);
 667
 668                         st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
 669                                                     cb, cb_data);
 670                         if (st)
 671                                 goto bvec_err;
 672
 673                         dma_cnt[tgt]++;
 674                         atomic_inc(n_dmas);
 675                         addr8  += RSXX_HW_BLK_SIZE;
 676                         bv_len -= RSXX_HW_BLK_SIZE;
 677                 }
 678         } else {
 679                 bio_for_each_segment(bvec, bio, i) {
 680                         bv_len = bvec->bv_len;
 681                         bv_off = bvec->bv_offset;
 682
 683                         while (bv_len > 0) {
 684                                 tgt   = rsxx_get_dma_tgt(card, addr8);
 685                                 laddr = rsxx_addr8_to_laddr(addr8, card);
 686                                 dma_off = addr8 & RSXX_HW_BLK_MASK;
 687                                 dma_len = min(bv_len,
 688                                               RSXX_HW_BLK_SIZE - dma_off);
 689
 690                                 st = rsxx_queue_dma(card, &dma_list[tgt],
 691                                                         bio_data_dir(bio),
 692                                                         dma_off, dma_len,
 693                                                         laddr, bvec->bv_page,
 694                                                         bv_off, cb, cb_data);
 695                                 if (st)
 696                                         goto bvec_err;
 697
 698                                 dma_cnt[tgt]++;
 699                                 atomic_inc(n_dmas);
 700                                 addr8  += dma_len;
 701                                 bv_off += dma_len;
 702                                 bv_len -= dma_len;
 703                         }
 704                 }
 705         }
 706
 707         for (i = 0; i < card->n_targets; i++) {
 708                 if (!list_empty(&dma_list[i])) {
 709                         spin_lock(&card->ctrl[i].queue_lock);
 710                         card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
 711                         list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
 712                         spin_unlock(&card->ctrl[i].queue_lock);
 713
 714                         queue_work(card->ctrl[i].issue_wq,
 715                                    &card->ctrl[i].issue_dma_work);
 716                 }
 717         }
 718
 719         return 0;
 720
 721 bvec_err:
 722         for (i = 0; i < card->n_targets; i++)
 723                 rsxx_cleanup_dma_queue(card, &dma_list[i]);
 724
 725         return st;
 726 }
 727
 728
 729 /*----------------- DMA Engine Initialization & Setup -------------------*/
 730 static int rsxx_dma_ctrl_init(struct pci_dev *dev,
 731                                   struct rsxx_dma_ctrl *ctrl)
 732 {
 733         int i;
 734
 735         memset(&ctrl->stats, 0, sizeof(ctrl->stats));
 736
 737         ctrl->status.buf = pci_alloc_consistent(dev, STATUS_BUFFER_SIZE8,
 738                                                 &ctrl->status.dma_addr);
 739         ctrl->cmd.buf = pci_alloc_consistent(dev, COMMAND_BUFFER_SIZE8,
 740                                              &ctrl->cmd.dma_addr);
 741         if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
 742                 return -ENOMEM;
 743
 744         ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
 745         if (!ctrl->trackers)
 746                 return -ENOMEM;
 747
 748         ctrl->trackers->head = 0;
 749         for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
 750                 ctrl->trackers->list[i].next_tag = i + 1;
 751                 ctrl->trackers->list[i].dma = NULL;
 752         }
 753         ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
 754         spin_lock_init(&ctrl->trackers->lock);
 755
 756         spin_lock_init(&ctrl->queue_lock);
 757         INIT_LIST_HEAD(&ctrl->queue);
 758
 759         setup_timer(&ctrl->activity_timer, dma_engine_stalled,
 760                                         (unsigned long)ctrl);
 761
 762         ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
 763         if (!ctrl->issue_wq)
 764                 return -ENOMEM;
 765
 766         ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
 767         if (!ctrl->done_wq)
 768                 return -ENOMEM;
 769
 770         INIT_WORK(&ctrl->issue_dma_work, rsxx_issue_dmas);
 771         INIT_WORK(&ctrl->dma_done_work, rsxx_dma_done);
 772
 773         memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
 774         iowrite32(lower_32_bits(ctrl->status.dma_addr),
 775                   ctrl->regmap + SB_ADD_LO);
 776         iowrite32(upper_32_bits(ctrl->status.dma_addr),
 777                   ctrl->regmap + SB_ADD_HI);
 778
 779         memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
 780         iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
 781         iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
 782
 783         ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
 784         if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 785                 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
 786                          ctrl->status.idx);
 787                 return -EINVAL;
 788         }
 789         iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
 790         iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
 791
 792         ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
 793         if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
 794                 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
 795                          ctrl->status.idx);
 796                 return -EINVAL;
 797         }
 798         iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
 799         iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
 800
 801         wmb();
 802
 803         return 0;
 804 }
 805
 806 static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
 807                               unsigned int stripe_size8)
 808 {
 809         if (!is_power_of_2(stripe_size8)) {
 810                 dev_err(CARD_TO_DEV(card),
 811                         "stripe_size is NOT a power of 2!\n");
 812                 return -EINVAL;
 813         }
 814
 815         card->_stripe.lower_mask = stripe_size8 - 1;
 816
 817         card->_stripe.upper_mask  = ~(card->_stripe.lower_mask);
 818         card->_stripe.upper_shift = ffs(card->n_targets) - 1;
 819
 820         card->_stripe.target_mask = card->n_targets - 1;
 821         card->_stripe.target_shift = ffs(stripe_size8) - 1;
 822
 823         dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask   = x%016llx\n",
 824                 card->_stripe.lower_mask);
 825         dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift  = x%016llx\n",
 826                 card->_stripe.upper_shift);
 827         dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask   = x%016llx\n",
 828                 card->_stripe.upper_mask);
 829         dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask  = x%016llx\n",
 830                 card->_stripe.target_mask);
 831         dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
 832                 card->_stripe.target_shift);
 833
 834         return 0;
 835 }
 836
 837 static int rsxx_dma_configure(struct rsxx_cardinfo *card)
 838 {
 839         u32 intr_coal;
 840
 841         intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
 842                                       card->config.data.intr_coal.count,
 843                                       card->config.data.intr_coal.latency);
 844         iowrite32(intr_coal, card->regmap + INTR_COAL);
 845
 846         return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
 847 }
 848
 849 int rsxx_dma_setup(struct rsxx_cardinfo *card)
 850 {
 851         unsigned long flags;
 852         int st;
 853         int i;
 854
 855         dev_info(CARD_TO_DEV(card),
 856                 "Initializing %d DMA targets\n",
 857                 card->n_targets);
 858
 859         /* Regmap is divided up into 4K chunks. One for each DMA channel */
 860         for (i = 0; i < card->n_targets; i++)
 861                 card->ctrl[i].regmap = card->regmap + (i * 4096);
 862
 863         card->dma_fault = 0;
 864
 865         /* Reset the DMA queues */
 866         rsxx_dma_queue_reset(card);
 867
 868         /************* Setup DMA Control *************/
 869         for (i = 0; i < card->n_targets; i++) {
 870                 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
 871                 if (st)
 872                         goto failed_dma_setup;
 873
 874                 card->ctrl[i].card = card;
 875                 card->ctrl[i].id = i;
 876         }
 877
 878         card->scrub_hard = 1;
 879
 880         if (card->config_valid)
 881                 rsxx_dma_configure(card);
 882
 883         /* Enable the interrupts after all setup has completed. */
 884         for (i = 0; i < card->n_targets; i++) {
 885                 spin_lock_irqsave(&card->irq_lock, flags);
 886                 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
 887                 spin_unlock_irqrestore(&card->irq_lock, flags);
 888         }
 889
 890         return 0;
 891
 892 failed_dma_setup:
 893         for (i = 0; i < card->n_targets; i++) {
 894                 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
 895
 896                 if (ctrl->issue_wq) {
 897                         destroy_workqueue(ctrl->issue_wq);
 898                         ctrl->issue_wq = NULL;
 899                 }
 900
 901                 if (ctrl->done_wq) {
 902                         destroy_workqueue(ctrl->done_wq);
 903                         ctrl->done_wq = NULL;
 904                 }
 905
 906                 if (ctrl->trackers)
 907                         vfree(ctrl->trackers);
 908
 909                 if (ctrl->status.buf)
 910                         pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 911                                             ctrl->status.buf,
 912                                             ctrl->status.dma_addr);
 913                 if (ctrl->cmd.buf)
 914                         pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 915                                             ctrl->cmd.buf, ctrl->cmd.dma_addr);
 916         }
 917
 918         return st;
 919 }
 920
 921
 922 void rsxx_dma_destroy(struct rsxx_cardinfo *card)
 923 {
 924         struct rsxx_dma_ctrl *ctrl;
 925         struct rsxx_dma *dma;
 926         int i, j;
 927         int cnt = 0;
 928
 929         for (i = 0; i < card->n_targets; i++) {
 930                 ctrl = &card->ctrl[i];
 931
 932                 if (ctrl->issue_wq) {
 933                         destroy_workqueue(ctrl->issue_wq);
 934                         ctrl->issue_wq = NULL;
 935                 }
 936
 937                 if (ctrl->done_wq) {
 938                         destroy_workqueue(ctrl->done_wq);
 939                         ctrl->done_wq = NULL;
 940                 }
 941
 942                 if (timer_pending(&ctrl->activity_timer))
 943                         del_timer_sync(&ctrl->activity_timer);
 944
 945                 /* Clean up the DMA queue */
 946                 spin_lock(&ctrl->queue_lock);
 947                 cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue);
 948                 spin_unlock(&ctrl->queue_lock);
 949
 950                 if (cnt)
 951                         dev_info(CARD_TO_DEV(card),
 952                                 "Freed %d queued DMAs on channel %d\n",
 953                                 cnt, i);
 954
 955                 /* Clean up issued DMAs */
 956                 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
 957                         dma = get_tracker_dma(ctrl->trackers, j);
 958                         if (dma) {
 959                                 pci_unmap_page(card->dev, dma->dma_addr,
 960                                                get_dma_size(dma),
 961                                                (dma->cmd == HW_CMD_BLK_WRITE) ?
 962                                                PCI_DMA_TODEVICE :
 963                                                PCI_DMA_FROMDEVICE);
 964                                 kmem_cache_free(rsxx_dma_pool, dma);
 965                                 cnt++;
 966                         }
 967                 }
 968
 969                 if (cnt)
 970                         dev_info(CARD_TO_DEV(card),
 971                                 "Freed %d pending DMAs on channel %d\n",
 972                                 cnt, i);
 973
 974                 vfree(ctrl->trackers);
 975
 976                 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
 977                                     ctrl->status.buf, ctrl->status.dma_addr);
 978                 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
 979                                     ctrl->cmd.buf, ctrl->cmd.dma_addr);
 980         }
 981 }
 982
 983
 984 int rsxx_dma_init(void)
 985 {
 986         rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
 987         if (!rsxx_dma_pool)
 988                 return -ENOMEM;
 989
 990         return 0;
 991 }
 992
 993
 994 void rsxx_dma_cleanup(void)
 995 {
 996         kmem_cache_destroy(rsxx_dma_pool);
 997 }
 998