2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
24 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
28 #include <linux/init.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/interrupt.h>
32 #include <linux/dmaengine.h>
33 #include <linux/delay.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/workqueue.h>
36 #include <linux/i7300_idle.h>
38 #include "registers.h"
41 static int ioat_pending_level = 4;
42 module_param(ioat_pending_level, int, 0644);
43 MODULE_PARM_DESC(ioat_pending_level,
44 "high-water mark for pushing ioat descriptors (default: 4)");
46 static void ioat_dma_chan_reset_part2(struct work_struct *work);
47 static void ioat_dma_chan_watchdog(struct work_struct *work);
49 /* internal functions */
50 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
51 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
53 static struct ioat_desc_sw *
54 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
55 static struct ioat_desc_sw *
56 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
58 static inline struct ioat_dma_chan *
59 ioat_chan_by_index(struct ioatdma_device *device, int index)
61 return device->idx[index];
65 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
67 * @data: interrupt data
69 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
71 struct ioatdma_device *instance = data;
72 struct ioat_dma_chan *ioat_chan;
73 unsigned long attnstatus;
77 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
79 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
82 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
83 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
87 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
88 for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
89 ioat_chan = ioat_chan_by_index(instance, bit);
90 tasklet_schedule(&ioat_chan->cleanup_task);
93 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
98 * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
100 * @data: interrupt data
102 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
104 struct ioat_dma_chan *ioat_chan = data;
106 tasklet_schedule(&ioat_chan->cleanup_task);
111 static void ioat_dma_cleanup_tasklet(unsigned long data);
114 * ioat_dma_enumerate_channels - find and initialize the device's channels
115 * @device: the device to be enumerated
117 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
122 struct ioat_dma_chan *ioat_chan;
123 struct device *dev = &device->pdev->dev;
124 struct dma_device *dma = &device->common;
126 INIT_LIST_HEAD(&dma->channels);
127 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
128 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
129 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
131 #ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
132 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
135 for (i = 0; i < dma->chancnt; i++) {
136 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
142 ioat_chan->device = device;
143 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
144 ioat_chan->xfercap = xfercap;
145 ioat_chan->desccount = 0;
146 INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
147 spin_lock_init(&ioat_chan->cleanup_lock);
148 spin_lock_init(&ioat_chan->desc_lock);
149 INIT_LIST_HEAD(&ioat_chan->free_desc);
150 INIT_LIST_HEAD(&ioat_chan->used_desc);
151 /* This should be made common somewhere in dmaengine.c */
152 ioat_chan->common.device = &device->common;
153 list_add_tail(&ioat_chan->common.device_node, &dma->channels);
154 device->idx[i] = ioat_chan;
155 tasklet_init(&ioat_chan->cleanup_task,
156 ioat_dma_cleanup_tasklet,
157 (unsigned long) ioat_chan);
158 tasklet_disable(&ioat_chan->cleanup_task);
164 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
166 * @chan: DMA channel handle
169 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
171 ioat_chan->pending = 0;
172 writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
175 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
177 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
179 if (ioat_chan->pending > 0) {
180 spin_lock_bh(&ioat_chan->desc_lock);
181 __ioat1_dma_memcpy_issue_pending(ioat_chan);
182 spin_unlock_bh(&ioat_chan->desc_lock);
187 __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
189 ioat_chan->pending = 0;
190 writew(ioat_chan->dmacount,
191 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
194 static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
196 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
198 if (ioat_chan->pending > 0) {
199 spin_lock_bh(&ioat_chan->desc_lock);
200 __ioat2_dma_memcpy_issue_pending(ioat_chan);
201 spin_unlock_bh(&ioat_chan->desc_lock);
207 * ioat_dma_chan_reset_part2 - reinit the channel after a reset
209 static void ioat_dma_chan_reset_part2(struct work_struct *work)
211 struct ioat_dma_chan *ioat_chan =
212 container_of(work, struct ioat_dma_chan, work.work);
213 struct ioat_desc_sw *desc;
215 spin_lock_bh(&ioat_chan->cleanup_lock);
216 spin_lock_bh(&ioat_chan->desc_lock);
218 ioat_chan->completion_virt->low = 0;
219 ioat_chan->completion_virt->high = 0;
220 ioat_chan->pending = 0;
223 * count the descriptors waiting, and be sure to do it
224 * right for both the CB1 line and the CB2 ring
226 ioat_chan->dmacount = 0;
227 if (ioat_chan->used_desc.prev) {
228 desc = to_ioat_desc(ioat_chan->used_desc.prev);
230 ioat_chan->dmacount++;
231 desc = to_ioat_desc(desc->node.next);
232 } while (&desc->node != ioat_chan->used_desc.next);
236 * write the new starting descriptor address
237 * this puts channel engine into ARMED state
239 desc = to_ioat_desc(ioat_chan->used_desc.prev);
240 switch (ioat_chan->device->version) {
242 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
243 ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
244 writel(((u64) desc->txd.phys) >> 32,
245 ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
247 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
248 + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
251 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
252 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
253 writel(((u64) desc->txd.phys) >> 32,
254 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
256 /* tell the engine to go with what's left to be done */
257 writew(ioat_chan->dmacount,
258 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
262 dev_err(to_dev(ioat_chan),
263 "chan%d reset - %d descs waiting, %d total desc\n",
264 chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
266 spin_unlock_bh(&ioat_chan->desc_lock);
267 spin_unlock_bh(&ioat_chan->cleanup_lock);
271 * ioat_dma_reset_channel - restart a channel
272 * @ioat_chan: IOAT DMA channel handle
274 static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat_chan)
276 u32 chansts, chanerr;
278 if (!ioat_chan->used_desc.prev)
281 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
282 chansts = (ioat_chan->completion_virt->low
283 & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
285 dev_err(to_dev(ioat_chan),
286 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
287 chan_num(ioat_chan), chansts, chanerr);
288 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
292 * whack it upside the head with a reset
293 * and wait for things to settle out.
294 * force the pending count to a really big negative
295 * to make sure no one forces an issue_pending
296 * while we're waiting.
299 spin_lock_bh(&ioat_chan->desc_lock);
300 ioat_chan->pending = INT_MIN;
301 writeb(IOAT_CHANCMD_RESET,
303 + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
304 spin_unlock_bh(&ioat_chan->desc_lock);
306 /* schedule the 2nd half instead of sleeping a long time */
307 schedule_delayed_work(&ioat_chan->work, RESET_DELAY);
311 * ioat_dma_chan_watchdog - watch for stuck channels
313 static void ioat_dma_chan_watchdog(struct work_struct *work)
315 struct ioatdma_device *device =
316 container_of(work, struct ioatdma_device, work.work);
317 struct ioat_dma_chan *ioat_chan;
327 unsigned long compl_desc_addr_hw;
329 for (i = 0; i < device->common.chancnt; i++) {
330 ioat_chan = ioat_chan_by_index(device, i);
332 if (ioat_chan->device->version == IOAT_VER_1_2
333 /* have we started processing anything yet */
334 && ioat_chan->last_completion
335 /* have we completed any since last watchdog cycle? */
336 && (ioat_chan->last_completion ==
337 ioat_chan->watchdog_completion)
338 /* has TCP stuck on one cookie since last watchdog? */
339 && (ioat_chan->watchdog_tcp_cookie ==
340 ioat_chan->watchdog_last_tcp_cookie)
341 && (ioat_chan->watchdog_tcp_cookie !=
342 ioat_chan->completed_cookie)
343 /* is there something in the chain to be processed? */
344 /* CB1 chain always has at least the last one processed */
345 && (ioat_chan->used_desc.prev != ioat_chan->used_desc.next)
346 && ioat_chan->pending == 0) {
349 * check CHANSTS register for completed
350 * descriptor address.
351 * if it is different than completion writeback,
353 * and it has changed since the last watchdog
354 * we can assume that channel
355 * is still working correctly
356 * and the problem is in completion writeback.
357 * update completion writeback
358 * with actual CHANSTS value
360 * try resetting the channel
363 completion_hw.low = readl(ioat_chan->reg_base +
364 IOAT_CHANSTS_OFFSET_LOW(ioat_chan->device->version));
365 completion_hw.high = readl(ioat_chan->reg_base +
366 IOAT_CHANSTS_OFFSET_HIGH(ioat_chan->device->version));
367 #if (BITS_PER_LONG == 64)
370 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
373 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
376 if ((compl_desc_addr_hw != 0)
377 && (compl_desc_addr_hw != ioat_chan->watchdog_completion)
378 && (compl_desc_addr_hw != ioat_chan->last_compl_desc_addr_hw)) {
379 ioat_chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
380 ioat_chan->completion_virt->low = completion_hw.low;
381 ioat_chan->completion_virt->high = completion_hw.high;
383 ioat_dma_reset_channel(ioat_chan);
384 ioat_chan->watchdog_completion = 0;
385 ioat_chan->last_compl_desc_addr_hw = 0;
389 * for version 2.0 if there are descriptors yet to be processed
390 * and the last completed hasn't changed since the last watchdog
391 * if they haven't hit the pending level
392 * issue the pending to push them through
394 * try resetting the channel
396 } else if (ioat_chan->device->version == IOAT_VER_2_0
397 && ioat_chan->used_desc.prev
398 && ioat_chan->last_completion
399 && ioat_chan->last_completion == ioat_chan->watchdog_completion) {
401 if (ioat_chan->pending < ioat_pending_level)
402 ioat2_dma_memcpy_issue_pending(&ioat_chan->common);
404 ioat_dma_reset_channel(ioat_chan);
405 ioat_chan->watchdog_completion = 0;
408 ioat_chan->last_compl_desc_addr_hw = 0;
409 ioat_chan->watchdog_completion
410 = ioat_chan->last_completion;
413 ioat_chan->watchdog_last_tcp_cookie =
414 ioat_chan->watchdog_tcp_cookie;
417 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
420 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
422 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
423 struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
424 struct ioat_desc_sw *prev, *new;
425 struct ioat_dma_descriptor *hw;
427 LIST_HEAD(new_chain);
431 unsigned long orig_flags;
432 unsigned int desc_count = 0;
434 /* src and dest and len are stored in the initial descriptor */
438 orig_flags = first->txd.flags;
441 spin_lock_bh(&ioat_chan->desc_lock);
442 prev = to_ioat_desc(ioat_chan->used_desc.prev);
445 copy = min_t(size_t, len, ioat_chan->xfercap);
447 async_tx_ack(&new->txd);
456 /* chain together the physical address list for the HW */
458 prev->hw->next = (u64) new->txd.phys;
464 list_add_tail(&new->node, &new_chain);
467 } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
470 dev_err(to_dev(ioat_chan), "tx submit failed\n");
471 spin_unlock_bh(&ioat_chan->desc_lock);
475 hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
476 if (first->txd.callback) {
477 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
479 /* move callback into to last desc */
480 new->txd.callback = first->txd.callback;
481 new->txd.callback_param
482 = first->txd.callback_param;
483 first->txd.callback = NULL;
484 first->txd.callback_param = NULL;
488 new->tx_cnt = desc_count;
489 new->txd.flags = orig_flags; /* client is in control of this ack */
491 /* store the original values for use in later cleanup */
493 new->src = first->src;
494 new->dst = first->dst;
495 new->len = first->len;
498 /* cookie incr and addition to used_list must be atomic */
499 cookie = ioat_chan->common.cookie;
503 ioat_chan->common.cookie = new->txd.cookie = cookie;
505 /* write address into NextDescriptor field of last desc in chain */
506 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
508 list_splice_tail(&new_chain, &ioat_chan->used_desc);
510 ioat_chan->dmacount += desc_count;
511 ioat_chan->pending += desc_count;
512 if (ioat_chan->pending >= ioat_pending_level)
513 __ioat1_dma_memcpy_issue_pending(ioat_chan);
514 spin_unlock_bh(&ioat_chan->desc_lock);
519 static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
521 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
522 struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
523 struct ioat_desc_sw *new;
524 struct ioat_dma_descriptor *hw;
529 unsigned long orig_flags;
530 unsigned int desc_count = 0;
532 /* src and dest and len are stored in the initial descriptor */
536 orig_flags = first->txd.flags;
540 * ioat_chan->desc_lock is still in force in version 2 path
541 * it gets unlocked at end of this function
544 copy = min_t(size_t, len, ioat_chan->xfercap);
546 async_tx_ack(&new->txd);
558 } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));
561 dev_err(to_dev(ioat_chan), "tx submit failed\n");
562 spin_unlock_bh(&ioat_chan->desc_lock);
566 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
567 if (first->txd.callback) {
568 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
570 /* move callback into to last desc */
571 new->txd.callback = first->txd.callback;
572 new->txd.callback_param
573 = first->txd.callback_param;
574 first->txd.callback = NULL;
575 first->txd.callback_param = NULL;
579 new->tx_cnt = desc_count;
580 new->txd.flags = orig_flags; /* client is in control of this ack */
582 /* store the original values for use in later cleanup */
584 new->src = first->src;
585 new->dst = first->dst;
586 new->len = first->len;
589 /* cookie incr and addition to used_list must be atomic */
590 cookie = ioat_chan->common.cookie;
594 ioat_chan->common.cookie = new->txd.cookie = cookie;
596 ioat_chan->dmacount += desc_count;
597 ioat_chan->pending += desc_count;
598 if (ioat_chan->pending >= ioat_pending_level)
599 __ioat2_dma_memcpy_issue_pending(ioat_chan);
600 spin_unlock_bh(&ioat_chan->desc_lock);
606 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
607 * @ioat_chan: the channel supplying the memory pool for the descriptors
608 * @flags: allocation flags
610 static struct ioat_desc_sw *
611 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat_chan, gfp_t flags)
613 struct ioat_dma_descriptor *desc;
614 struct ioat_desc_sw *desc_sw;
615 struct ioatdma_device *ioatdma_device;
618 ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
619 desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
623 desc_sw = kzalloc(sizeof(*desc_sw), flags);
624 if (unlikely(!desc_sw)) {
625 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
629 memset(desc, 0, sizeof(*desc));
630 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat_chan->common);
631 switch (ioat_chan->device->version) {
633 desc_sw->txd.tx_submit = ioat1_tx_submit;
637 desc_sw->txd.tx_submit = ioat2_tx_submit;
642 desc_sw->txd.phys = phys;
647 static int ioat_initial_desc_count = 256;
648 module_param(ioat_initial_desc_count, int, 0644);
649 MODULE_PARM_DESC(ioat_initial_desc_count,
650 "initial descriptors per channel (default: 256)");
653 * ioat2_dma_massage_chan_desc - link the descriptors into a circle
654 * @ioat_chan: the channel to be massaged
656 static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
658 struct ioat_desc_sw *desc, *_desc;
660 /* setup used_desc */
661 ioat_chan->used_desc.next = ioat_chan->free_desc.next;
662 ioat_chan->used_desc.prev = NULL;
664 /* pull free_desc out of the circle so that every node is a hw
665 * descriptor, but leave it pointing to the list
667 ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
668 ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;
670 /* circle link the hw descriptors */
671 desc = to_ioat_desc(ioat_chan->free_desc.next);
672 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
673 list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
674 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
679 * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
680 * @chan: the channel to be filled out
682 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
684 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
685 struct ioat_desc_sw *desc;
691 /* have we already been set up? */
692 if (!list_empty(&ioat_chan->free_desc))
693 return ioat_chan->desccount;
695 /* Setup register to interrupt and write completion status on error */
696 chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
697 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
698 IOAT_CHANCTRL_ERR_COMPLETION_EN;
699 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
701 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
703 dev_err(to_dev(ioat_chan), "CHANERR = %x, clearing\n", chanerr);
704 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
707 /* Allocate descriptors */
708 for (i = 0; i < ioat_initial_desc_count; i++) {
709 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
711 dev_err(to_dev(ioat_chan),
712 "Only %d initial descriptors\n", i);
715 list_add_tail(&desc->node, &tmp_list);
717 spin_lock_bh(&ioat_chan->desc_lock);
718 ioat_chan->desccount = i;
719 list_splice(&tmp_list, &ioat_chan->free_desc);
720 if (ioat_chan->device->version != IOAT_VER_1_2)
721 ioat2_dma_massage_chan_desc(ioat_chan);
722 spin_unlock_bh(&ioat_chan->desc_lock);
724 /* allocate a completion writeback area */
725 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
726 ioat_chan->completion_virt =
727 pci_pool_alloc(ioat_chan->device->completion_pool,
729 &ioat_chan->completion_addr);
730 memset(ioat_chan->completion_virt, 0,
731 sizeof(*ioat_chan->completion_virt));
732 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
733 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
734 writel(((u64) ioat_chan->completion_addr) >> 32,
735 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
737 tasklet_enable(&ioat_chan->cleanup_task);
738 ioat_dma_start_null_desc(ioat_chan); /* give chain to dma device */
739 return ioat_chan->desccount;
743 * ioat_dma_free_chan_resources - release all the descriptors
744 * @chan: the channel to be cleaned
746 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
748 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
749 struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
750 struct ioat_desc_sw *desc, *_desc;
751 int in_use_descs = 0;
753 /* Before freeing channel resources first check
754 * if they have been previously allocated for this channel.
756 if (ioat_chan->desccount == 0)
759 tasklet_disable(&ioat_chan->cleanup_task);
760 ioat_dma_memcpy_cleanup(ioat_chan);
762 /* Delay 100ms after reset to allow internal DMA logic to quiesce
763 * before removing DMA descriptor resources.
765 writeb(IOAT_CHANCMD_RESET,
767 + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
770 spin_lock_bh(&ioat_chan->desc_lock);
771 switch (ioat_chan->device->version) {
773 list_for_each_entry_safe(desc, _desc,
774 &ioat_chan->used_desc, node) {
776 list_del(&desc->node);
777 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
781 list_for_each_entry_safe(desc, _desc,
782 &ioat_chan->free_desc, node) {
783 list_del(&desc->node);
784 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
791 list_for_each_entry_safe(desc, _desc,
792 ioat_chan->free_desc.next, node) {
793 list_del(&desc->node);
794 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
798 desc = to_ioat_desc(ioat_chan->free_desc.next);
799 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
802 INIT_LIST_HEAD(&ioat_chan->free_desc);
803 INIT_LIST_HEAD(&ioat_chan->used_desc);
806 spin_unlock_bh(&ioat_chan->desc_lock);
808 pci_pool_free(ioatdma_device->completion_pool,
809 ioat_chan->completion_virt,
810 ioat_chan->completion_addr);
812 /* one is ok since we left it on there on purpose */
813 if (in_use_descs > 1)
814 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
817 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
818 ioat_chan->pending = 0;
819 ioat_chan->dmacount = 0;
820 ioat_chan->desccount = 0;
821 ioat_chan->watchdog_completion = 0;
822 ioat_chan->last_compl_desc_addr_hw = 0;
823 ioat_chan->watchdog_tcp_cookie =
824 ioat_chan->watchdog_last_tcp_cookie = 0;
828 * ioat_dma_get_next_descriptor - return the next available descriptor
829 * @ioat_chan: IOAT DMA channel handle
831 * Gets the next descriptor from the chain, and must be called with the
832 * channel's desc_lock held. Allocates more descriptors if the channel
835 static struct ioat_desc_sw *
836 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
838 struct ioat_desc_sw *new;
840 if (!list_empty(&ioat_chan->free_desc)) {
841 new = to_ioat_desc(ioat_chan->free_desc.next);
842 list_del(&new->node);
844 /* try to get another desc */
845 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
847 dev_err(to_dev(ioat_chan), "alloc failed\n");
856 static struct ioat_desc_sw *
857 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
859 struct ioat_desc_sw *new;
862 * used.prev points to where to start processing
863 * used.next points to next free descriptor
864 * if used.prev == NULL, there are none waiting to be processed
865 * if used.next == used.prev.prev, there is only one free descriptor,
866 * and we need to use it to as a noop descriptor before
867 * linking in a new set of descriptors, since the device
868 * has probably already read the pointer to it
870 if (ioat_chan->used_desc.prev &&
871 ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {
873 struct ioat_desc_sw *desc;
874 struct ioat_desc_sw *noop_desc;
877 /* set up the noop descriptor */
878 noop_desc = to_ioat_desc(ioat_chan->used_desc.next);
879 /* set size to non-zero value (channel returns error when size is 0) */
880 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
881 noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
882 noop_desc->hw->src_addr = 0;
883 noop_desc->hw->dst_addr = 0;
885 ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;
886 ioat_chan->pending++;
887 ioat_chan->dmacount++;
889 /* try to get a few more descriptors */
890 for (i = 16; i; i--) {
891 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
893 dev_err(to_dev(ioat_chan), "alloc failed\n");
896 list_add_tail(&desc->node, ioat_chan->used_desc.next);
899 = to_ioat_desc(desc->node.next)->txd.phys;
900 to_ioat_desc(desc->node.prev)->hw->next
902 ioat_chan->desccount++;
905 ioat_chan->used_desc.next = noop_desc->node.next;
907 new = to_ioat_desc(ioat_chan->used_desc.next);
909 ioat_chan->used_desc.next = new->node.next;
911 if (ioat_chan->used_desc.prev == NULL)
912 ioat_chan->used_desc.prev = &new->node;
918 static struct ioat_desc_sw *
919 ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
924 switch (ioat_chan->device->version) {
926 return ioat1_dma_get_next_descriptor(ioat_chan);
929 return ioat2_dma_get_next_descriptor(ioat_chan);
934 static struct dma_async_tx_descriptor *
935 ioat1_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
936 dma_addr_t dma_src, size_t len, unsigned long flags)
938 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
939 struct ioat_desc_sw *new;
941 spin_lock_bh(&ioat_chan->desc_lock);
942 new = ioat_dma_get_next_descriptor(ioat_chan);
943 spin_unlock_bh(&ioat_chan->desc_lock);
949 new->txd.flags = flags;
952 dev_err(to_dev(ioat_chan),
953 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
954 chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
959 static struct dma_async_tx_descriptor *
960 ioat2_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
961 dma_addr_t dma_src, size_t len, unsigned long flags)
963 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
964 struct ioat_desc_sw *new;
966 spin_lock_bh(&ioat_chan->desc_lock);
967 new = ioat2_dma_get_next_descriptor(ioat_chan);
970 * leave ioat_chan->desc_lock set in ioat 2 path
971 * it will get unlocked at end of tx_submit
978 new->txd.flags = flags;
981 spin_unlock_bh(&ioat_chan->desc_lock);
982 dev_err(to_dev(ioat_chan),
983 "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
984 chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
989 static void ioat_dma_cleanup_tasklet(unsigned long data)
991 struct ioat_dma_chan *chan = (void *)data;
992 ioat_dma_memcpy_cleanup(chan);
993 writew(IOAT_CHANCTRL_INT_DISABLE,
994 chan->reg_base + IOAT_CHANCTRL_OFFSET);
998 ioat_dma_unmap(struct ioat_dma_chan *ioat_chan, struct ioat_desc_sw *desc)
1000 if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1001 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1002 pci_unmap_single(ioat_chan->device->pdev,
1003 pci_unmap_addr(desc, dst),
1004 pci_unmap_len(desc, len),
1005 PCI_DMA_FROMDEVICE);
1007 pci_unmap_page(ioat_chan->device->pdev,
1008 pci_unmap_addr(desc, dst),
1009 pci_unmap_len(desc, len),
1010 PCI_DMA_FROMDEVICE);
1013 if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1014 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1015 pci_unmap_single(ioat_chan->device->pdev,
1016 pci_unmap_addr(desc, src),
1017 pci_unmap_len(desc, len),
1020 pci_unmap_page(ioat_chan->device->pdev,
1021 pci_unmap_addr(desc, src),
1022 pci_unmap_len(desc, len),
1028 * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
1029 * @chan: ioat channel to be cleaned up
1031 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
1033 unsigned long phys_complete;
1034 struct ioat_desc_sw *desc, *_desc;
1035 dma_cookie_t cookie = 0;
1036 unsigned long desc_phys;
1037 struct ioat_desc_sw *latest_desc;
1038 struct dma_async_tx_descriptor *tx;
1040 prefetch(ioat_chan->completion_virt);
1042 if (!spin_trylock_bh(&ioat_chan->cleanup_lock))
1045 /* The completion writeback can happen at any time,
1046 so reads by the driver need to be atomic operations
1047 The descriptor physical addresses are limited to 32-bits
1048 when the CPU can only do a 32-bit mov */
1050 #if (BITS_PER_LONG == 64)
1052 ioat_chan->completion_virt->full
1053 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1056 ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
1059 if ((ioat_chan->completion_virt->full
1060 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
1061 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
1062 dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
1063 readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
1065 /* TODO do something to salvage the situation */
1068 if (phys_complete == ioat_chan->last_completion) {
1069 spin_unlock_bh(&ioat_chan->cleanup_lock);
1071 * perhaps we're stuck so hard that the watchdog can't go off?
1072 * try to catch it after 2 seconds
1074 if (ioat_chan->device->version != IOAT_VER_3_0) {
1075 if (time_after(jiffies,
1076 ioat_chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1077 ioat_dma_chan_watchdog(&(ioat_chan->device->work.work));
1078 ioat_chan->last_completion_time = jiffies;
1083 ioat_chan->last_completion_time = jiffies;
1086 if (!spin_trylock_bh(&ioat_chan->desc_lock)) {
1087 spin_unlock_bh(&ioat_chan->cleanup_lock);
1091 switch (ioat_chan->device->version) {
1093 list_for_each_entry_safe(desc, _desc,
1094 &ioat_chan->used_desc, node) {
1097 * Incoming DMA requests may use multiple descriptors,
1098 * due to exceeding xfercap, perhaps. If so, only the
1099 * last one will have a cookie, and require unmapping.
1102 cookie = tx->cookie;
1103 ioat_dma_unmap(ioat_chan, desc);
1105 tx->callback(tx->callback_param);
1106 tx->callback = NULL;
1110 if (tx->phys != phys_complete) {
1112 * a completed entry, but not the last, so clean
1113 * up if the client is done with the descriptor
1115 if (async_tx_test_ack(tx)) {
1116 list_move_tail(&desc->node,
1117 &ioat_chan->free_desc);
1122 * last used desc. Do not remove, so we can
1123 * append from it, but don't look at it next
1128 /* TODO check status bits? */
1135 /* has some other thread has already cleaned up? */
1136 if (ioat_chan->used_desc.prev == NULL)
1139 /* work backwards to find latest finished desc */
1140 desc = to_ioat_desc(ioat_chan->used_desc.next);
1144 desc = to_ioat_desc(desc->node.prev);
1145 desc_phys = (unsigned long)tx->phys
1146 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1147 if (desc_phys == phys_complete) {
1151 } while (&desc->node != ioat_chan->used_desc.prev);
1153 if (latest_desc != NULL) {
1154 /* work forwards to clear finished descriptors */
1155 for (desc = to_ioat_desc(ioat_chan->used_desc.prev);
1156 &desc->node != latest_desc->node.next &&
1157 &desc->node != ioat_chan->used_desc.next;
1158 desc = to_ioat_desc(desc->node.next)) {
1160 cookie = tx->cookie;
1162 ioat_dma_unmap(ioat_chan, desc);
1164 tx->callback(tx->callback_param);
1165 tx->callback = NULL;
1170 /* move used.prev up beyond those that are finished */
1171 if (&desc->node == ioat_chan->used_desc.next)
1172 ioat_chan->used_desc.prev = NULL;
1174 ioat_chan->used_desc.prev = &desc->node;
1179 spin_unlock_bh(&ioat_chan->desc_lock);
1181 ioat_chan->last_completion = phys_complete;
1183 ioat_chan->completed_cookie = cookie;
1185 spin_unlock_bh(&ioat_chan->cleanup_lock);
1189 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1190 * @chan: IOAT DMA channel handle
1191 * @cookie: DMA transaction identifier
1192 * @done: if not %NULL, updated with last completed transaction
1193 * @used: if not %NULL, updated with last used transaction
1195 static enum dma_status
1196 ioat_dma_is_complete(struct dma_chan *chan, dma_cookie_t cookie,
1197 dma_cookie_t *done, dma_cookie_t *used)
1199 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
1200 dma_cookie_t last_used;
1201 dma_cookie_t last_complete;
1202 enum dma_status ret;
1204 last_used = chan->cookie;
1205 last_complete = ioat_chan->completed_cookie;
1206 ioat_chan->watchdog_tcp_cookie = cookie;
1209 *done = last_complete;
1213 ret = dma_async_is_complete(cookie, last_complete, last_used);
1214 if (ret == DMA_SUCCESS)
1217 ioat_dma_memcpy_cleanup(ioat_chan);
1219 last_used = chan->cookie;
1220 last_complete = ioat_chan->completed_cookie;
1223 *done = last_complete;
1227 return dma_async_is_complete(cookie, last_complete, last_used);
1230 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
1232 struct ioat_desc_sw *desc;
1234 spin_lock_bh(&ioat_chan->desc_lock);
1236 desc = ioat_dma_get_next_descriptor(ioat_chan);
1239 dev_err(to_dev(ioat_chan),
1240 "Unable to start null desc - get next desc failed\n");
1241 spin_unlock_bh(&ioat_chan->desc_lock);
1245 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL
1246 | IOAT_DMA_DESCRIPTOR_CTL_INT_GN
1247 | IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
1248 /* set size to non-zero value (channel returns error when size is 0) */
1249 desc->hw->size = NULL_DESC_BUFFER_SIZE;
1250 desc->hw->src_addr = 0;
1251 desc->hw->dst_addr = 0;
1252 async_tx_ack(&desc->txd);
1253 switch (ioat_chan->device->version) {
1256 list_add_tail(&desc->node, &ioat_chan->used_desc);
1258 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1259 ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1260 writel(((u64) desc->txd.phys) >> 32,
1261 ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1263 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
1264 + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
1268 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1269 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1270 writel(((u64) desc->txd.phys) >> 32,
1271 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1273 ioat_chan->dmacount++;
1274 __ioat2_dma_memcpy_issue_pending(ioat_chan);
1277 spin_unlock_bh(&ioat_chan->desc_lock);
1281 * Perform a IOAT transaction to verify the HW works.
1283 #define IOAT_TEST_SIZE 2000
1285 static void ioat_dma_test_callback(void *dma_async_param)
1287 struct completion *cmp = dma_async_param;
1293 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
1294 * @device: device to be tested
1296 static int ioat_dma_self_test(struct ioatdma_device *device)
1301 struct dma_device *dma = &device->common;
1302 struct device *dev = &device->pdev->dev;
1303 struct dma_chan *dma_chan;
1304 struct dma_async_tx_descriptor *tx;
1305 dma_addr_t dma_dest, dma_src;
1306 dma_cookie_t cookie;
1308 struct completion cmp;
1310 unsigned long flags;
1312 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1315 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1321 /* Fill in src buffer */
1322 for (i = 0; i < IOAT_TEST_SIZE; i++)
1325 /* Start copy, using first DMA channel */
1326 dma_chan = container_of(dma->channels.next, struct dma_chan,
1328 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1329 dev_err(dev, "selftest cannot allocate chan resource\n");
1334 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
1335 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
1336 flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE;
1337 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
1338 IOAT_TEST_SIZE, flags);
1340 dev_err(dev, "Self-test prep failed, disabling\n");
1342 goto free_resources;
1346 init_completion(&cmp);
1347 tx->callback = ioat_dma_test_callback;
1348 tx->callback_param = &cmp;
1349 cookie = tx->tx_submit(tx);
1351 dev_err(dev, "Self-test setup failed, disabling\n");
1353 goto free_resources;
1355 dma->device_issue_pending(dma_chan);
1357 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1360 dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
1362 dev_err(dev, "Self-test copy timed out, disabling\n");
1364 goto free_resources;
1366 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
1367 dev_err(dev, "Self-test copy failed compare, disabling\n");
1369 goto free_resources;
1373 dma->device_free_chan_resources(dma_chan);
1380 static char ioat_interrupt_style[32] = "msix";
1381 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
1382 sizeof(ioat_interrupt_style), 0644);
1383 MODULE_PARM_DESC(ioat_interrupt_style,
1384 "set ioat interrupt style: msix (default), "
1385 "msix-single-vector, msi, intx)");
1388 * ioat_dma_setup_interrupts - setup interrupt handler
1389 * @device: ioat device
1391 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
1393 struct ioat_dma_chan *ioat_chan;
1394 struct pci_dev *pdev = device->pdev;
1395 struct device *dev = &pdev->dev;
1396 struct msix_entry *msix;
1401 if (!strcmp(ioat_interrupt_style, "msix"))
1403 if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1404 goto msix_single_vector;
1405 if (!strcmp(ioat_interrupt_style, "msi"))
1407 if (!strcmp(ioat_interrupt_style, "intx"))
1409 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1413 /* The number of MSI-X vectors should equal the number of channels */
1414 msixcnt = device->common.chancnt;
1415 for (i = 0; i < msixcnt; i++)
1416 device->msix_entries[i].entry = i;
1418 err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1422 goto msix_single_vector;
1424 for (i = 0; i < msixcnt; i++) {
1425 msix = &device->msix_entries[i];
1426 ioat_chan = ioat_chan_by_index(device, i);
1427 err = devm_request_irq(dev, msix->vector,
1428 ioat_dma_do_interrupt_msix, 0,
1429 "ioat-msix", ioat_chan);
1431 for (j = 0; j < i; j++) {
1432 msix = &device->msix_entries[j];
1433 ioat_chan = ioat_chan_by_index(device, j);
1434 devm_free_irq(dev, msix->vector, ioat_chan);
1436 goto msix_single_vector;
1439 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1443 msix = &device->msix_entries[0];
1445 err = pci_enable_msix(pdev, device->msix_entries, 1);
1449 err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1450 "ioat-msix", device);
1452 pci_disable_msix(pdev);
1458 err = pci_enable_msi(pdev);
1462 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1463 "ioat-msi", device);
1465 pci_disable_msi(pdev);
1471 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1472 IRQF_SHARED, "ioat-intx", device);
1477 if (device->intr_quirk)
1478 device->intr_quirk(device);
1479 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1480 writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1484 /* Disable all interrupt generation */
1485 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1486 dev_err(dev, "no usable interrupts\n");
1490 static void ioat_disable_interrupts(struct ioatdma_device *device)
1492 /* Disable all interrupt generation */
1493 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1496 static int ioat_probe(struct ioatdma_device *device)
1499 struct dma_device *dma = &device->common;
1500 struct pci_dev *pdev = device->pdev;
1501 struct device *dev = &pdev->dev;
1503 /* DMA coherent memory pool for DMA descriptor allocations */
1504 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1505 sizeof(struct ioat_dma_descriptor),
1507 if (!device->dma_pool) {
1512 device->completion_pool = pci_pool_create("completion_pool", pdev,
1513 sizeof(u64), SMP_CACHE_BYTES,
1515 if (!device->completion_pool) {
1517 goto err_completion_pool;
1520 ioat_dma_enumerate_channels(device);
1522 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1523 dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1524 dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1525 dma->device_is_tx_complete = ioat_dma_is_complete;
1526 dma->dev = &pdev->dev;
1528 dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1529 " %d channels, device version 0x%02x, driver version %s\n",
1530 dma->chancnt, device->version, IOAT_DMA_VERSION);
1532 if (!dma->chancnt) {
1533 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1534 "zero channels detected\n");
1535 goto err_setup_interrupts;
1538 err = ioat_dma_setup_interrupts(device);
1540 goto err_setup_interrupts;
1542 err = ioat_dma_self_test(device);
1549 ioat_disable_interrupts(device);
1550 err_setup_interrupts:
1551 pci_pool_destroy(device->completion_pool);
1552 err_completion_pool:
1553 pci_pool_destroy(device->dma_pool);
1558 static int ioat_register(struct ioatdma_device *device)
1560 int err = dma_async_device_register(&device->common);
1563 ioat_disable_interrupts(device);
1564 pci_pool_destroy(device->completion_pool);
1565 pci_pool_destroy(device->dma_pool);
1571 /* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
1572 static void ioat1_intr_quirk(struct ioatdma_device *device)
1574 struct pci_dev *pdev = device->pdev;
1577 pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1578 if (pdev->msi_enabled)
1579 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1581 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1582 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1585 int ioat1_dma_probe(struct ioatdma_device *device, int dca)
1587 struct pci_dev *pdev = device->pdev;
1588 struct dma_device *dma;
1591 device->intr_quirk = ioat1_intr_quirk;
1592 dma = &device->common;
1593 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1594 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1596 err = ioat_probe(device);
1599 ioat_set_tcp_copy_break(4096);
1600 err = ioat_register(device);
1604 device->dca = ioat_dca_init(pdev, device->reg_base);
1606 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1607 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1612 int ioat2_dma_probe(struct ioatdma_device *device, int dca)
1614 struct pci_dev *pdev = device->pdev;
1615 struct dma_device *dma;
1616 struct dma_chan *chan;
1617 struct ioat_dma_chan *ioat_chan;
1620 dma = &device->common;
1621 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1622 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1624 err = ioat_probe(device);
1627 ioat_set_tcp_copy_break(2048);
1629 list_for_each_entry(chan, &dma->channels, device_node) {
1630 ioat_chan = to_ioat_chan(chan);
1631 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
1632 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1635 err = ioat_register(device);
1639 device->dca = ioat2_dca_init(pdev, device->reg_base);
1641 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1642 schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1647 int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1649 struct pci_dev *pdev = device->pdev;
1650 struct dma_device *dma;
1651 struct dma_chan *chan;
1652 struct ioat_dma_chan *ioat_chan;
1656 dma = &device->common;
1657 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1658 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1660 /* -= IOAT ver.3 workarounds =- */
1661 /* Write CHANERRMSK_INT with 3E07h to mask out the errors
1662 * that can cause stability issues for IOAT ver.3
1664 pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
1666 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1667 * (workaround for spurious config parity error after restart)
1669 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1670 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
1671 pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
1673 err = ioat_probe(device);
1676 ioat_set_tcp_copy_break(262144);
1678 list_for_each_entry(chan, &dma->channels, device_node) {
1679 ioat_chan = to_ioat_chan(chan);
1680 writel(IOAT_DMA_DCA_ANY_CPU,
1681 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1684 err = ioat_register(device);
1688 device->dca = ioat3_dca_init(pdev, device->reg_base);
1693 void ioat_dma_remove(struct ioatdma_device *device)
1695 struct dma_chan *chan, *_chan;
1696 struct ioat_dma_chan *ioat_chan;
1697 struct dma_device *dma = &device->common;
1699 if (device->version != IOAT_VER_3_0)
1700 cancel_delayed_work(&device->work);
1702 ioat_disable_interrupts(device);
1704 dma_async_device_unregister(dma);
1706 pci_pool_destroy(device->dma_pool);
1707 pci_pool_destroy(device->completion_pool);
1709 list_for_each_entry_safe(chan, _chan, &dma->channels, device_node) {
1710 ioat_chan = to_ioat_chan(chan);
1711 list_del(&chan->device_node);