int i;
struct ioat_dma_chan *ioat_chan;
struct device *dev = &device->pdev->dev;
+ struct dma_device *dma = &device->common;
- /*
- * IOAT ver.3 workarounds
- */
- if (device->version == IOAT_VER_3_0) {
- u32 chan_err_mask;
- u16 dev_id;
- u32 dmauncerrsts;
-
- /*
- * Write CHANERRMSK_INT with 3E07h to mask out the errors
- * that can cause stability issues for IOAT ver.3
- */
- chan_err_mask = 0x3E07;
- pci_write_config_dword(device->pdev,
- IOAT_PCI_CHANERRMASK_INT_OFFSET,
- chan_err_mask);
-
- /*
- * Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
- * (workaround for spurious config parity error after restart)
- */
- pci_read_config_word(device->pdev,
- IOAT_PCI_DEVICE_ID_OFFSET,
- &dev_id);
- if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
- dmauncerrsts = 0x10;
- pci_write_config_dword(device->pdev,
- IOAT_PCI_DMAUNCERRSTS_OFFSET,
- dmauncerrsts);
- }
- }
-
- device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+ INIT_LIST_HEAD(&dma->channels);
+ dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
- if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) {
- device->common.chancnt--;
- }
+ if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+ dma->chancnt--;
#endif
- for (i = 0; i < device->common.chancnt; i++) {
+ for (i = 0; i < dma->chancnt; i++) {
ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
if (!ioat_chan) {
- device->common.chancnt = i;
+ dma->chancnt = i;
break;
}
ioat_chan->xfercap = xfercap;
ioat_chan->desccount = 0;
INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
- if (ioat_chan->device->version == IOAT_VER_2_0)
- writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE |
- IOAT_DMA_DCA_ANY_CPU,
- ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
- else if (ioat_chan->device->version == IOAT_VER_3_0)
- writel(IOAT_DMA_DCA_ANY_CPU,
- ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
spin_lock_init(&ioat_chan->cleanup_lock);
spin_lock_init(&ioat_chan->desc_lock);
INIT_LIST_HEAD(&ioat_chan->free_desc);
INIT_LIST_HEAD(&ioat_chan->used_desc);
/* This should be made common somewhere in dmaengine.c */
ioat_chan->common.device = &device->common;
- list_add_tail(&ioat_chan->common.device_node,
- &device->common.channels);
+ list_add_tail(&ioat_chan->common.device_node, &dma->channels);
device->idx[i] = ioat_chan;
tasklet_init(&ioat_chan->cleanup_task,
ioat_dma_cleanup_tasklet,
(unsigned long) ioat_chan);
tasklet_disable(&ioat_chan->cleanup_task);
}
- return device->common.chancnt;
+ return dma->chancnt;
}
/**
pci_disable_msi(pdev);
goto intx;
}
- /*
- * CB 1.2 devices need a bit set in configuration space to enable MSI
- */
- if (device->version == IOAT_VER_1_2) {
- u32 dmactrl;
- pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
- dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
- pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
- }
goto done;
intx:
goto err_no_irq;
done:
+ if (device->intr_quirk)
+ device->intr_quirk(device);
intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
return 0;
writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
}
-struct ioatdma_device *
-ioat_dma_probe(struct pci_dev *pdev, void __iomem *iobase)
+static int ioat_probe(struct ioatdma_device *device)
{
- int err;
+ int err = -ENODEV;
+ struct dma_device *dma = &device->common;
+ struct pci_dev *pdev = device->pdev;
struct device *dev = &pdev->dev;
- struct ioatdma_device *device;
- struct dma_device *dma;
-
- device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
- if (!device)
- err = -ENOMEM;
- device->pdev = pdev;
- device->reg_base = iobase;
- device->version = readb(device->reg_base + IOAT_VER_OFFSET);
- dma = &device->common;
/* DMA coherent memory pool for DMA descriptor allocations */
device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
goto err_completion_pool;
}
- INIT_LIST_HEAD(&dma->channels);
ioat_dma_enumerate_channels(device);
+ dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
dma->device_free_chan_resources = ioat_dma_free_chan_resources;
- dma->dev = &pdev->dev;
-
- dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->device_is_tx_complete = ioat_dma_is_complete;
- switch (device->version) {
- case IOAT_VER_1_2:
- dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
- dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
- break;
- case IOAT_VER_2_0:
- case IOAT_VER_3_0:
- dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
- dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
- break;
- }
+ dma->dev = &pdev->dev;
dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
" %d channels, device version 0x%02x, driver version %s\n",
if (err)
goto err_self_test;
- err = dma_async_device_register(dma);
- if (err)
- goto err_self_test;
-
- ioat_set_tcp_copy_break(device);
-
- if (device->version != IOAT_VER_3_0) {
- INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
- schedule_delayed_work(&device->work,
- WATCHDOG_DELAY);
- }
-
- return device;
+ return 0;
err_self_test:
ioat_disable_interrupts(device);
err_completion_pool:
pci_pool_destroy(device->dma_pool);
err_dma_pool:
- return NULL;
+ return err;
+}
+
+static int ioat_register(struct ioatdma_device *device)
+{
+ int err = dma_async_device_register(&device->common);
+
+ if (err) {
+ ioat_disable_interrupts(device);
+ pci_pool_destroy(device->completion_pool);
+ pci_pool_destroy(device->dma_pool);
+ }
+
+ return err;
+}
+
+/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
+static void ioat1_intr_quirk(struct ioatdma_device *device)
+{
+ struct pci_dev *pdev = device->pdev;
+ u32 dmactrl;
+
+ pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
+ if (pdev->msi_enabled)
+ dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
+ else
+ dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
+ pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
+}
+
+int ioat1_dma_probe(struct ioatdma_device *device, int dca)
+{
+ struct pci_dev *pdev = device->pdev;
+ struct dma_device *dma;
+ int err;
+
+ device->intr_quirk = ioat1_intr_quirk;
+ dma = &device->common;
+ dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
+ dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
+
+ err = ioat_probe(device);
+ if (err)
+ return err;
+ ioat_set_tcp_copy_break(4096);
+ err = ioat_register(device);
+ if (err)
+ return err;
+ if (dca)
+ device->dca = ioat_dca_init(pdev, device->reg_base);
+
+ INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
+ schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+
+ return err;
+}
+
+int ioat2_dma_probe(struct ioatdma_device *device, int dca)
+{
+ struct pci_dev *pdev = device->pdev;
+ struct dma_device *dma;
+ struct dma_chan *chan;
+ struct ioat_dma_chan *ioat_chan;
+ int err;
+
+ dma = &device->common;
+ dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
+ dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
+
+ err = ioat_probe(device);
+ if (err)
+ return err;
+ ioat_set_tcp_copy_break(2048);
+
+ list_for_each_entry(chan, &dma->channels, device_node) {
+ ioat_chan = to_ioat_chan(chan);
+ writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
+ ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
+ }
+
+ err = ioat_register(device);
+ if (err)
+ return err;
+ if (dca)
+ device->dca = ioat2_dca_init(pdev, device->reg_base);
+
+ INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
+ schedule_delayed_work(&device->work, WATCHDOG_DELAY);
+
+ return err;
+}
+
+int ioat3_dma_probe(struct ioatdma_device *device, int dca)
+{
+ struct pci_dev *pdev = device->pdev;
+ struct dma_device *dma;
+ struct dma_chan *chan;
+ struct ioat_dma_chan *ioat_chan;
+ int err;
+ u16 dev_id;
+
+ dma = &device->common;
+ dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
+ dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
+
+ /* -= IOAT ver.3 workarounds =- */
+ /* Write CHANERRMSK_INT with 3E07h to mask out the errors
+ * that can cause stability issues for IOAT ver.3
+ */
+ pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07);
+
+ /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
+ * (workaround for spurious config parity error after restart)
+ */
+ pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
+ if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0)
+ pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10);
+
+ err = ioat_probe(device);
+ if (err)
+ return err;
+ ioat_set_tcp_copy_break(262144);
+
+ list_for_each_entry(chan, &dma->channels, device_node) {
+ ioat_chan = to_ioat_chan(chan);
+ writel(IOAT_DMA_DCA_ANY_CPU,
+ ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
+ }
+
+ err = ioat_register(device);
+ if (err)
+ return err;
+ if (dca)
+ device->dca = ioat3_dca_init(pdev, device->reg_base);
+
+ return err;
}
void ioat_dma_remove(struct ioatdma_device *device)
{ 0, }
};
-struct ioat_device {
- struct pci_dev *pdev;
- struct ioatdma_device *dma;
- struct dca_provider *dca;
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
- const struct pci_device_id *id);
+static int __devinit ioat_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id);
static void __devexit ioat_remove(struct pci_dev *pdev);
static int ioat_dca_enabled = 1;
static struct pci_driver ioat_pci_driver = {
.name = DRV_NAME,
.id_table = ioat_pci_tbl,
- .probe = ioat_probe,
+ .probe = ioat_pci_probe,
.remove = __devexit_p(ioat_remove),
};
-static int __devinit ioat_probe(struct pci_dev *pdev,
- const struct pci_device_id *id)
+static struct ioatdma_device *
+alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
+{
+ struct device *dev = &pdev->dev;
+ struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
+
+ if (!d)
+ return NULL;
+ d->pdev = pdev;
+ d->reg_base = iobase;
+ return d;
+}
+
+static int __devinit ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
void __iomem * const *iomap;
- void __iomem *iobase;
struct device *dev = &pdev->dev;
- struct ioat_device *device;
+ struct ioatdma_device *device;
int err;
err = pcim_enable_device(pdev);
if (!device)
return -ENOMEM;
- device->pdev = pdev;
- pci_set_drvdata(pdev, device);
- iobase = iomap[IOAT_MMIO_BAR];
-
pci_set_master(pdev);
- switch (readb(iobase + IOAT_VER_OFFSET)) {
- case IOAT_VER_1_2:
- device->dma = ioat_dma_probe(pdev, iobase);
- if (device->dma && ioat_dca_enabled)
- device->dca = ioat_dca_init(pdev, iobase);
- break;
- case IOAT_VER_2_0:
- device->dma = ioat_dma_probe(pdev, iobase);
- if (device->dma && ioat_dca_enabled)
- device->dca = ioat2_dca_init(pdev, iobase);
- break;
- case IOAT_VER_3_0:
- device->dma = ioat_dma_probe(pdev, iobase);
- if (device->dma && ioat_dca_enabled)
- device->dca = ioat3_dca_init(pdev, iobase);
- break;
- default:
+ device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
+ if (!device)
+ return -ENOMEM;
+ pci_set_drvdata(pdev, device);
+
+ device->version = readb(device->reg_base + IOAT_VER_OFFSET);
+ if (device->version == IOAT_VER_1_2)
+ err = ioat1_dma_probe(device, ioat_dca_enabled);
+ else if (device->version == IOAT_VER_2_0)
+ err = ioat2_dma_probe(device, ioat_dca_enabled);
+ else if (device->version >= IOAT_VER_3_0)
+ err = ioat3_dma_probe(device, ioat_dca_enabled);
+ else
return -ENODEV;
- }
- if (!device->dma) {
+ if (err) {
dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
return -ENODEV;
}
static void __devexit ioat_remove(struct pci_dev *pdev)
{
- struct ioat_device *device = pci_get_drvdata(pdev);
+ struct ioatdma_device *device = pci_get_drvdata(pdev);
+
+ if (!device)
+ return;
dev_err(&pdev->dev, "Removing dma and dca services\n");
if (device->dca) {
free_dca_provider(device->dca);
device->dca = NULL;
}
-
- if (device->dma) {
- ioat_dma_remove(device->dma);
- device->dma = NULL;
- }
+ ioat_dma_remove(device);
}
static int __init ioat_init_module(void)
projects / karo-tx-linux.git / commitdiff