vfio: powerpc/spapr/iommu/powernv/ioda2: Rework IOMMU ownership control

[karo-tx-linux.git] / arch / powerpc / platforms / powernv / pci-ioda.c

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index f710729a4a35f8ac27cb9cc1e76f0cb425d298bf..17a77522ea0dd23ee6f85e358e62874f387bc12a 100644 (file)
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1087,10 +1087,6 @@ static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
                return;
        }
 
-       pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
-                       GFP_KERNEL, hose->node);
-       pe->tce32_table->data = pe;
-
        /* Associate it with all child devices */
        pnv_ioda_setup_same_PE(bus, pe);
 
@@ -1296,7 +1292,7 @@ static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe
        bus = dev->bus;
        hose = pci_bus_to_host(bus);
        phb = hose->private_data;
-       tbl = pe->tce32_table;
+       tbl = pe->table_group.tables[0];
        addr = tbl->it_base;
 
        opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
@@ -1311,13 +1307,13 @@ static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe
        if (rc)
                pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
 
-       if (tbl->it_group) {
-               iommu_group_put(tbl->it_group);
-               BUG_ON(tbl->it_group);
+       pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
+       if (pe->table_group.group) {
+               iommu_group_put(pe->table_group.group);
+               BUG_ON(pe->table_group.group);
        }
        iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
        free_pages(addr, get_order(TCE32_TABLE_SIZE));
-       pe->tce32_table = NULL;
 }
 
 static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs)
@@ -1465,10 +1461,6 @@ static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
                        continue;
                }
 
-               pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
-                               GFP_KERNEL, hose->node);
-               pe->tce32_table->data = pe;
-
                /* Put PE to the list */
                mutex_lock(&phb->ioda.pe_list_mutex);
                list_add_tail(&pe->list, &phb->ioda.pe_list);
@@ -1603,7 +1595,7 @@ static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev
 
        pe = &phb->ioda.pe_array[pdn->pe_number];
        WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
-       set_iommu_table_base(&pdev->dev, pe->tce32_table);
+       set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
        /*
         * Note: iommu_add_device() will fail here as
         * for physical PE: the device is already added by now;
@@ -1637,7 +1629,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
        } else {
                dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
                set_dma_ops(&pdev->dev, &dma_iommu_ops);
-               set_iommu_table_base(&pdev->dev, pe->tce32_table);
+               set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
        }
        *pdev->dev.dma_mask = dma_mask;
        return 0;
@@ -1671,7 +1663,7 @@ static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
        struct pci_dev *dev;
 
        list_for_each_entry(dev, &bus->devices, bus_list) {
-               set_iommu_table_base(&dev->dev, pe->tce32_table);
+               set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
                iommu_add_device(&dev->dev);
 
                if (dev->subordinate)
@@ -1682,7 +1674,11 @@ static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,
                unsigned long index, unsigned long npages, bool rm)
 {
-       struct pnv_ioda_pe *pe = tbl->data;
+       struct iommu_table_group_link *tgl = list_first_entry_or_null(
+                       &tbl->it_group_list, struct iommu_table_group_link,
+                       next);
+       struct pnv_ioda_pe *pe = container_of(tgl->table_group,
+                       struct pnv_ioda_pe, table_group);
        __be64 __iomem *invalidate = rm ?
                (__be64 __iomem *)pe->tce_inval_reg_phys :
                (__be64 __iomem *)tbl->it_index;
@@ -1759,7 +1755,11 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = {
 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
                unsigned long index, unsigned long npages, bool rm)
 {
-       struct pnv_ioda_pe *pe = tbl->data;
+       struct iommu_table_group_link *tgl = list_first_entry_or_null(
+                       &tbl->it_group_list, struct iommu_table_group_link,
+                       next);
+       struct pnv_ioda_pe *pe = container_of(tgl->table_group,
+                       struct pnv_ioda_pe, table_group);
        unsigned long start, end, inc;
        __be64 __iomem *invalidate = rm ?
                (__be64 __iomem *)pe->tce_inval_reg_phys :
@@ -1835,8 +1835,10 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
        if (WARN_ON(pe->tce32_seg >= 0))
                return;
 
-       tbl = pe->tce32_table;
-       iommu_register_group(tbl, phb->hose->global_number, pe->pe_number);
+       tbl = pnv_pci_table_alloc(phb->hose->node);
+       iommu_register_group(&pe->table_group, phb->hose->global_number,
+                       pe->pe_number);
+       pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
 
        /* Grab a 32-bit TCE table */
        pe->tce32_seg = base;
@@ -1911,11 +1913,14 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
                pe->tce32_seg = -1;
        if (tce_mem)
                __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
+       if (tbl) {
+               pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
+               iommu_free_table(tbl, "pnv");
+       }
 }
 
-static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
+static void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
 {
-       struct pnv_ioda_pe *pe = tbl->data;
        uint16_t window_id = (pe->pe_number << 1 ) + 1;
        int64_t rc;
 
@@ -1942,19 +1947,31 @@ static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
                pe->tce_bypass_enabled = enable;
 }
 
-static void pnv_pci_ioda2_setup_bypass_pe(struct pnv_phb *phb,
-                                         struct pnv_ioda_pe *pe)
+#ifdef CONFIG_IOMMU_API
+static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
 {
-       /* TVE #1 is selected by PCI address bit 59 */
-       pe->tce_bypass_base = 1ull << 59;
+       struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
+                                               table_group);
+
+       iommu_take_ownership(table_group->tables[0]);
+       pnv_pci_ioda2_set_bypass(pe, false);
+}
 
-       /* Install set_bypass callback for VFIO */
-       pe->tce32_table->set_bypass = pnv_pci_ioda2_set_bypass;
+static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
+{
+       struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
+                                               table_group);
 
-       /* Enable bypass by default */
-       pnv_pci_ioda2_set_bypass(pe->tce32_table, true);
+       iommu_release_ownership(table_group->tables[0]);
+       pnv_pci_ioda2_set_bypass(pe, true);
 }
 
+static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
+       .take_ownership = pnv_ioda2_take_ownership,
+       .release_ownership = pnv_ioda2_release_ownership,
+};
+#endif
+
 static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
                                       struct pnv_ioda_pe *pe)
 {
@@ -1969,8 +1986,13 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
        if (WARN_ON(pe->tce32_seg >= 0))
                return;
 
-       tbl = pe->tce32_table;
-       iommu_register_group(tbl, phb->hose->global_number, pe->pe_number);
+       /* TVE #1 is selected by PCI address bit 59 */
+       pe->tce_bypass_base = 1ull << 59;
+
+       tbl = pnv_pci_table_alloc(phb->hose->node);
+       iommu_register_group(&pe->table_group, phb->hose->global_number,
+                       pe->pe_number);
+       pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
 
        /* The PE will reserve all possible 32-bits space */
        pe->tce32_seg = 0;
@@ -2021,6 +2043,9 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
        }
        tbl->it_ops = &pnv_ioda2_iommu_ops;
        iommu_init_table(tbl, phb->hose->node);
+#ifdef CONFIG_IOMMU_API
+       pe->table_group.ops = &pnv_pci_ioda2_ops;
+#endif
 
        if (pe->flags & PNV_IODA_PE_DEV) {
                /*
@@ -2035,7 +2060,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
 
        /* Also create a bypass window */
        if (!pnv_iommu_bypass_disabled)
-               pnv_pci_ioda2_setup_bypass_pe(phb, pe);
+               pnv_pci_ioda2_set_bypass(pe, true);
 
        return;
 fail:
@@ -2043,6 +2068,10 @@ fail:
                pe->tce32_seg = -1;
        if (tce_mem)
                __free_pages(tce_mem, get_order(tce_table_size));
+       if (tbl) {
+               pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
+               iommu_free_table(tbl, "pnv");
+       }
 }
 
 static void pnv_ioda_setup_dma(struct pnv_phb *phb)