amd-iommu: make address allocator aware of multiple aperture ranges

This patch changes the AMD IOMMU address allocator to allow up to 32
aperture ranges per dma_ops domain.

Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h
index 4c64c9bc68395c3f5dd5e8d545c84a13501380d4..eca912931a85ac1c7f1d09fe1eec22952b67f87f 100644 (file)
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@ -195,7 +195,12 @@
 #define PD_DEFAULT_MASK                (1UL << 1) /* domain is a default dma_ops
                                              domain for an IOMMU */
 
-#define APERTURE_RANGE_SIZE    (128 * 1024 * 1024)
+#define APERTURE_RANGE_SHIFT   27      /* 128 MB */
+#define APERTURE_RANGE_SIZE    (1ULL << APERTURE_RANGE_SHIFT)
+#define APERTURE_RANGE_PAGES   (APERTURE_RANGE_SIZE >> PAGE_SHIFT)
+#define APERTURE_MAX_RANGES    32      /* allows 4GB of DMA address space */
+#define APERTURE_RANGE_INDEX(a)        ((a) >> APERTURE_RANGE_SHIFT)
+#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL)
 
 /*
  * This structure contains generic data for  IOMMU protection domains
@@ -227,6 +232,8 @@ struct aperture_range {
         * just calculate its address in constant time.
         */
        u64 *pte_pages[64];
+
+       unsigned long offset;
 };
 
 /*
@@ -245,7 +252,7 @@ struct dma_ops_domain {
        unsigned long next_bit;
 
        /* address space relevant data */
-       struct aperture_range aperture;
+       struct aperture_range *aperture[APERTURE_MAX_RANGES];
 
        /* This will be set to true when TLB needs to be flushed */
        bool need_flush;

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index a467addb44b7b93d14fb556649042b5c47f46a1e..794163ae97b49e2c93a7e2f785bf2ebf7e8fa306 100644 (file)
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -578,7 +578,7 @@ static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
                 */
                if (addr < dma_dom->aperture_size)
                        __set_bit(addr >> PAGE_SHIFT,
-                                 dma_dom->aperture.bitmap);
+                                 dma_dom->aperture[0]->bitmap);
        }
 
        return 0;
@@ -615,43 +615,74 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  ****************************************************************************/
 
 /*
- * The address allocator core function.
+ * The address allocator core functions.
  *
  * called with domain->lock held
  */
+
+static unsigned long dma_ops_area_alloc(struct device *dev,
+                                       struct dma_ops_domain *dom,
+                                       unsigned int pages,
+                                       unsigned long align_mask,
+                                       u64 dma_mask,
+                                       unsigned long start)
+{
+       unsigned long next_bit = dom->next_bit % APERTURE_RANGE_PAGES;
+       int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
+       int i = start >> APERTURE_RANGE_SHIFT;
+       unsigned long boundary_size;
+       unsigned long address = -1;
+       unsigned long limit;
+
+       boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
+                       PAGE_SIZE) >> PAGE_SHIFT;
+
+       for (;i < max_index; ++i) {
+               unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
+
+               if (dom->aperture[i]->offset >= dma_mask)
+                       break;
+
+               limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
+                                              dma_mask >> PAGE_SHIFT);
+
+               address = iommu_area_alloc(dom->aperture[i]->bitmap,
+                                          limit, next_bit, pages, 0,
+                                           boundary_size, align_mask);
+               if (address != -1) {
+                       address = dom->aperture[i]->offset +
+                                 (address << PAGE_SHIFT);
+                       dom->next_bit = (address >> PAGE_SHIFT) + pages;
+                       break;
+               }
+
+               next_bit = 0;
+       }
+
+       return address;
+}
+
 static unsigned long dma_ops_alloc_addresses(struct device *dev,
                                             struct dma_ops_domain *dom,
                                             unsigned int pages,
                                             unsigned long align_mask,
                                             u64 dma_mask)
 {
-       unsigned long limit;
        unsigned long address;
-       unsigned long boundary_size;
+       unsigned long start = dom->next_bit << PAGE_SHIFT;
 
-       boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
-                       PAGE_SIZE) >> PAGE_SHIFT;
-       limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
-                                      dma_mask >> PAGE_SHIFT);
 
-       if (dom->next_bit >= limit) {
-               dom->next_bit = 0;
-               dom->need_flush = true;
-       }
+       address = dma_ops_area_alloc(dev, dom, pages, align_mask,
+                                    dma_mask, start);
 
-       address = iommu_area_alloc(dom->aperture.bitmap, limit, dom->next_bit,
-                                  pages, 0 , boundary_size, align_mask);
        if (address == -1) {
-               address = iommu_area_alloc(dom->aperture.bitmap, limit, 0,
-                                          pages, 0, boundary_size,
-                                          align_mask);
+               dom->next_bit = 0;
+               address = dma_ops_area_alloc(dev, dom, pages, align_mask,
+                                            dma_mask, 0);
                dom->need_flush = true;
        }
 
-       if (likely(address != -1)) {
-               dom->next_bit = address + pages;
-               address <<= PAGE_SHIFT;
-       } else
+       if (unlikely(address == -1))
                address = bad_dma_address;
 
        WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
@@ -668,11 +699,17 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
                                   unsigned long address,
                                   unsigned int pages)
 {
-       address >>= PAGE_SHIFT;
-       iommu_area_free(dom->aperture.bitmap, address, pages);
+       unsigned i = address >> APERTURE_RANGE_SHIFT;
+       struct aperture_range *range = dom->aperture[i];
+
+       BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
 
-       if (address >= dom->next_bit)
+       if ((address >> PAGE_SHIFT) >= dom->next_bit)
                dom->need_flush = true;
+
+       address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
+       iommu_area_free(range->bitmap, address, pages);
+
 }
 
 /****************************************************************************
@@ -720,12 +757,16 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
                                      unsigned long start_page,
                                      unsigned int pages)
 {
-       unsigned int last_page = dom->aperture_size >> PAGE_SHIFT;
+       unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
 
        if (start_page + pages > last_page)
                pages = last_page - start_page;
 
-       iommu_area_reserve(dom->aperture.bitmap, start_page, pages);
+       for (i = start_page; i < start_page + pages; ++i) {
+               int index = i / APERTURE_RANGE_PAGES;
+               int page  = i % APERTURE_RANGE_PAGES;
+               __set_bit(page, dom->aperture[index]->bitmap);
+       }
 }
 
 static void free_pagetable(struct protection_domain *domain)
@@ -764,12 +805,19 @@ static void free_pagetable(struct protection_domain *domain)
  */
 static void dma_ops_domain_free(struct dma_ops_domain *dom)
 {
+       int i;
+
        if (!dom)
                return;
 
        free_pagetable(&dom->domain);
 
-       free_page((unsigned long)dom->aperture.bitmap);
+       for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
+               if (!dom->aperture[i])
+                       continue;
+               free_page((unsigned long)dom->aperture[i]->bitmap);
+               kfree(dom->aperture[i]);
+       }
 
        kfree(dom);
 }
@@ -797,6 +845,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
        if (!dma_dom)
                return NULL;
 
+       dma_dom->aperture[0] = kzalloc(sizeof(struct aperture_range),
+                                      GFP_KERNEL);
+       if (!dma_dom->aperture[0])
+               goto free_dma_dom;
+
        spin_lock_init(&dma_dom->domain.lock);
 
        dma_dom->domain.id = domain_id_alloc();
@@ -809,14 +862,14 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
        if (!dma_dom->domain.pt_root)
                goto free_dma_dom;
        dma_dom->aperture_size = APERTURE_RANGE_SIZE;
-       dma_dom->aperture.bitmap = (void *)get_zeroed_page(GFP_KERNEL);
-       if (!dma_dom->aperture.bitmap)
+       dma_dom->aperture[0]->bitmap = (void *)get_zeroed_page(GFP_KERNEL);
+       if (!dma_dom->aperture[0]->bitmap)
                goto free_dma_dom;
        /*
         * mark the first page as allocated so we never return 0 as
         * a valid dma-address. So we can use 0 as error value
         */
-       dma_dom->aperture.bitmap[0] = 1;
+       dma_dom->aperture[0]->bitmap[0] = 1;
        dma_dom->next_bit = 0;
 
        dma_dom->need_flush = false;
@@ -846,7 +899,7 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
        dma_dom->domain.pt_root[0] = IOMMU_L2_PDE(virt_to_phys(l2_pde));
 
        for (i = 0; i < num_pte_pages; ++i) {
-               u64 **pte_page = &dma_dom->aperture.pte_pages[i];
+               u64 **pte_page = &dma_dom->aperture[0]->pte_pages[i];
                *pte_page = (u64 *)get_zeroed_page(GFP_KERNEL);
                if (!*pte_page)
                        goto free_dma_dom;
@@ -1164,14 +1217,19 @@ static u64* alloc_pte(struct protection_domain *dom,
 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
                            unsigned long address)
 {
-       struct aperture_range *aperture = &dom->aperture;
+       struct aperture_range *aperture;
        u64 *pte, *pte_page;
 
-       pte = aperture->pte_pages[IOMMU_PTE_L1_INDEX(address)];
+       aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
+       if (!aperture)
+               return NULL;
+
+       pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
        if (!pte) {
                pte = alloc_pte(&dom->domain, address, &pte_page, GFP_ATOMIC);
-               aperture->pte_pages[IOMMU_PTE_L1_INDEX(address)] = pte_page;
-       }
+               aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
+       } else
+               pte += IOMMU_PTE_L0_INDEX(address);
 
        return pte;
 }
@@ -1219,14 +1277,20 @@ static void dma_ops_domain_unmap(struct amd_iommu *iommu,
                                 struct dma_ops_domain *dom,
                                 unsigned long address)
 {
+       struct aperture_range *aperture;
        u64 *pte;
 
        if (address >= dom->aperture_size)
                return;
 
-       WARN_ON(address & ~PAGE_MASK || address >= dom->aperture_size);
+       aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
+       if (!aperture)
+               return;
+
+       pte  = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
+       if (!pte)
+               return;
 
-       pte  = dom->aperture.pte_pages[IOMMU_PTE_L1_INDEX(address)];
        pte += IOMMU_PTE_L0_INDEX(address);
 
        WARN_ON(!*pte);