Merge branches 'core/iommu', 'x86/amd-iommu' and 'x86/iommu' into x86-v28-for-linus...

[mv-sheeva.git] / arch / x86 / kernel / amd_iommu.c

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index db64482b179639d1cfeeed6181903b4dbacc827c..34e4d112b1ef1bf54720a17f71756f8dd1394dfa 100644 (file)
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -201,10 +201,10 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  */
 static int iommu_completion_wait(struct amd_iommu *iommu)
 {
-       int ret, ready = 0;
+       int ret = 0, ready = 0;
        unsigned status = 0;
        struct iommu_cmd cmd;
-       unsigned long i = 0;
+       unsigned long flags, i = 0;
 
        memset(&cmd, 0, sizeof(cmd));
        cmd.data[0] = CMD_COMPL_WAIT_INT_MASK;
@@ -212,10 +212,12 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
 
        iommu->need_sync = 0;
 
-       ret = iommu_queue_command(iommu, &cmd);
+       spin_lock_irqsave(&iommu->lock, flags);
+
+       ret = __iommu_queue_command(iommu, &cmd);
 
        if (ret)
-               return ret;
+               goto out;
 
        while (!ready && (i < EXIT_LOOP_COUNT)) {
                ++i;
@@ -230,6 +232,8 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
 
        if (unlikely((i == EXIT_LOOP_COUNT) && printk_ratelimit()))
                printk(KERN_WARNING "AMD IOMMU: Completion wait loop failed\n");
+out:
+       spin_unlock_irqrestore(&iommu->lock, flags);
 
        return 0;
 }
@@ -240,6 +244,7 @@ static int iommu_completion_wait(struct amd_iommu *iommu)
 static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
 {
        struct iommu_cmd cmd;
+       int ret;
 
        BUG_ON(iommu == NULL);
 
@@ -247,9 +252,11 @@ static int iommu_queue_inv_dev_entry(struct amd_iommu *iommu, u16 devid)
        CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY);
        cmd.data[0] = devid;
 
+       ret = iommu_queue_command(iommu, &cmd);
+
        iommu->need_sync = 1;
 
-       return iommu_queue_command(iommu, &cmd);
+       return ret;
 }
 
 /*
@@ -259,6 +266,7 @@ static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
                u64 address, u16 domid, int pde, int s)
 {
        struct iommu_cmd cmd;
+       int ret;
 
        memset(&cmd, 0, sizeof(cmd));
        address &= PAGE_MASK;
@@ -271,9 +279,11 @@ static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu,
        if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
                cmd.data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
 
+       ret = iommu_queue_command(iommu, &cmd);
+
        iommu->need_sync = 1;
 
-       return iommu_queue_command(iommu, &cmd);
+       return ret;
 }
 
 /*
@@ -470,10 +480,6 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  * efficient allocator.
  *
  ****************************************************************************/
-static unsigned long dma_mask_to_pages(unsigned long mask)
-{
-       return PAGE_ALIGN(mask) >> PAGE_SHIFT;
-}
 
 /*
  * The address allocator core function.
@@ -483,16 +489,17 @@ static unsigned long dma_mask_to_pages(unsigned long mask)
 static unsigned long dma_ops_alloc_addresses(struct device *dev,
                                             struct dma_ops_domain *dom,
                                             unsigned int pages,
-                                            unsigned long align_mask)
+                                            unsigned long align_mask,
+                                            u64 dma_mask)
 {
-       unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
+       unsigned long limit;
        unsigned long address;
-       unsigned long size = dom->aperture_size >> PAGE_SHIFT;
        unsigned long boundary_size;
 
        boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
                        PAGE_SIZE) >> PAGE_SHIFT;
-       limit = limit < size ? limit : size;
+       limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
+                                      dma_mask >> PAGE_SHIFT);
 
        if (dom->next_bit >= limit) {
                dom->next_bit = 0;
@@ -571,7 +578,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
        if (start_page + pages > last_page)
                pages = last_page - start_page;
 
-       set_bit_string(dom->bitmap, start_page, pages);
+       iommu_area_reserve(dom->bitmap, start_page, pages);
 }
 
 static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
@@ -919,7 +926,8 @@ static dma_addr_t __map_single(struct device *dev,
                               phys_addr_t paddr,
                               size_t size,
                               int dir,
-                              bool align)
+                              bool align,
+                              u64 dma_mask)
 {
        dma_addr_t offset = paddr & ~PAGE_MASK;
        dma_addr_t address, start;
@@ -933,7 +941,8 @@ static dma_addr_t __map_single(struct device *dev,
        if (align)
                align_mask = (1UL << get_order(size)) - 1;
 
-       address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask);
+       address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
+                                         dma_mask);
        if (unlikely(address == bad_dma_address))
                goto out;
 
@@ -945,7 +954,7 @@ static dma_addr_t __map_single(struct device *dev,
        }
        address += offset;
 
-       if (unlikely(dma_dom->need_flush && !iommu_fullflush)) {
+       if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
                iommu_flush_tlb(iommu, dma_dom->domain.id);
                dma_dom->need_flush = false;
        } else if (unlikely(iommu_has_npcache(iommu)))
@@ -982,7 +991,7 @@ static void __unmap_single(struct amd_iommu *iommu,
 
        dma_ops_free_addresses(dma_dom, dma_addr, pages);
 
-       if (iommu_fullflush)
+       if (amd_iommu_unmap_flush)
                iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
 }
 
@@ -997,10 +1006,13 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
        struct protection_domain *domain;
        u16 devid;
        dma_addr_t addr;
+       u64 dma_mask;
 
        if (!check_device(dev))
                return bad_dma_address;
 
+       dma_mask = *dev->dma_mask;
+
        get_device_resources(dev, &iommu, &domain, &devid);
 
        if (iommu == NULL || domain == NULL)
@@ -1008,7 +1020,8 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
                return (dma_addr_t)paddr;
 
        spin_lock_irqsave(&domain->lock, flags);
-       addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false);
+       addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
+                           dma_mask);
        if (addr == bad_dma_address)
                goto out;
 
@@ -1080,10 +1093,13 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
        struct scatterlist *s;
        phys_addr_t paddr;
        int mapped_elems = 0;
+       u64 dma_mask;
 
        if (!check_device(dev))
                return 0;
 
+       dma_mask = *dev->dma_mask;
+
        get_device_resources(dev, &iommu, &domain, &devid);
 
        if (!iommu || !domain)
@@ -1095,7 +1111,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
                paddr = sg_phys(s);
 
                s->dma_address = __map_single(dev, iommu, domain->priv,
-                                             paddr, s->length, dir, false);
+                                             paddr, s->length, dir, false,
+                                             dma_mask);
 
                if (s->dma_address) {
                        s->dma_length = s->length;
@@ -1168,6 +1185,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
        struct protection_domain *domain;
        u16 devid;
        phys_addr_t paddr;
+       u64 dma_mask = dev->coherent_dma_mask;
 
        if (!check_device(dev))
                return NULL;
@@ -1187,10 +1205,13 @@ static void *alloc_coherent(struct device *dev, size_t size,
                return virt_addr;
        }
 
+       if (!dma_mask)
+               dma_mask = *dev->dma_mask;
+
        spin_lock_irqsave(&domain->lock, flags);
 
        *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
-                                size, DMA_BIDIRECTIONAL, true);
+                                size, DMA_BIDIRECTIONAL, true, dma_mask);
 
        if (*dma_addr == bad_dma_address) {
                free_pages((unsigned long)virt_addr, get_order(size));