2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 #include <linux/init.h>
23 #include <linux/bitmap.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/sysdev.h>
28 #include <linux/spinlock.h>
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/dma-mapping.h>
32 #include <linux/mempool.h>
34 #include "intel-iommu.h"
35 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
36 #include <asm/cacheflush.h>
37 #include <asm/iommu.h>
40 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
41 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
43 #define IOAPIC_RANGE_START (0xfee00000)
44 #define IOAPIC_RANGE_END (0xfeefffff)
45 #define IOVA_START_ADDR (0x1000)
47 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
49 #define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
51 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
53 static void domain_remove_dev_info(struct dmar_domain *domain);
55 static int dmar_disabled;
56 static int __initdata dmar_map_gfx = 1;
57 static int dmar_forcedac;
59 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
60 static DEFINE_SPINLOCK(device_domain_lock);
61 static LIST_HEAD(device_domain_list);
63 static int __init intel_iommu_setup(char *str)
68 if (!strncmp(str, "off", 3)) {
70 printk(KERN_INFO"Intel-IOMMU: disabled\n");
71 } else if (!strncmp(str, "igfx_off", 8)) {
74 "Intel-IOMMU: disable GFX device mapping\n");
75 } else if (!strncmp(str, "forcedac", 8)) {
77 "Intel-IOMMU: Forcing DAC for PCI devices\n");
81 str += strcspn(str, ",");
87 __setup("intel_iommu=", intel_iommu_setup);
89 static struct kmem_cache *iommu_domain_cache;
90 static struct kmem_cache *iommu_devinfo_cache;
91 static struct kmem_cache *iommu_iova_cache;
93 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
98 /* trying to avoid low memory issues */
99 flags = current->flags & PF_MEMALLOC;
100 current->flags |= PF_MEMALLOC;
101 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
102 current->flags &= (~PF_MEMALLOC | flags);
107 static inline void *alloc_pgtable_page(void)
112 /* trying to avoid low memory issues */
113 flags = current->flags & PF_MEMALLOC;
114 current->flags |= PF_MEMALLOC;
115 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
116 current->flags &= (~PF_MEMALLOC | flags);
120 static inline void free_pgtable_page(void *vaddr)
122 free_page((unsigned long)vaddr);
125 static inline void *alloc_domain_mem(void)
127 return iommu_kmem_cache_alloc(iommu_domain_cache);
130 static inline void free_domain_mem(void *vaddr)
132 kmem_cache_free(iommu_domain_cache, vaddr);
135 static inline void * alloc_devinfo_mem(void)
137 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
140 static inline void free_devinfo_mem(void *vaddr)
142 kmem_cache_free(iommu_devinfo_cache, vaddr);
145 struct iova *alloc_iova_mem(void)
147 return iommu_kmem_cache_alloc(iommu_iova_cache);
150 void free_iova_mem(struct iova *iova)
152 kmem_cache_free(iommu_iova_cache, iova);
155 static inline void __iommu_flush_cache(
156 struct intel_iommu *iommu, void *addr, int size)
158 if (!ecap_coherent(iommu->ecap))
159 clflush_cache_range(addr, size);
162 /* Gets context entry for a given bus and devfn */
163 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
166 struct root_entry *root;
167 struct context_entry *context;
168 unsigned long phy_addr;
171 spin_lock_irqsave(&iommu->lock, flags);
172 root = &iommu->root_entry[bus];
173 context = get_context_addr_from_root(root);
175 context = (struct context_entry *)alloc_pgtable_page();
177 spin_unlock_irqrestore(&iommu->lock, flags);
180 __iommu_flush_cache(iommu, (void *)context, PAGE_SIZE_4K);
181 phy_addr = virt_to_phys((void *)context);
182 set_root_value(root, phy_addr);
183 set_root_present(root);
184 __iommu_flush_cache(iommu, root, sizeof(*root));
186 spin_unlock_irqrestore(&iommu->lock, flags);
187 return &context[devfn];
190 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
192 struct root_entry *root;
193 struct context_entry *context;
197 spin_lock_irqsave(&iommu->lock, flags);
198 root = &iommu->root_entry[bus];
199 context = get_context_addr_from_root(root);
204 ret = context_present(context[devfn]);
206 spin_unlock_irqrestore(&iommu->lock, flags);
210 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
212 struct root_entry *root;
213 struct context_entry *context;
216 spin_lock_irqsave(&iommu->lock, flags);
217 root = &iommu->root_entry[bus];
218 context = get_context_addr_from_root(root);
220 context_clear_entry(context[devfn]);
221 __iommu_flush_cache(iommu, &context[devfn], \
224 spin_unlock_irqrestore(&iommu->lock, flags);
227 static void free_context_table(struct intel_iommu *iommu)
229 struct root_entry *root;
232 struct context_entry *context;
234 spin_lock_irqsave(&iommu->lock, flags);
235 if (!iommu->root_entry) {
238 for (i = 0; i < ROOT_ENTRY_NR; i++) {
239 root = &iommu->root_entry[i];
240 context = get_context_addr_from_root(root);
242 free_pgtable_page(context);
244 free_pgtable_page(iommu->root_entry);
245 iommu->root_entry = NULL;
247 spin_unlock_irqrestore(&iommu->lock, flags);
250 /* page table handling */
251 #define LEVEL_STRIDE (9)
252 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
254 static inline int agaw_to_level(int agaw)
259 static inline int agaw_to_width(int agaw)
261 return 30 + agaw * LEVEL_STRIDE;
265 static inline int width_to_agaw(int width)
267 return (width - 30) / LEVEL_STRIDE;
270 static inline unsigned int level_to_offset_bits(int level)
272 return (12 + (level - 1) * LEVEL_STRIDE);
275 static inline int address_level_offset(u64 addr, int level)
277 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
280 static inline u64 level_mask(int level)
282 return ((u64)-1 << level_to_offset_bits(level));
285 static inline u64 level_size(int level)
287 return ((u64)1 << level_to_offset_bits(level));
290 static inline u64 align_to_level(u64 addr, int level)
292 return ((addr + level_size(level) - 1) & level_mask(level));
295 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
297 int addr_width = agaw_to_width(domain->agaw);
298 struct dma_pte *parent, *pte = NULL;
299 int level = agaw_to_level(domain->agaw);
303 BUG_ON(!domain->pgd);
305 addr &= (((u64)1) << addr_width) - 1;
306 parent = domain->pgd;
308 spin_lock_irqsave(&domain->mapping_lock, flags);
312 offset = address_level_offset(addr, level);
313 pte = &parent[offset];
317 if (!dma_pte_present(*pte)) {
318 tmp_page = alloc_pgtable_page();
321 spin_unlock_irqrestore(&domain->mapping_lock,
325 __iommu_flush_cache(domain->iommu, tmp_page,
327 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
329 * high level table always sets r/w, last level page
330 * table control read/write
332 dma_set_pte_readable(*pte);
333 dma_set_pte_writable(*pte);
334 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
336 parent = phys_to_virt(dma_pte_addr(*pte));
340 spin_unlock_irqrestore(&domain->mapping_lock, flags);
344 /* return address's pte at specific level */
345 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
348 struct dma_pte *parent, *pte = NULL;
349 int total = agaw_to_level(domain->agaw);
352 parent = domain->pgd;
353 while (level <= total) {
354 offset = address_level_offset(addr, total);
355 pte = &parent[offset];
359 if (!dma_pte_present(*pte))
361 parent = phys_to_virt(dma_pte_addr(*pte));
367 /* clear one page's page table */
368 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
370 struct dma_pte *pte = NULL;
372 /* get last level pte */
373 pte = dma_addr_level_pte(domain, addr, 1);
377 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
381 /* clear last level pte, a tlb flush should be followed */
382 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
384 int addr_width = agaw_to_width(domain->agaw);
386 start &= (((u64)1) << addr_width) - 1;
387 end &= (((u64)1) << addr_width) - 1;
388 /* in case it's partial page */
389 start = PAGE_ALIGN_4K(start);
392 /* we don't need lock here, nobody else touches the iova range */
393 while (start < end) {
394 dma_pte_clear_one(domain, start);
395 start += PAGE_SIZE_4K;
399 /* free page table pages. last level pte should already be cleared */
400 static void dma_pte_free_pagetable(struct dmar_domain *domain,
403 int addr_width = agaw_to_width(domain->agaw);
405 int total = agaw_to_level(domain->agaw);
409 start &= (((u64)1) << addr_width) - 1;
410 end &= (((u64)1) << addr_width) - 1;
412 /* we don't need lock here, nobody else touches the iova range */
414 while (level <= total) {
415 tmp = align_to_level(start, level);
416 if (tmp >= end || (tmp + level_size(level) > end))
420 pte = dma_addr_level_pte(domain, tmp, level);
423 phys_to_virt(dma_pte_addr(*pte)));
425 __iommu_flush_cache(domain->iommu,
428 tmp += level_size(level);
433 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
434 free_pgtable_page(domain->pgd);
440 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
442 struct root_entry *root;
445 root = (struct root_entry *)alloc_pgtable_page();
449 __iommu_flush_cache(iommu, root, PAGE_SIZE_4K);
451 spin_lock_irqsave(&iommu->lock, flags);
452 iommu->root_entry = root;
453 spin_unlock_irqrestore(&iommu->lock, flags);
458 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
460 unsigned long start_time = jiffies;\
462 sts = op (iommu->reg + offset);\
465 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
466 panic("DMAR hardware is malfunctioning\n");\
471 static void iommu_set_root_entry(struct intel_iommu *iommu)
477 addr = iommu->root_entry;
479 spin_lock_irqsave(&iommu->register_lock, flag);
480 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
482 cmd = iommu->gcmd | DMA_GCMD_SRTP;
483 writel(cmd, iommu->reg + DMAR_GCMD_REG);
485 /* Make sure hardware complete it */
486 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
487 readl, (sts & DMA_GSTS_RTPS), sts);
489 spin_unlock_irqrestore(&iommu->register_lock, flag);
492 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
497 if (!cap_rwbf(iommu->cap))
499 val = iommu->gcmd | DMA_GCMD_WBF;
501 spin_lock_irqsave(&iommu->register_lock, flag);
502 writel(val, iommu->reg + DMAR_GCMD_REG);
504 /* Make sure hardware complete it */
505 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
506 readl, (!(val & DMA_GSTS_WBFS)), val);
508 spin_unlock_irqrestore(&iommu->register_lock, flag);
511 /* return value determine if we need a write buffer flush */
512 static int __iommu_flush_context(struct intel_iommu *iommu,
513 u16 did, u16 source_id, u8 function_mask, u64 type,
514 int non_present_entry_flush)
520 * In the non-present entry flush case, if hardware doesn't cache
521 * non-present entry we do nothing and if hardware cache non-present
522 * entry, we flush entries of domain 0 (the domain id is used to cache
523 * any non-present entries)
525 if (non_present_entry_flush) {
526 if (!cap_caching_mode(iommu->cap))
533 case DMA_CCMD_GLOBAL_INVL:
534 val = DMA_CCMD_GLOBAL_INVL;
536 case DMA_CCMD_DOMAIN_INVL:
537 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
539 case DMA_CCMD_DEVICE_INVL:
540 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
541 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
548 spin_lock_irqsave(&iommu->register_lock, flag);
549 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
551 /* Make sure hardware complete it */
552 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
553 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
555 spin_unlock_irqrestore(&iommu->register_lock, flag);
557 /* flush context entry will implictly flush write buffer */
561 static int inline iommu_flush_context_global(struct intel_iommu *iommu,
562 int non_present_entry_flush)
564 return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
565 non_present_entry_flush);
568 static int inline iommu_flush_context_domain(struct intel_iommu *iommu, u16 did,
569 int non_present_entry_flush)
571 return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
572 non_present_entry_flush);
575 static int inline iommu_flush_context_device(struct intel_iommu *iommu,
576 u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
578 return __iommu_flush_context(iommu, did, source_id, function_mask,
579 DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
582 /* return value determine if we need a write buffer flush */
583 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
584 u64 addr, unsigned int size_order, u64 type,
585 int non_present_entry_flush)
587 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
588 u64 val = 0, val_iva = 0;
592 * In the non-present entry flush case, if hardware doesn't cache
593 * non-present entry we do nothing and if hardware cache non-present
594 * entry, we flush entries of domain 0 (the domain id is used to cache
595 * any non-present entries)
597 if (non_present_entry_flush) {
598 if (!cap_caching_mode(iommu->cap))
605 case DMA_TLB_GLOBAL_FLUSH:
606 /* global flush doesn't need set IVA_REG */
607 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
609 case DMA_TLB_DSI_FLUSH:
610 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
612 case DMA_TLB_PSI_FLUSH:
613 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
614 /* Note: always flush non-leaf currently */
615 val_iva = size_order | addr;
620 /* Note: set drain read/write */
623 * This is probably to be super secure.. Looks like we can
624 * ignore it without any impact.
626 if (cap_read_drain(iommu->cap))
627 val |= DMA_TLB_READ_DRAIN;
629 if (cap_write_drain(iommu->cap))
630 val |= DMA_TLB_WRITE_DRAIN;
632 spin_lock_irqsave(&iommu->register_lock, flag);
633 /* Note: Only uses first TLB reg currently */
635 dmar_writeq(iommu->reg + tlb_offset, val_iva);
636 dmar_writeq(iommu->reg + tlb_offset + 8, val);
638 /* Make sure hardware complete it */
639 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
640 dmar_readq, (!(val & DMA_TLB_IVT)), val);
642 spin_unlock_irqrestore(&iommu->register_lock, flag);
644 /* check IOTLB invalidation granularity */
645 if (DMA_TLB_IAIG(val) == 0)
646 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
647 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
648 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
649 DMA_TLB_IIRG(type), DMA_TLB_IAIG(val));
650 /* flush context entry will implictly flush write buffer */
654 static int inline iommu_flush_iotlb_global(struct intel_iommu *iommu,
655 int non_present_entry_flush)
657 return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
658 non_present_entry_flush);
661 static int inline iommu_flush_iotlb_dsi(struct intel_iommu *iommu, u16 did,
662 int non_present_entry_flush)
664 return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
665 non_present_entry_flush);
668 static int iommu_get_alignment(u64 base, unsigned int size)
673 end = base + size - 1;
674 while (base != end) {
682 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
683 u64 addr, unsigned int pages, int non_present_entry_flush)
687 BUG_ON(addr & (~PAGE_MASK_4K));
690 /* Fallback to domain selective flush if no PSI support */
691 if (!cap_pgsel_inv(iommu->cap))
692 return iommu_flush_iotlb_dsi(iommu, did,
693 non_present_entry_flush);
696 * PSI requires page size to be 2 ^ x, and the base address is naturally
697 * aligned to the size
699 align = iommu_get_alignment(addr >> PAGE_SHIFT_4K, pages);
700 /* Fallback to domain selective flush if size is too big */
701 if (align > cap_max_amask_val(iommu->cap))
702 return iommu_flush_iotlb_dsi(iommu, did,
703 non_present_entry_flush);
705 addr >>= PAGE_SHIFT_4K + align;
706 addr <<= PAGE_SHIFT_4K + align;
708 return __iommu_flush_iotlb(iommu, did, addr, align,
709 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
712 static int iommu_enable_translation(struct intel_iommu *iommu)
717 spin_lock_irqsave(&iommu->register_lock, flags);
718 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
720 /* Make sure hardware complete it */
721 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
722 readl, (sts & DMA_GSTS_TES), sts);
724 iommu->gcmd |= DMA_GCMD_TE;
725 spin_unlock_irqrestore(&iommu->register_lock, flags);
729 static int iommu_disable_translation(struct intel_iommu *iommu)
734 spin_lock_irqsave(&iommu->register_lock, flag);
735 iommu->gcmd &= ~DMA_GCMD_TE;
736 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
738 /* Make sure hardware complete it */
739 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
740 readl, (!(sts & DMA_GSTS_TES)), sts);
742 spin_unlock_irqrestore(&iommu->register_lock, flag);
746 static int iommu_init_domains(struct intel_iommu *iommu)
748 unsigned long ndomains;
749 unsigned long nlongs;
751 ndomains = cap_ndoms(iommu->cap);
752 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
753 nlongs = BITS_TO_LONGS(ndomains);
755 /* TBD: there might be 64K domains,
756 * consider other allocation for future chip
758 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
759 if (!iommu->domain_ids) {
760 printk(KERN_ERR "Allocating domain id array failed\n");
763 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
765 if (!iommu->domains) {
766 printk(KERN_ERR "Allocating domain array failed\n");
767 kfree(iommu->domain_ids);
772 * if Caching mode is set, then invalid translations are tagged
773 * with domainid 0. Hence we need to pre-allocate it.
775 if (cap_caching_mode(iommu->cap))
776 set_bit(0, iommu->domain_ids);
780 static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
782 struct intel_iommu *iommu;
787 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
790 iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
792 printk(KERN_ERR "IOMMU: can't map the region\n");
795 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
796 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
798 /* the registers might be more than one page */
799 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
800 cap_max_fault_reg_offset(iommu->cap));
801 map_size = PAGE_ALIGN_4K(map_size);
802 if (map_size > PAGE_SIZE_4K) {
804 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
806 printk(KERN_ERR "IOMMU: can't map the region\n");
811 ver = readl(iommu->reg + DMAR_VER_REG);
812 pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
813 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
814 iommu->cap, iommu->ecap);
815 ret = iommu_init_domains(iommu);
818 spin_lock_init(&iommu->lock);
819 spin_lock_init(&iommu->register_lock);
831 static void domain_exit(struct dmar_domain *domain);
832 static void free_iommu(struct intel_iommu *iommu)
834 struct dmar_domain *domain;
840 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
841 for (; i < cap_ndoms(iommu->cap); ) {
842 domain = iommu->domains[i];
843 clear_bit(i, iommu->domain_ids);
845 i = find_next_bit(iommu->domain_ids,
846 cap_ndoms(iommu->cap), i+1);
849 if (iommu->gcmd & DMA_GCMD_TE)
850 iommu_disable_translation(iommu);
853 set_irq_data(iommu->irq, NULL);
854 /* This will mask the irq */
855 free_irq(iommu->irq, iommu);
856 destroy_irq(iommu->irq);
859 kfree(iommu->domains);
860 kfree(iommu->domain_ids);
862 /* free context mapping */
863 free_context_table(iommu);
870 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
873 unsigned long ndomains;
874 struct dmar_domain *domain;
877 domain = alloc_domain_mem();
881 ndomains = cap_ndoms(iommu->cap);
883 spin_lock_irqsave(&iommu->lock, flags);
884 num = find_first_zero_bit(iommu->domain_ids, ndomains);
885 if (num >= ndomains) {
886 spin_unlock_irqrestore(&iommu->lock, flags);
887 free_domain_mem(domain);
888 printk(KERN_ERR "IOMMU: no free domain ids\n");
892 set_bit(num, iommu->domain_ids);
894 domain->iommu = iommu;
895 iommu->domains[num] = domain;
896 spin_unlock_irqrestore(&iommu->lock, flags);
901 static void iommu_free_domain(struct dmar_domain *domain)
905 spin_lock_irqsave(&domain->iommu->lock, flags);
906 clear_bit(domain->id, domain->iommu->domain_ids);
907 spin_unlock_irqrestore(&domain->iommu->lock, flags);
910 static struct iova_domain reserved_iova_list;
912 static void dmar_init_reserved_ranges(void)
914 struct pci_dev *pdev = NULL;
919 init_iova_domain(&reserved_iova_list);
921 /* IOAPIC ranges shouldn't be accessed by DMA */
922 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
923 IOVA_PFN(IOAPIC_RANGE_END));
925 printk(KERN_ERR "Reserve IOAPIC range failed\n");
927 /* Reserve all PCI MMIO to avoid peer-to-peer access */
928 for_each_pci_dev(pdev) {
931 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
932 r = &pdev->resource[i];
933 if (!r->flags || !(r->flags & IORESOURCE_MEM))
936 addr &= PAGE_MASK_4K;
937 size = r->end - addr;
938 size = PAGE_ALIGN_4K(size);
939 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
940 IOVA_PFN(size + addr) - 1);
942 printk(KERN_ERR "Reserve iova failed\n");
948 static void domain_reserve_special_ranges(struct dmar_domain *domain)
950 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
953 static inline int guestwidth_to_adjustwidth(int gaw)
956 int r = (gaw - 12) % 9;
967 static int domain_init(struct dmar_domain *domain, int guest_width)
969 struct intel_iommu *iommu;
970 int adjust_width, agaw;
973 init_iova_domain(&domain->iovad);
974 spin_lock_init(&domain->mapping_lock);
976 domain_reserve_special_ranges(domain);
979 iommu = domain->iommu;
980 if (guest_width > cap_mgaw(iommu->cap))
981 guest_width = cap_mgaw(iommu->cap);
982 domain->gaw = guest_width;
983 adjust_width = guestwidth_to_adjustwidth(guest_width);
984 agaw = width_to_agaw(adjust_width);
985 sagaw = cap_sagaw(iommu->cap);
986 if (!test_bit(agaw, &sagaw)) {
987 /* hardware doesn't support it, choose a bigger one */
988 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
989 agaw = find_next_bit(&sagaw, 5, agaw);
994 INIT_LIST_HEAD(&domain->devices);
996 /* always allocate the top pgd */
997 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1000 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE_4K);
1004 static void domain_exit(struct dmar_domain *domain)
1008 /* Domain 0 is reserved, so dont process it */
1012 domain_remove_dev_info(domain);
1014 put_iova_domain(&domain->iovad);
1015 end = DOMAIN_MAX_ADDR(domain->gaw);
1016 end = end & (~PAGE_MASK_4K);
1019 dma_pte_clear_range(domain, 0, end);
1021 /* free page tables */
1022 dma_pte_free_pagetable(domain, 0, end);
1024 iommu_free_domain(domain);
1025 free_domain_mem(domain);
1028 static int domain_context_mapping_one(struct dmar_domain *domain,
1031 struct context_entry *context;
1032 struct intel_iommu *iommu = domain->iommu;
1033 unsigned long flags;
1035 pr_debug("Set context mapping for %02x:%02x.%d\n",
1036 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1037 BUG_ON(!domain->pgd);
1038 context = device_to_context_entry(iommu, bus, devfn);
1041 spin_lock_irqsave(&iommu->lock, flags);
1042 if (context_present(*context)) {
1043 spin_unlock_irqrestore(&iommu->lock, flags);
1047 context_set_domain_id(*context, domain->id);
1048 context_set_address_width(*context, domain->agaw);
1049 context_set_address_root(*context, virt_to_phys(domain->pgd));
1050 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1051 context_set_fault_enable(*context);
1052 context_set_present(*context);
1053 __iommu_flush_cache(iommu, context, sizeof(*context));
1055 /* it's a non-present to present mapping */
1056 if (iommu_flush_context_device(iommu, domain->id,
1057 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, 1))
1058 iommu_flush_write_buffer(iommu);
1060 iommu_flush_iotlb_dsi(iommu, 0, 0);
1061 spin_unlock_irqrestore(&iommu->lock, flags);
1066 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1069 struct pci_dev *tmp, *parent;
1071 ret = domain_context_mapping_one(domain, pdev->bus->number,
1076 /* dependent device mapping */
1077 tmp = pci_find_upstream_pcie_bridge(pdev);
1080 /* Secondary interface's bus number and devfn 0 */
1081 parent = pdev->bus->self;
1082 while (parent != tmp) {
1083 ret = domain_context_mapping_one(domain, parent->bus->number,
1087 parent = parent->bus->self;
1089 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1090 return domain_context_mapping_one(domain,
1091 tmp->subordinate->number, 0);
1092 else /* this is a legacy PCI bridge */
1093 return domain_context_mapping_one(domain,
1094 tmp->bus->number, tmp->devfn);
1097 static int domain_context_mapped(struct dmar_domain *domain,
1098 struct pci_dev *pdev)
1101 struct pci_dev *tmp, *parent;
1103 ret = device_context_mapped(domain->iommu,
1104 pdev->bus->number, pdev->devfn);
1107 /* dependent device mapping */
1108 tmp = pci_find_upstream_pcie_bridge(pdev);
1111 /* Secondary interface's bus number and devfn 0 */
1112 parent = pdev->bus->self;
1113 while (parent != tmp) {
1114 ret = device_context_mapped(domain->iommu, parent->bus->number,
1118 parent = parent->bus->self;
1121 return device_context_mapped(domain->iommu,
1122 tmp->subordinate->number, 0);
1124 return device_context_mapped(domain->iommu,
1125 tmp->bus->number, tmp->devfn);
1129 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1130 u64 hpa, size_t size, int prot)
1132 u64 start_pfn, end_pfn;
1133 struct dma_pte *pte;
1136 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1138 iova &= PAGE_MASK_4K;
1139 start_pfn = ((u64)hpa) >> PAGE_SHIFT_4K;
1140 end_pfn = (PAGE_ALIGN_4K(((u64)hpa) + size)) >> PAGE_SHIFT_4K;
1142 while (start_pfn < end_pfn) {
1143 pte = addr_to_dma_pte(domain, iova + PAGE_SIZE_4K * index);
1146 /* We don't need lock here, nobody else
1147 * touches the iova range
1149 BUG_ON(dma_pte_addr(*pte));
1150 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1151 dma_set_pte_prot(*pte, prot);
1152 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1159 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1161 clear_context_table(domain->iommu, bus, devfn);
1162 iommu_flush_context_global(domain->iommu, 0);
1163 iommu_flush_iotlb_global(domain->iommu, 0);
1166 static void domain_remove_dev_info(struct dmar_domain *domain)
1168 struct device_domain_info *info;
1169 unsigned long flags;
1171 spin_lock_irqsave(&device_domain_lock, flags);
1172 while (!list_empty(&domain->devices)) {
1173 info = list_entry(domain->devices.next,
1174 struct device_domain_info, link);
1175 list_del(&info->link);
1176 list_del(&info->global);
1178 info->dev->sysdata = NULL;
1179 spin_unlock_irqrestore(&device_domain_lock, flags);
1181 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1182 free_devinfo_mem(info);
1184 spin_lock_irqsave(&device_domain_lock, flags);
1186 spin_unlock_irqrestore(&device_domain_lock, flags);
1191 * Note: we use struct pci_dev->sysdata stores the info
1193 struct dmar_domain *
1194 find_domain(struct pci_dev *pdev)
1196 struct device_domain_info *info;
1198 /* No lock here, assumes no domain exit in normal case */
1199 info = pdev->sysdata;
1201 return info->domain;
1205 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
1206 struct pci_dev *dev)
1211 for (index = 0; index < cnt; index ++)
1212 if (dev == devices[index])
1215 /* Check our parent */
1216 dev = dev->bus->self;
1222 static struct dmar_drhd_unit *
1223 dmar_find_matched_drhd_unit(struct pci_dev *dev)
1225 struct dmar_drhd_unit *drhd = NULL;
1227 list_for_each_entry(drhd, &dmar_drhd_units, list) {
1228 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
1229 drhd->devices_cnt, dev))
1236 /* domain is initialized */
1237 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1239 struct dmar_domain *domain, *found = NULL;
1240 struct intel_iommu *iommu;
1241 struct dmar_drhd_unit *drhd;
1242 struct device_domain_info *info, *tmp;
1243 struct pci_dev *dev_tmp;
1244 unsigned long flags;
1245 int bus = 0, devfn = 0;
1247 domain = find_domain(pdev);
1251 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1253 if (dev_tmp->is_pcie) {
1254 bus = dev_tmp->subordinate->number;
1257 bus = dev_tmp->bus->number;
1258 devfn = dev_tmp->devfn;
1260 spin_lock_irqsave(&device_domain_lock, flags);
1261 list_for_each_entry(info, &device_domain_list, global) {
1262 if (info->bus == bus && info->devfn == devfn) {
1263 found = info->domain;
1267 spin_unlock_irqrestore(&device_domain_lock, flags);
1268 /* pcie-pci bridge already has a domain, uses it */
1275 /* Allocate new domain for the device */
1276 drhd = dmar_find_matched_drhd_unit(pdev);
1278 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1282 iommu = drhd->iommu;
1284 domain = iommu_alloc_domain(iommu);
1288 if (domain_init(domain, gaw)) {
1289 domain_exit(domain);
1293 /* register pcie-to-pci device */
1295 info = alloc_devinfo_mem();
1297 domain_exit(domain);
1301 info->devfn = devfn;
1303 info->domain = domain;
1304 /* This domain is shared by devices under p2p bridge */
1305 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1307 /* pcie-to-pci bridge already has a domain, uses it */
1309 spin_lock_irqsave(&device_domain_lock, flags);
1310 list_for_each_entry(tmp, &device_domain_list, global) {
1311 if (tmp->bus == bus && tmp->devfn == devfn) {
1312 found = tmp->domain;
1317 free_devinfo_mem(info);
1318 domain_exit(domain);
1321 list_add(&info->link, &domain->devices);
1322 list_add(&info->global, &device_domain_list);
1324 spin_unlock_irqrestore(&device_domain_lock, flags);
1328 info = alloc_devinfo_mem();
1331 info->bus = pdev->bus->number;
1332 info->devfn = pdev->devfn;
1334 info->domain = domain;
1335 spin_lock_irqsave(&device_domain_lock, flags);
1336 /* somebody is fast */
1337 found = find_domain(pdev);
1338 if (found != NULL) {
1339 spin_unlock_irqrestore(&device_domain_lock, flags);
1340 if (found != domain) {
1341 domain_exit(domain);
1344 free_devinfo_mem(info);
1347 list_add(&info->link, &domain->devices);
1348 list_add(&info->global, &device_domain_list);
1349 pdev->sysdata = info;
1350 spin_unlock_irqrestore(&device_domain_lock, flags);
1353 /* recheck it here, maybe others set it */
1354 return find_domain(pdev);
1357 static int iommu_prepare_identity_map(struct pci_dev *pdev, u64 start, u64 end)
1359 struct dmar_domain *domain;
1365 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1366 pci_name(pdev), start, end);
1367 /* page table init */
1368 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1372 /* The address might not be aligned */
1373 base = start & PAGE_MASK_4K;
1375 size = PAGE_ALIGN_4K(size);
1376 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1377 IOVA_PFN(base + size) - 1)) {
1378 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1383 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1384 size, base, pci_name(pdev));
1386 * RMRR range might have overlap with physical memory range,
1389 dma_pte_clear_range(domain, base, base + size);
1391 ret = domain_page_mapping(domain, base, base, size,
1392 DMA_PTE_READ|DMA_PTE_WRITE);
1396 /* context entry init */
1397 ret = domain_context_mapping(domain, pdev);
1401 domain_exit(domain);
1406 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1407 struct pci_dev *pdev)
1409 if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1411 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1412 rmrr->end_address + 1);
1415 int __init init_dmars(void)
1417 struct dmar_drhd_unit *drhd;
1418 struct dmar_rmrr_unit *rmrr;
1419 struct pci_dev *pdev;
1420 struct intel_iommu *iommu;
1426 * initialize and program root entry to not present
1429 for_each_drhd_unit(drhd) {
1432 iommu = alloc_iommu(drhd);
1440 * we could share the same root & context tables
1441 * amoung all IOMMU's. Need to Split it later.
1443 ret = iommu_alloc_root_entry(iommu);
1445 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1452 * for each dev attached to rmrr
1454 * locate drhd for dev, alloc domain for dev
1455 * allocate free domain
1456 * allocate page table entries for rmrr
1457 * if context not allocated for bus
1458 * allocate and init context
1459 * set present in root table for this bus
1460 * init context with domain, translation etc
1464 for_each_rmrr_units(rmrr) {
1466 for (i = 0; i < rmrr->devices_cnt; i++) {
1467 pdev = rmrr->devices[i];
1468 /* some BIOS lists non-exist devices in DMAR table */
1471 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1474 "IOMMU: mapping reserved region failed\n");
1481 * global invalidate context cache
1482 * global invalidate iotlb
1483 * enable translation
1485 for_each_drhd_unit(drhd) {
1488 iommu = drhd->iommu;
1489 sprintf (iommu->name, "dmar%d", unit++);
1491 iommu_flush_write_buffer(iommu);
1493 iommu_set_root_entry(iommu);
1495 iommu_flush_context_global(iommu, 0);
1496 iommu_flush_iotlb_global(iommu, 0);
1498 ret = iommu_enable_translation(iommu);
1505 for_each_drhd_unit(drhd) {
1508 iommu = drhd->iommu;
1514 static inline u64 aligned_size(u64 host_addr, size_t size)
1517 addr = (host_addr & (~PAGE_MASK_4K)) + size;
1518 return PAGE_ALIGN_4K(addr);
1522 iommu_alloc_iova(struct dmar_domain *domain, void *host_addr, size_t size,
1528 /* Make sure it's in range */
1529 if ((start > DOMAIN_MAX_ADDR(domain->gaw)) || end < start)
1532 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1533 start_addr = PAGE_ALIGN_4K(start);
1534 size = aligned_size((u64)host_addr, size);
1535 if (!size || (start_addr + size > end))
1538 piova = alloc_iova(&domain->iovad,
1539 size >> PAGE_SHIFT_4K, IOVA_PFN(end));
1544 static dma_addr_t __intel_map_single(struct device *dev, void *addr,
1545 size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1547 struct dmar_domain *domain;
1548 struct pci_dev *pdev = to_pci_dev(dev);
1551 struct iova *iova = NULL;
1554 addr = (void *)virt_to_phys(addr);
1556 domain = get_domain_for_dev(pdev,
1557 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1560 "Allocating domain for %s failed", pci_name(pdev));
1564 start_addr = IOVA_START_ADDR;
1566 if ((pdev->dma_mask <= DMA_32BIT_MASK) || (dmar_forcedac)) {
1567 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1571 * First try to allocate an io virtual address in
1572 * DMA_32BIT_MASK and if that fails then try allocating
1575 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1578 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1583 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1587 /* make sure context mapping is ok */
1588 if (unlikely(!domain_context_mapped(domain, pdev))) {
1589 ret = domain_context_mapping(domain, pdev);
1595 * Check if DMAR supports zero-length reads on write only
1598 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1599 !cap_zlr(domain->iommu->cap))
1600 prot |= DMA_PTE_READ;
1601 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1602 prot |= DMA_PTE_WRITE;
1604 * addr - (addr + size) might be partial page, we should map the whole
1605 * page. Note: if two part of one page are separately mapped, we
1606 * might have two guest_addr mapping to the same host addr, but this
1607 * is not a big problem
1609 ret = domain_page_mapping(domain, iova->pfn_lo << PAGE_SHIFT_4K,
1610 ((u64)addr) & PAGE_MASK_4K,
1611 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K, prot);
1615 pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
1616 pci_name(pdev), size, (u64)addr,
1617 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1618 (u64)(iova->pfn_lo << PAGE_SHIFT_4K), dir);
1620 *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1621 *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1622 return (iova->pfn_lo << PAGE_SHIFT_4K) + ((u64)addr & (~PAGE_MASK_4K));
1624 __free_iova(&domain->iovad, iova);
1625 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1626 pci_name(pdev), size, (u64)addr, dir);
1630 static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
1631 size_t size, int dir)
1633 struct pci_dev *pdev = to_pci_dev(hwdev);
1635 struct dmar_domain *domain;
1637 unsigned int flush_size;
1639 BUG_ON(dir == DMA_NONE);
1640 if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1641 return virt_to_bus(addr);
1643 ret = __intel_map_single(hwdev, addr, size,
1644 dir, &flush_addr, &flush_size);
1646 domain = find_domain(pdev);
1647 /* it's a non-present to present mapping */
1648 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
1649 flush_addr, flush_size >> PAGE_SHIFT_4K, 1))
1650 iommu_flush_write_buffer(domain->iommu);
1655 static void __intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1656 size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1658 struct dmar_domain *domain;
1659 struct pci_dev *pdev = to_pci_dev(dev);
1662 domain = find_domain(pdev);
1665 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1670 pr_debug("Device %s unmapping: %lx@%llx\n",
1672 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1673 (u64)(iova->pfn_lo << PAGE_SHIFT_4K));
1675 *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1676 *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1677 /* clear the whole page, not just dev_addr - (dev_addr + size) */
1678 dma_pte_clear_range(domain, *flush_addr, *flush_addr + *flush_size);
1679 /* free page tables */
1680 dma_pte_free_pagetable(domain, *flush_addr, *flush_addr + *flush_size);
1682 __free_iova(&domain->iovad, iova);
1685 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1686 size_t size, int dir)
1688 struct pci_dev *pdev = to_pci_dev(dev);
1689 struct dmar_domain *domain;
1691 unsigned int flush_size;
1693 if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1696 domain = find_domain(pdev);
1697 __intel_unmap_single(dev, dev_addr, size,
1698 dir, &flush_addr, &flush_size);
1699 if (flush_size == 0)
1701 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, flush_addr,
1702 flush_size >> PAGE_SHIFT_4K, 0))
1703 iommu_flush_write_buffer(domain->iommu);
1706 static void * intel_alloc_coherent(struct device *hwdev, size_t size,
1707 dma_addr_t *dma_handle, gfp_t flags)
1712 size = PAGE_ALIGN_4K(size);
1713 order = get_order(size);
1714 flags &= ~(GFP_DMA | GFP_DMA32);
1716 vaddr = (void *)__get_free_pages(flags, order);
1719 memset(vaddr, 0, size);
1721 *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
1724 free_pages((unsigned long)vaddr, order);
1728 static void intel_free_coherent(struct device *hwdev, size_t size,
1729 void *vaddr, dma_addr_t dma_handle)
1733 size = PAGE_ALIGN_4K(size);
1734 order = get_order(size);
1736 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
1737 free_pages((unsigned long)vaddr, order);
1740 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sg,
1741 int nelems, int dir)
1744 struct pci_dev *pdev = to_pci_dev(hwdev);
1745 struct dmar_domain *domain;
1747 unsigned int flush_size;
1749 if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1752 domain = find_domain(pdev);
1753 for (i = 0; i < nelems; i++, sg++)
1754 __intel_unmap_single(hwdev, sg->dma_address,
1755 sg->dma_length, dir, &flush_addr, &flush_size);
1757 if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 0))
1758 iommu_flush_write_buffer(domain->iommu);
1761 #define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
1762 static int intel_nontranslate_map_sg(struct device *hddev,
1763 struct scatterlist *sg, int nelems, int dir)
1767 for (i = 0; i < nelems; i++) {
1768 struct scatterlist *s = &sg[i];
1770 s->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(s));
1771 s->dma_length = s->length;
1776 static int intel_map_sg(struct device *hwdev, struct scatterlist *sg,
1777 int nelems, int dir)
1781 dma_addr_t dma_handle;
1782 struct pci_dev *pdev = to_pci_dev(hwdev);
1783 struct dmar_domain *domain;
1785 unsigned int flush_size;
1787 BUG_ON(dir == DMA_NONE);
1788 if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1789 return intel_nontranslate_map_sg(hwdev, sg, nelems, dir);
1791 for (i = 0; i < nelems; i++, sg++) {
1792 addr = SG_ENT_VIRT_ADDRESS(sg);
1793 dma_handle = __intel_map_single(hwdev, addr,
1794 sg->length, dir, &flush_addr, &flush_size);
1796 intel_unmap_sg(hwdev, sg - i, i, dir);
1797 sg[0].dma_length = 0;
1800 sg->dma_address = dma_handle;
1801 sg->dma_length = sg->length;
1804 domain = find_domain(pdev);
1806 /* it's a non-present to present mapping */
1807 if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 1))
1808 iommu_flush_write_buffer(domain->iommu);
1812 static struct dma_mapping_ops intel_dma_ops = {
1813 .alloc_coherent = intel_alloc_coherent,
1814 .free_coherent = intel_free_coherent,
1815 .map_single = intel_map_single,
1816 .unmap_single = intel_unmap_single,
1817 .map_sg = intel_map_sg,
1818 .unmap_sg = intel_unmap_sg,
1821 static inline int iommu_domain_cache_init(void)
1825 iommu_domain_cache = kmem_cache_create("iommu_domain",
1826 sizeof(struct dmar_domain),
1831 if (!iommu_domain_cache) {
1832 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
1839 static inline int iommu_devinfo_cache_init(void)
1843 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
1844 sizeof(struct device_domain_info),
1849 if (!iommu_devinfo_cache) {
1850 printk(KERN_ERR "Couldn't create devinfo cache\n");
1857 static inline int iommu_iova_cache_init(void)
1861 iommu_iova_cache = kmem_cache_create("iommu_iova",
1862 sizeof(struct iova),
1867 if (!iommu_iova_cache) {
1868 printk(KERN_ERR "Couldn't create iova cache\n");
1875 static int __init iommu_init_mempool(void)
1878 ret = iommu_iova_cache_init();
1882 ret = iommu_domain_cache_init();
1886 ret = iommu_devinfo_cache_init();
1890 kmem_cache_destroy(iommu_domain_cache);
1892 kmem_cache_destroy(iommu_iova_cache);
1897 static void __init iommu_exit_mempool(void)
1899 kmem_cache_destroy(iommu_devinfo_cache);
1900 kmem_cache_destroy(iommu_domain_cache);
1901 kmem_cache_destroy(iommu_iova_cache);
1905 void __init detect_intel_iommu(void)
1907 if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
1909 if (early_dmar_detect()) {
1914 static void __init init_no_remapping_devices(void)
1916 struct dmar_drhd_unit *drhd;
1918 for_each_drhd_unit(drhd) {
1919 if (!drhd->include_all) {
1921 for (i = 0; i < drhd->devices_cnt; i++)
1922 if (drhd->devices[i] != NULL)
1924 /* ignore DMAR unit if no pci devices exist */
1925 if (i == drhd->devices_cnt)
1933 for_each_drhd_unit(drhd) {
1935 if (drhd->ignored || drhd->include_all)
1938 for (i = 0; i < drhd->devices_cnt; i++)
1939 if (drhd->devices[i] &&
1940 !IS_GFX_DEVICE(drhd->devices[i]))
1943 if (i < drhd->devices_cnt)
1946 /* bypass IOMMU if it is just for gfx devices */
1948 for (i = 0; i < drhd->devices_cnt; i++) {
1949 if (!drhd->devices[i])
1951 drhd->devices[i]->sysdata = DUMMY_DEVICE_DOMAIN_INFO;
1956 int __init intel_iommu_init(void)
1960 if (no_iommu || swiotlb || dmar_disabled)
1963 if (dmar_table_init())
1966 iommu_init_mempool();
1967 dmar_init_reserved_ranges();
1969 init_no_remapping_devices();
1973 printk(KERN_ERR "IOMMU: dmar init failed\n");
1974 put_iova_domain(&reserved_iova_list);
1975 iommu_exit_mempool();
1979 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
1982 dma_ops = &intel_dma_ops;