2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/intel-iommu.h>
38 #include <asm/cacheflush.h>
39 #include <asm/iommu.h>
42 #define ROOT_SIZE VTD_PAGE_SIZE
43 #define CONTEXT_SIZE VTD_PAGE_SIZE
45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48 #define IOAPIC_RANGE_START (0xfee00000)
49 #define IOAPIC_RANGE_END (0xfeefffff)
50 #define IOVA_START_ADDR (0x1000)
52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
57 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
58 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
60 /* global iommu list, set NULL for ignored DMAR units */
61 static struct intel_iommu **g_iommus;
66 * 12-63: Context Ptr (12 - (haw-1))
73 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
74 static inline bool root_present(struct root_entry *root)
76 return (root->val & 1);
78 static inline void set_root_present(struct root_entry *root)
82 static inline void set_root_value(struct root_entry *root, unsigned long value)
84 root->val |= value & VTD_PAGE_MASK;
87 static inline struct context_entry *
88 get_context_addr_from_root(struct root_entry *root)
90 return (struct context_entry *)
91 (root_present(root)?phys_to_virt(
92 root->val & VTD_PAGE_MASK) :
99 * 1: fault processing disable
100 * 2-3: translation type
101 * 12-63: address space root
107 struct context_entry {
112 static inline bool context_present(struct context_entry *context)
114 return (context->lo & 1);
116 static inline void context_set_present(struct context_entry *context)
121 static inline void context_set_fault_enable(struct context_entry *context)
123 context->lo &= (((u64)-1) << 2) | 1;
126 #define CONTEXT_TT_MULTI_LEVEL 0
128 static inline void context_set_translation_type(struct context_entry *context,
131 context->lo &= (((u64)-1) << 4) | 3;
132 context->lo |= (value & 3) << 2;
135 static inline void context_set_address_root(struct context_entry *context,
138 context->lo |= value & VTD_PAGE_MASK;
141 static inline void context_set_address_width(struct context_entry *context,
144 context->hi |= value & 7;
147 static inline void context_set_domain_id(struct context_entry *context,
150 context->hi |= (value & ((1 << 16) - 1)) << 8;
153 static inline void context_clear_entry(struct context_entry *context)
165 * 12-63: Host physcial address
171 static inline void dma_clear_pte(struct dma_pte *pte)
176 static inline void dma_set_pte_readable(struct dma_pte *pte)
178 pte->val |= DMA_PTE_READ;
181 static inline void dma_set_pte_writable(struct dma_pte *pte)
183 pte->val |= DMA_PTE_WRITE;
186 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
188 pte->val = (pte->val & ~3) | (prot & 3);
191 static inline u64 dma_pte_addr(struct dma_pte *pte)
193 return (pte->val & VTD_PAGE_MASK);
196 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
198 pte->val |= (addr & VTD_PAGE_MASK);
201 static inline bool dma_pte_present(struct dma_pte *pte)
203 return (pte->val & 3) != 0;
206 /* devices under the same p2p bridge are owned in one domain */
207 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 < 0)
210 int id; /* domain id */
211 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
213 struct list_head devices; /* all devices' list */
214 struct iova_domain iovad; /* iova's that belong to this domain */
216 struct dma_pte *pgd; /* virtual address */
217 spinlock_t mapping_lock; /* page table lock */
218 int gaw; /* max guest address width */
220 /* adjusted guest address width, 0 is level 2 30-bit */
223 int flags; /* flags to find out type of domain */
226 /* PCI domain-device relationship */
227 struct device_domain_info {
228 struct list_head link; /* link to domain siblings */
229 struct list_head global; /* link to global list */
230 u8 bus; /* PCI bus numer */
231 u8 devfn; /* PCI devfn number */
232 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
233 struct dmar_domain *domain; /* pointer to domain */
236 static void flush_unmaps_timeout(unsigned long data);
238 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
240 #define HIGH_WATER_MARK 250
241 struct deferred_flush_tables {
243 struct iova *iova[HIGH_WATER_MARK];
244 struct dmar_domain *domain[HIGH_WATER_MARK];
247 static struct deferred_flush_tables *deferred_flush;
249 /* bitmap for indexing intel_iommus */
250 static int g_num_of_iommus;
252 static DEFINE_SPINLOCK(async_umap_flush_lock);
253 static LIST_HEAD(unmaps_to_do);
256 static long list_size;
258 static void domain_remove_dev_info(struct dmar_domain *domain);
261 static int __initdata dmar_map_gfx = 1;
262 static int dmar_forcedac;
263 static int intel_iommu_strict;
265 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
266 static DEFINE_SPINLOCK(device_domain_lock);
267 static LIST_HEAD(device_domain_list);
269 static int __init intel_iommu_setup(char *str)
274 if (!strncmp(str, "off", 3)) {
276 printk(KERN_INFO"Intel-IOMMU: disabled\n");
277 } else if (!strncmp(str, "igfx_off", 8)) {
280 "Intel-IOMMU: disable GFX device mapping\n");
281 } else if (!strncmp(str, "forcedac", 8)) {
283 "Intel-IOMMU: Forcing DAC for PCI devices\n");
285 } else if (!strncmp(str, "strict", 6)) {
287 "Intel-IOMMU: disable batched IOTLB flush\n");
288 intel_iommu_strict = 1;
291 str += strcspn(str, ",");
297 __setup("intel_iommu=", intel_iommu_setup);
299 static struct kmem_cache *iommu_domain_cache;
300 static struct kmem_cache *iommu_devinfo_cache;
301 static struct kmem_cache *iommu_iova_cache;
303 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
308 /* trying to avoid low memory issues */
309 flags = current->flags & PF_MEMALLOC;
310 current->flags |= PF_MEMALLOC;
311 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
312 current->flags &= (~PF_MEMALLOC | flags);
317 static inline void *alloc_pgtable_page(void)
322 /* trying to avoid low memory issues */
323 flags = current->flags & PF_MEMALLOC;
324 current->flags |= PF_MEMALLOC;
325 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
326 current->flags &= (~PF_MEMALLOC | flags);
330 static inline void free_pgtable_page(void *vaddr)
332 free_page((unsigned long)vaddr);
335 static inline void *alloc_domain_mem(void)
337 return iommu_kmem_cache_alloc(iommu_domain_cache);
340 static void free_domain_mem(void *vaddr)
342 kmem_cache_free(iommu_domain_cache, vaddr);
345 static inline void * alloc_devinfo_mem(void)
347 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
350 static inline void free_devinfo_mem(void *vaddr)
352 kmem_cache_free(iommu_devinfo_cache, vaddr);
355 struct iova *alloc_iova_mem(void)
357 return iommu_kmem_cache_alloc(iommu_iova_cache);
360 void free_iova_mem(struct iova *iova)
362 kmem_cache_free(iommu_iova_cache, iova);
366 static inline int width_to_agaw(int width);
368 /* calculate agaw for each iommu.
369 * "SAGAW" may be different across iommus, use a default agaw, and
370 * get a supported less agaw for iommus that don't support the default agaw.
372 int iommu_calculate_agaw(struct intel_iommu *iommu)
377 sagaw = cap_sagaw(iommu->cap);
378 for (agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
380 if (test_bit(agaw, &sagaw))
387 /* in native case, each domain is related to only one iommu */
388 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
392 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
393 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
396 return g_iommus[iommu_id];
399 /* Gets context entry for a given bus and devfn */
400 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
403 struct root_entry *root;
404 struct context_entry *context;
405 unsigned long phy_addr;
408 spin_lock_irqsave(&iommu->lock, flags);
409 root = &iommu->root_entry[bus];
410 context = get_context_addr_from_root(root);
412 context = (struct context_entry *)alloc_pgtable_page();
414 spin_unlock_irqrestore(&iommu->lock, flags);
417 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
418 phy_addr = virt_to_phys((void *)context);
419 set_root_value(root, phy_addr);
420 set_root_present(root);
421 __iommu_flush_cache(iommu, root, sizeof(*root));
423 spin_unlock_irqrestore(&iommu->lock, flags);
424 return &context[devfn];
427 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
429 struct root_entry *root;
430 struct context_entry *context;
434 spin_lock_irqsave(&iommu->lock, flags);
435 root = &iommu->root_entry[bus];
436 context = get_context_addr_from_root(root);
441 ret = context_present(&context[devfn]);
443 spin_unlock_irqrestore(&iommu->lock, flags);
447 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
449 struct root_entry *root;
450 struct context_entry *context;
453 spin_lock_irqsave(&iommu->lock, flags);
454 root = &iommu->root_entry[bus];
455 context = get_context_addr_from_root(root);
457 context_clear_entry(&context[devfn]);
458 __iommu_flush_cache(iommu, &context[devfn], \
461 spin_unlock_irqrestore(&iommu->lock, flags);
464 static void free_context_table(struct intel_iommu *iommu)
466 struct root_entry *root;
469 struct context_entry *context;
471 spin_lock_irqsave(&iommu->lock, flags);
472 if (!iommu->root_entry) {
475 for (i = 0; i < ROOT_ENTRY_NR; i++) {
476 root = &iommu->root_entry[i];
477 context = get_context_addr_from_root(root);
479 free_pgtable_page(context);
481 free_pgtable_page(iommu->root_entry);
482 iommu->root_entry = NULL;
484 spin_unlock_irqrestore(&iommu->lock, flags);
487 /* page table handling */
488 #define LEVEL_STRIDE (9)
489 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
491 static inline int agaw_to_level(int agaw)
496 static inline int agaw_to_width(int agaw)
498 return 30 + agaw * LEVEL_STRIDE;
502 static inline int width_to_agaw(int width)
504 return (width - 30) / LEVEL_STRIDE;
507 static inline unsigned int level_to_offset_bits(int level)
509 return (12 + (level - 1) * LEVEL_STRIDE);
512 static inline int address_level_offset(u64 addr, int level)
514 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
517 static inline u64 level_mask(int level)
519 return ((u64)-1 << level_to_offset_bits(level));
522 static inline u64 level_size(int level)
524 return ((u64)1 << level_to_offset_bits(level));
527 static inline u64 align_to_level(u64 addr, int level)
529 return ((addr + level_size(level) - 1) & level_mask(level));
532 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
534 int addr_width = agaw_to_width(domain->agaw);
535 struct dma_pte *parent, *pte = NULL;
536 int level = agaw_to_level(domain->agaw);
539 struct intel_iommu *iommu = domain_get_iommu(domain);
541 BUG_ON(!domain->pgd);
543 addr &= (((u64)1) << addr_width) - 1;
544 parent = domain->pgd;
546 spin_lock_irqsave(&domain->mapping_lock, flags);
550 offset = address_level_offset(addr, level);
551 pte = &parent[offset];
555 if (!dma_pte_present(pte)) {
556 tmp_page = alloc_pgtable_page();
559 spin_unlock_irqrestore(&domain->mapping_lock,
563 __iommu_flush_cache(iommu, tmp_page,
565 dma_set_pte_addr(pte, virt_to_phys(tmp_page));
567 * high level table always sets r/w, last level page
568 * table control read/write
570 dma_set_pte_readable(pte);
571 dma_set_pte_writable(pte);
572 __iommu_flush_cache(iommu, pte, sizeof(*pte));
574 parent = phys_to_virt(dma_pte_addr(pte));
578 spin_unlock_irqrestore(&domain->mapping_lock, flags);
582 /* return address's pte at specific level */
583 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
586 struct dma_pte *parent, *pte = NULL;
587 int total = agaw_to_level(domain->agaw);
590 parent = domain->pgd;
591 while (level <= total) {
592 offset = address_level_offset(addr, total);
593 pte = &parent[offset];
597 if (!dma_pte_present(pte))
599 parent = phys_to_virt(dma_pte_addr(pte));
605 /* clear one page's page table */
606 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
608 struct dma_pte *pte = NULL;
609 struct intel_iommu *iommu = domain_get_iommu(domain);
611 /* get last level pte */
612 pte = dma_addr_level_pte(domain, addr, 1);
616 __iommu_flush_cache(iommu, pte, sizeof(*pte));
620 /* clear last level pte, a tlb flush should be followed */
621 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
623 int addr_width = agaw_to_width(domain->agaw);
625 start &= (((u64)1) << addr_width) - 1;
626 end &= (((u64)1) << addr_width) - 1;
627 /* in case it's partial page */
628 start = PAGE_ALIGN(start);
631 /* we don't need lock here, nobody else touches the iova range */
632 while (start < end) {
633 dma_pte_clear_one(domain, start);
634 start += VTD_PAGE_SIZE;
638 /* free page table pages. last level pte should already be cleared */
639 static void dma_pte_free_pagetable(struct dmar_domain *domain,
642 int addr_width = agaw_to_width(domain->agaw);
644 int total = agaw_to_level(domain->agaw);
647 struct intel_iommu *iommu = domain_get_iommu(domain);
649 start &= (((u64)1) << addr_width) - 1;
650 end &= (((u64)1) << addr_width) - 1;
652 /* we don't need lock here, nobody else touches the iova range */
654 while (level <= total) {
655 tmp = align_to_level(start, level);
656 if (tmp >= end || (tmp + level_size(level) > end))
660 pte = dma_addr_level_pte(domain, tmp, level);
663 phys_to_virt(dma_pte_addr(pte)));
665 __iommu_flush_cache(iommu,
668 tmp += level_size(level);
673 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
674 free_pgtable_page(domain->pgd);
680 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
682 struct root_entry *root;
685 root = (struct root_entry *)alloc_pgtable_page();
689 __iommu_flush_cache(iommu, root, ROOT_SIZE);
691 spin_lock_irqsave(&iommu->lock, flags);
692 iommu->root_entry = root;
693 spin_unlock_irqrestore(&iommu->lock, flags);
698 static void iommu_set_root_entry(struct intel_iommu *iommu)
704 addr = iommu->root_entry;
706 spin_lock_irqsave(&iommu->register_lock, flag);
707 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
709 cmd = iommu->gcmd | DMA_GCMD_SRTP;
710 writel(cmd, iommu->reg + DMAR_GCMD_REG);
712 /* Make sure hardware complete it */
713 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
714 readl, (sts & DMA_GSTS_RTPS), sts);
716 spin_unlock_irqrestore(&iommu->register_lock, flag);
719 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
724 if (!cap_rwbf(iommu->cap))
726 val = iommu->gcmd | DMA_GCMD_WBF;
728 spin_lock_irqsave(&iommu->register_lock, flag);
729 writel(val, iommu->reg + DMAR_GCMD_REG);
731 /* Make sure hardware complete it */
732 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
733 readl, (!(val & DMA_GSTS_WBFS)), val);
735 spin_unlock_irqrestore(&iommu->register_lock, flag);
738 /* return value determine if we need a write buffer flush */
739 static int __iommu_flush_context(struct intel_iommu *iommu,
740 u16 did, u16 source_id, u8 function_mask, u64 type,
741 int non_present_entry_flush)
747 * In the non-present entry flush case, if hardware doesn't cache
748 * non-present entry we do nothing and if hardware cache non-present
749 * entry, we flush entries of domain 0 (the domain id is used to cache
750 * any non-present entries)
752 if (non_present_entry_flush) {
753 if (!cap_caching_mode(iommu->cap))
760 case DMA_CCMD_GLOBAL_INVL:
761 val = DMA_CCMD_GLOBAL_INVL;
763 case DMA_CCMD_DOMAIN_INVL:
764 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
766 case DMA_CCMD_DEVICE_INVL:
767 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
768 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
775 spin_lock_irqsave(&iommu->register_lock, flag);
776 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
778 /* Make sure hardware complete it */
779 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
780 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
782 spin_unlock_irqrestore(&iommu->register_lock, flag);
784 /* flush context entry will implicitly flush write buffer */
788 /* return value determine if we need a write buffer flush */
789 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
790 u64 addr, unsigned int size_order, u64 type,
791 int non_present_entry_flush)
793 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
794 u64 val = 0, val_iva = 0;
798 * In the non-present entry flush case, if hardware doesn't cache
799 * non-present entry we do nothing and if hardware cache non-present
800 * entry, we flush entries of domain 0 (the domain id is used to cache
801 * any non-present entries)
803 if (non_present_entry_flush) {
804 if (!cap_caching_mode(iommu->cap))
811 case DMA_TLB_GLOBAL_FLUSH:
812 /* global flush doesn't need set IVA_REG */
813 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
815 case DMA_TLB_DSI_FLUSH:
816 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
818 case DMA_TLB_PSI_FLUSH:
819 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
820 /* Note: always flush non-leaf currently */
821 val_iva = size_order | addr;
826 /* Note: set drain read/write */
829 * This is probably to be super secure.. Looks like we can
830 * ignore it without any impact.
832 if (cap_read_drain(iommu->cap))
833 val |= DMA_TLB_READ_DRAIN;
835 if (cap_write_drain(iommu->cap))
836 val |= DMA_TLB_WRITE_DRAIN;
838 spin_lock_irqsave(&iommu->register_lock, flag);
839 /* Note: Only uses first TLB reg currently */
841 dmar_writeq(iommu->reg + tlb_offset, val_iva);
842 dmar_writeq(iommu->reg + tlb_offset + 8, val);
844 /* Make sure hardware complete it */
845 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
846 dmar_readq, (!(val & DMA_TLB_IVT)), val);
848 spin_unlock_irqrestore(&iommu->register_lock, flag);
850 /* check IOTLB invalidation granularity */
851 if (DMA_TLB_IAIG(val) == 0)
852 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
853 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
854 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
855 (unsigned long long)DMA_TLB_IIRG(type),
856 (unsigned long long)DMA_TLB_IAIG(val));
857 /* flush iotlb entry will implicitly flush write buffer */
861 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
862 u64 addr, unsigned int pages, int non_present_entry_flush)
866 BUG_ON(addr & (~VTD_PAGE_MASK));
869 /* Fallback to domain selective flush if no PSI support */
870 if (!cap_pgsel_inv(iommu->cap))
871 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
873 non_present_entry_flush);
876 * PSI requires page size to be 2 ^ x, and the base address is naturally
877 * aligned to the size
879 mask = ilog2(__roundup_pow_of_two(pages));
880 /* Fallback to domain selective flush if size is too big */
881 if (mask > cap_max_amask_val(iommu->cap))
882 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
883 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
885 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
887 non_present_entry_flush);
890 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
895 spin_lock_irqsave(&iommu->register_lock, flags);
896 pmen = readl(iommu->reg + DMAR_PMEN_REG);
897 pmen &= ~DMA_PMEN_EPM;
898 writel(pmen, iommu->reg + DMAR_PMEN_REG);
900 /* wait for the protected region status bit to clear */
901 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
902 readl, !(pmen & DMA_PMEN_PRS), pmen);
904 spin_unlock_irqrestore(&iommu->register_lock, flags);
907 static int iommu_enable_translation(struct intel_iommu *iommu)
912 spin_lock_irqsave(&iommu->register_lock, flags);
913 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
915 /* Make sure hardware complete it */
916 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
917 readl, (sts & DMA_GSTS_TES), sts);
919 iommu->gcmd |= DMA_GCMD_TE;
920 spin_unlock_irqrestore(&iommu->register_lock, flags);
924 static int iommu_disable_translation(struct intel_iommu *iommu)
929 spin_lock_irqsave(&iommu->register_lock, flag);
930 iommu->gcmd &= ~DMA_GCMD_TE;
931 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
933 /* Make sure hardware complete it */
934 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
935 readl, (!(sts & DMA_GSTS_TES)), sts);
937 spin_unlock_irqrestore(&iommu->register_lock, flag);
941 /* iommu interrupt handling. Most stuff are MSI-like. */
943 static const char *fault_reason_strings[] =
946 "Present bit in root entry is clear",
947 "Present bit in context entry is clear",
948 "Invalid context entry",
949 "Access beyond MGAW",
950 "PTE Write access is not set",
951 "PTE Read access is not set",
952 "Next page table ptr is invalid",
953 "Root table address invalid",
954 "Context table ptr is invalid",
955 "non-zero reserved fields in RTP",
956 "non-zero reserved fields in CTP",
957 "non-zero reserved fields in PTE",
959 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
961 const char *dmar_get_fault_reason(u8 fault_reason)
963 if (fault_reason > MAX_FAULT_REASON_IDX)
966 return fault_reason_strings[fault_reason];
969 void dmar_msi_unmask(unsigned int irq)
971 struct intel_iommu *iommu = get_irq_data(irq);
975 spin_lock_irqsave(&iommu->register_lock, flag);
976 writel(0, iommu->reg + DMAR_FECTL_REG);
977 /* Read a reg to force flush the post write */
978 readl(iommu->reg + DMAR_FECTL_REG);
979 spin_unlock_irqrestore(&iommu->register_lock, flag);
982 void dmar_msi_mask(unsigned int irq)
985 struct intel_iommu *iommu = get_irq_data(irq);
988 spin_lock_irqsave(&iommu->register_lock, flag);
989 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
990 /* Read a reg to force flush the post write */
991 readl(iommu->reg + DMAR_FECTL_REG);
992 spin_unlock_irqrestore(&iommu->register_lock, flag);
995 void dmar_msi_write(int irq, struct msi_msg *msg)
997 struct intel_iommu *iommu = get_irq_data(irq);
1000 spin_lock_irqsave(&iommu->register_lock, flag);
1001 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1002 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1003 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1004 spin_unlock_irqrestore(&iommu->register_lock, flag);
1007 void dmar_msi_read(int irq, struct msi_msg *msg)
1009 struct intel_iommu *iommu = get_irq_data(irq);
1012 spin_lock_irqsave(&iommu->register_lock, flag);
1013 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1014 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1015 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1016 spin_unlock_irqrestore(&iommu->register_lock, flag);
1019 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
1020 u8 fault_reason, u16 source_id, unsigned long long addr)
1024 reason = dmar_get_fault_reason(fault_reason);
1027 "DMAR:[%s] Request device [%02x:%02x.%d] "
1028 "fault addr %llx \n"
1029 "DMAR:[fault reason %02d] %s\n",
1030 (type ? "DMA Read" : "DMA Write"),
1031 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1032 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1036 #define PRIMARY_FAULT_REG_LEN (16)
1037 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
1039 struct intel_iommu *iommu = dev_id;
1040 int reg, fault_index;
1044 spin_lock_irqsave(&iommu->register_lock, flag);
1045 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1047 /* TBD: ignore advanced fault log currently */
1048 if (!(fault_status & DMA_FSTS_PPF))
1049 goto clear_overflow;
1051 fault_index = dma_fsts_fault_record_index(fault_status);
1052 reg = cap_fault_reg_offset(iommu->cap);
1060 /* highest 32 bits */
1061 data = readl(iommu->reg + reg +
1062 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1063 if (!(data & DMA_FRCD_F))
1066 fault_reason = dma_frcd_fault_reason(data);
1067 type = dma_frcd_type(data);
1069 data = readl(iommu->reg + reg +
1070 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1071 source_id = dma_frcd_source_id(data);
1073 guest_addr = dmar_readq(iommu->reg + reg +
1074 fault_index * PRIMARY_FAULT_REG_LEN);
1075 guest_addr = dma_frcd_page_addr(guest_addr);
1076 /* clear the fault */
1077 writel(DMA_FRCD_F, iommu->reg + reg +
1078 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1080 spin_unlock_irqrestore(&iommu->register_lock, flag);
1082 iommu_page_fault_do_one(iommu, type, fault_reason,
1083 source_id, guest_addr);
1086 if (fault_index > cap_num_fault_regs(iommu->cap))
1088 spin_lock_irqsave(&iommu->register_lock, flag);
1091 /* clear primary fault overflow */
1092 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1093 if (fault_status & DMA_FSTS_PFO)
1094 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
1096 spin_unlock_irqrestore(&iommu->register_lock, flag);
1100 int dmar_set_interrupt(struct intel_iommu *iommu)
1106 printk(KERN_ERR "IOMMU: no free vectors\n");
1110 set_irq_data(irq, iommu);
1113 ret = arch_setup_dmar_msi(irq);
1115 set_irq_data(irq, NULL);
1121 /* Force fault register is cleared */
1122 iommu_page_fault(irq, iommu);
1124 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
1126 printk(KERN_ERR "IOMMU: can't request irq\n");
1130 static int iommu_init_domains(struct intel_iommu *iommu)
1132 unsigned long ndomains;
1133 unsigned long nlongs;
1135 ndomains = cap_ndoms(iommu->cap);
1136 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1137 nlongs = BITS_TO_LONGS(ndomains);
1139 /* TBD: there might be 64K domains,
1140 * consider other allocation for future chip
1142 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1143 if (!iommu->domain_ids) {
1144 printk(KERN_ERR "Allocating domain id array failed\n");
1147 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1149 if (!iommu->domains) {
1150 printk(KERN_ERR "Allocating domain array failed\n");
1151 kfree(iommu->domain_ids);
1155 spin_lock_init(&iommu->lock);
1158 * if Caching mode is set, then invalid translations are tagged
1159 * with domainid 0. Hence we need to pre-allocate it.
1161 if (cap_caching_mode(iommu->cap))
1162 set_bit(0, iommu->domain_ids);
1167 static void domain_exit(struct dmar_domain *domain);
1169 void free_dmar_iommu(struct intel_iommu *iommu)
1171 struct dmar_domain *domain;
1174 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1175 for (; i < cap_ndoms(iommu->cap); ) {
1176 domain = iommu->domains[i];
1177 clear_bit(i, iommu->domain_ids);
1178 domain_exit(domain);
1179 i = find_next_bit(iommu->domain_ids,
1180 cap_ndoms(iommu->cap), i+1);
1183 if (iommu->gcmd & DMA_GCMD_TE)
1184 iommu_disable_translation(iommu);
1187 set_irq_data(iommu->irq, NULL);
1188 /* This will mask the irq */
1189 free_irq(iommu->irq, iommu);
1190 destroy_irq(iommu->irq);
1193 kfree(iommu->domains);
1194 kfree(iommu->domain_ids);
1196 g_iommus[iommu->seq_id] = NULL;
1198 /* if all iommus are freed, free g_iommus */
1199 for (i = 0; i < g_num_of_iommus; i++) {
1204 if (i == g_num_of_iommus)
1207 /* free context mapping */
1208 free_context_table(iommu);
1211 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1214 unsigned long ndomains;
1215 struct dmar_domain *domain;
1216 unsigned long flags;
1218 domain = alloc_domain_mem();
1222 ndomains = cap_ndoms(iommu->cap);
1224 spin_lock_irqsave(&iommu->lock, flags);
1225 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1226 if (num >= ndomains) {
1227 spin_unlock_irqrestore(&iommu->lock, flags);
1228 free_domain_mem(domain);
1229 printk(KERN_ERR "IOMMU: no free domain ids\n");
1233 set_bit(num, iommu->domain_ids);
1235 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1236 set_bit(iommu->seq_id, &domain->iommu_bmp);
1238 iommu->domains[num] = domain;
1239 spin_unlock_irqrestore(&iommu->lock, flags);
1244 static void iommu_free_domain(struct dmar_domain *domain)
1246 unsigned long flags;
1247 struct intel_iommu *iommu;
1249 iommu = domain_get_iommu(domain);
1251 spin_lock_irqsave(&iommu->lock, flags);
1252 clear_bit(domain->id, iommu->domain_ids);
1253 spin_unlock_irqrestore(&iommu->lock, flags);
1256 static struct iova_domain reserved_iova_list;
1257 static struct lock_class_key reserved_alloc_key;
1258 static struct lock_class_key reserved_rbtree_key;
1260 static void dmar_init_reserved_ranges(void)
1262 struct pci_dev *pdev = NULL;
1267 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1269 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1270 &reserved_alloc_key);
1271 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1272 &reserved_rbtree_key);
1274 /* IOAPIC ranges shouldn't be accessed by DMA */
1275 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1276 IOVA_PFN(IOAPIC_RANGE_END));
1278 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1280 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1281 for_each_pci_dev(pdev) {
1284 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1285 r = &pdev->resource[i];
1286 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1290 size = r->end - addr;
1291 size = PAGE_ALIGN(size);
1292 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1293 IOVA_PFN(size + addr) - 1);
1295 printk(KERN_ERR "Reserve iova failed\n");
1301 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1303 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1306 static inline int guestwidth_to_adjustwidth(int gaw)
1309 int r = (gaw - 12) % 9;
1320 static int domain_init(struct dmar_domain *domain, int guest_width)
1322 struct intel_iommu *iommu;
1323 int adjust_width, agaw;
1324 unsigned long sagaw;
1326 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1327 spin_lock_init(&domain->mapping_lock);
1329 domain_reserve_special_ranges(domain);
1331 /* calculate AGAW */
1332 iommu = domain_get_iommu(domain);
1333 if (guest_width > cap_mgaw(iommu->cap))
1334 guest_width = cap_mgaw(iommu->cap);
1335 domain->gaw = guest_width;
1336 adjust_width = guestwidth_to_adjustwidth(guest_width);
1337 agaw = width_to_agaw(adjust_width);
1338 sagaw = cap_sagaw(iommu->cap);
1339 if (!test_bit(agaw, &sagaw)) {
1340 /* hardware doesn't support it, choose a bigger one */
1341 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1342 agaw = find_next_bit(&sagaw, 5, agaw);
1346 domain->agaw = agaw;
1347 INIT_LIST_HEAD(&domain->devices);
1349 /* always allocate the top pgd */
1350 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1353 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1357 static void domain_exit(struct dmar_domain *domain)
1361 /* Domain 0 is reserved, so dont process it */
1365 domain_remove_dev_info(domain);
1367 put_iova_domain(&domain->iovad);
1368 end = DOMAIN_MAX_ADDR(domain->gaw);
1369 end = end & (~PAGE_MASK);
1372 dma_pte_clear_range(domain, 0, end);
1374 /* free page tables */
1375 dma_pte_free_pagetable(domain, 0, end);
1377 iommu_free_domain(domain);
1378 free_domain_mem(domain);
1381 static int domain_context_mapping_one(struct dmar_domain *domain,
1384 struct context_entry *context;
1385 struct intel_iommu *iommu = domain_get_iommu(domain);
1386 unsigned long flags;
1388 pr_debug("Set context mapping for %02x:%02x.%d\n",
1389 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1390 BUG_ON(!domain->pgd);
1391 context = device_to_context_entry(iommu, bus, devfn);
1394 spin_lock_irqsave(&iommu->lock, flags);
1395 if (context_present(context)) {
1396 spin_unlock_irqrestore(&iommu->lock, flags);
1400 context_set_domain_id(context, domain->id);
1401 context_set_address_width(context, domain->agaw);
1402 context_set_address_root(context, virt_to_phys(domain->pgd));
1403 context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1404 context_set_fault_enable(context);
1405 context_set_present(context);
1406 __iommu_flush_cache(iommu, context, sizeof(*context));
1408 /* it's a non-present to present mapping */
1409 if (iommu->flush.flush_context(iommu, domain->id,
1410 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1411 DMA_CCMD_DEVICE_INVL, 1))
1412 iommu_flush_write_buffer(iommu);
1414 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1416 spin_unlock_irqrestore(&iommu->lock, flags);
1421 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1424 struct pci_dev *tmp, *parent;
1426 ret = domain_context_mapping_one(domain, pdev->bus->number,
1431 /* dependent device mapping */
1432 tmp = pci_find_upstream_pcie_bridge(pdev);
1435 /* Secondary interface's bus number and devfn 0 */
1436 parent = pdev->bus->self;
1437 while (parent != tmp) {
1438 ret = domain_context_mapping_one(domain, parent->bus->number,
1442 parent = parent->bus->self;
1444 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1445 return domain_context_mapping_one(domain,
1446 tmp->subordinate->number, 0);
1447 else /* this is a legacy PCI bridge */
1448 return domain_context_mapping_one(domain,
1449 tmp->bus->number, tmp->devfn);
1452 static int domain_context_mapped(struct dmar_domain *domain,
1453 struct pci_dev *pdev)
1456 struct pci_dev *tmp, *parent;
1457 struct intel_iommu *iommu = domain_get_iommu(domain);
1459 ret = device_context_mapped(iommu,
1460 pdev->bus->number, pdev->devfn);
1463 /* dependent device mapping */
1464 tmp = pci_find_upstream_pcie_bridge(pdev);
1467 /* Secondary interface's bus number and devfn 0 */
1468 parent = pdev->bus->self;
1469 while (parent != tmp) {
1470 ret = device_context_mapped(iommu, parent->bus->number,
1474 parent = parent->bus->self;
1477 return device_context_mapped(iommu,
1478 tmp->subordinate->number, 0);
1480 return device_context_mapped(iommu,
1481 tmp->bus->number, tmp->devfn);
1485 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1486 u64 hpa, size_t size, int prot)
1488 u64 start_pfn, end_pfn;
1489 struct dma_pte *pte;
1491 int addr_width = agaw_to_width(domain->agaw);
1492 struct intel_iommu *iommu = domain_get_iommu(domain);
1494 hpa &= (((u64)1) << addr_width) - 1;
1496 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1499 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1500 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1502 while (start_pfn < end_pfn) {
1503 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1506 /* We don't need lock here, nobody else
1507 * touches the iova range
1509 BUG_ON(dma_pte_addr(pte));
1510 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1511 dma_set_pte_prot(pte, prot);
1512 __iommu_flush_cache(iommu, pte, sizeof(*pte));
1519 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1521 struct intel_iommu *iommu = domain_get_iommu(domain);
1523 clear_context_table(iommu, bus, devfn);
1524 iommu->flush.flush_context(iommu, 0, 0, 0,
1525 DMA_CCMD_GLOBAL_INVL, 0);
1526 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1527 DMA_TLB_GLOBAL_FLUSH, 0);
1530 static void domain_remove_dev_info(struct dmar_domain *domain)
1532 struct device_domain_info *info;
1533 unsigned long flags;
1535 spin_lock_irqsave(&device_domain_lock, flags);
1536 while (!list_empty(&domain->devices)) {
1537 info = list_entry(domain->devices.next,
1538 struct device_domain_info, link);
1539 list_del(&info->link);
1540 list_del(&info->global);
1542 info->dev->dev.archdata.iommu = NULL;
1543 spin_unlock_irqrestore(&device_domain_lock, flags);
1545 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1546 free_devinfo_mem(info);
1548 spin_lock_irqsave(&device_domain_lock, flags);
1550 spin_unlock_irqrestore(&device_domain_lock, flags);
1555 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1557 static struct dmar_domain *
1558 find_domain(struct pci_dev *pdev)
1560 struct device_domain_info *info;
1562 /* No lock here, assumes no domain exit in normal case */
1563 info = pdev->dev.archdata.iommu;
1565 return info->domain;
1569 /* domain is initialized */
1570 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1572 struct dmar_domain *domain, *found = NULL;
1573 struct intel_iommu *iommu;
1574 struct dmar_drhd_unit *drhd;
1575 struct device_domain_info *info, *tmp;
1576 struct pci_dev *dev_tmp;
1577 unsigned long flags;
1578 int bus = 0, devfn = 0;
1580 domain = find_domain(pdev);
1584 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1586 if (dev_tmp->is_pcie) {
1587 bus = dev_tmp->subordinate->number;
1590 bus = dev_tmp->bus->number;
1591 devfn = dev_tmp->devfn;
1593 spin_lock_irqsave(&device_domain_lock, flags);
1594 list_for_each_entry(info, &device_domain_list, global) {
1595 if (info->bus == bus && info->devfn == devfn) {
1596 found = info->domain;
1600 spin_unlock_irqrestore(&device_domain_lock, flags);
1601 /* pcie-pci bridge already has a domain, uses it */
1608 /* Allocate new domain for the device */
1609 drhd = dmar_find_matched_drhd_unit(pdev);
1611 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1615 iommu = drhd->iommu;
1617 domain = iommu_alloc_domain(iommu);
1621 if (domain_init(domain, gaw)) {
1622 domain_exit(domain);
1626 /* register pcie-to-pci device */
1628 info = alloc_devinfo_mem();
1630 domain_exit(domain);
1634 info->devfn = devfn;
1636 info->domain = domain;
1637 /* This domain is shared by devices under p2p bridge */
1638 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1640 /* pcie-to-pci bridge already has a domain, uses it */
1642 spin_lock_irqsave(&device_domain_lock, flags);
1643 list_for_each_entry(tmp, &device_domain_list, global) {
1644 if (tmp->bus == bus && tmp->devfn == devfn) {
1645 found = tmp->domain;
1650 free_devinfo_mem(info);
1651 domain_exit(domain);
1654 list_add(&info->link, &domain->devices);
1655 list_add(&info->global, &device_domain_list);
1657 spin_unlock_irqrestore(&device_domain_lock, flags);
1661 info = alloc_devinfo_mem();
1664 info->bus = pdev->bus->number;
1665 info->devfn = pdev->devfn;
1667 info->domain = domain;
1668 spin_lock_irqsave(&device_domain_lock, flags);
1669 /* somebody is fast */
1670 found = find_domain(pdev);
1671 if (found != NULL) {
1672 spin_unlock_irqrestore(&device_domain_lock, flags);
1673 if (found != domain) {
1674 domain_exit(domain);
1677 free_devinfo_mem(info);
1680 list_add(&info->link, &domain->devices);
1681 list_add(&info->global, &device_domain_list);
1682 pdev->dev.archdata.iommu = info;
1683 spin_unlock_irqrestore(&device_domain_lock, flags);
1686 /* recheck it here, maybe others set it */
1687 return find_domain(pdev);
1690 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1691 unsigned long long start,
1692 unsigned long long end)
1694 struct dmar_domain *domain;
1696 unsigned long long base;
1700 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1701 pci_name(pdev), start, end);
1702 /* page table init */
1703 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1707 /* The address might not be aligned */
1708 base = start & PAGE_MASK;
1710 size = PAGE_ALIGN(size);
1711 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1712 IOVA_PFN(base + size) - 1)) {
1713 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1718 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1719 size, base, pci_name(pdev));
1721 * RMRR range might have overlap with physical memory range,
1724 dma_pte_clear_range(domain, base, base + size);
1726 ret = domain_page_mapping(domain, base, base, size,
1727 DMA_PTE_READ|DMA_PTE_WRITE);
1731 /* context entry init */
1732 ret = domain_context_mapping(domain, pdev);
1736 domain_exit(domain);
1741 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1742 struct pci_dev *pdev)
1744 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1746 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1747 rmrr->end_address + 1);
1750 #ifdef CONFIG_DMAR_GFX_WA
1751 struct iommu_prepare_data {
1752 struct pci_dev *pdev;
1756 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1757 unsigned long end_pfn, void *datax)
1759 struct iommu_prepare_data *data;
1761 data = (struct iommu_prepare_data *)datax;
1763 data->ret = iommu_prepare_identity_map(data->pdev,
1764 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1769 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1772 struct iommu_prepare_data data;
1777 for_each_online_node(nid) {
1778 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1785 static void __init iommu_prepare_gfx_mapping(void)
1787 struct pci_dev *pdev = NULL;
1790 for_each_pci_dev(pdev) {
1791 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1792 !IS_GFX_DEVICE(pdev))
1794 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1796 ret = iommu_prepare_with_active_regions(pdev);
1798 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1801 #else /* !CONFIG_DMAR_GFX_WA */
1802 static inline void iommu_prepare_gfx_mapping(void)
1808 #ifdef CONFIG_DMAR_FLOPPY_WA
1809 static inline void iommu_prepare_isa(void)
1811 struct pci_dev *pdev;
1814 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1818 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1819 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1822 printk("IOMMU: Failed to create 0-64M identity map, "
1823 "floppy might not work\n");
1827 static inline void iommu_prepare_isa(void)
1831 #endif /* !CONFIG_DMAR_FLPY_WA */
1833 static int __init init_dmars(void)
1835 struct dmar_drhd_unit *drhd;
1836 struct dmar_rmrr_unit *rmrr;
1837 struct pci_dev *pdev;
1838 struct intel_iommu *iommu;
1839 int i, ret, unit = 0;
1844 * initialize and program root entry to not present
1847 for_each_drhd_unit(drhd) {
1850 * lock not needed as this is only incremented in the single
1851 * threaded kernel __init code path all other access are read
1856 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
1859 printk(KERN_ERR "Allocating global iommu array failed\n");
1864 deferred_flush = kzalloc(g_num_of_iommus *
1865 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1866 if (!deferred_flush) {
1872 for_each_drhd_unit(drhd) {
1876 iommu = drhd->iommu;
1877 g_iommus[iommu->seq_id] = iommu;
1879 ret = iommu_init_domains(iommu);
1885 * we could share the same root & context tables
1886 * amoung all IOMMU's. Need to Split it later.
1888 ret = iommu_alloc_root_entry(iommu);
1890 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1895 for_each_drhd_unit(drhd) {
1899 iommu = drhd->iommu;
1900 if (dmar_enable_qi(iommu)) {
1902 * Queued Invalidate not enabled, use Register Based
1905 iommu->flush.flush_context = __iommu_flush_context;
1906 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1907 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1909 (unsigned long long)drhd->reg_base_addr);
1911 iommu->flush.flush_context = qi_flush_context;
1912 iommu->flush.flush_iotlb = qi_flush_iotlb;
1913 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1915 (unsigned long long)drhd->reg_base_addr);
1921 * for each dev attached to rmrr
1923 * locate drhd for dev, alloc domain for dev
1924 * allocate free domain
1925 * allocate page table entries for rmrr
1926 * if context not allocated for bus
1927 * allocate and init context
1928 * set present in root table for this bus
1929 * init context with domain, translation etc
1933 for_each_rmrr_units(rmrr) {
1934 for (i = 0; i < rmrr->devices_cnt; i++) {
1935 pdev = rmrr->devices[i];
1936 /* some BIOS lists non-exist devices in DMAR table */
1939 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1942 "IOMMU: mapping reserved region failed\n");
1946 iommu_prepare_gfx_mapping();
1948 iommu_prepare_isa();
1953 * global invalidate context cache
1954 * global invalidate iotlb
1955 * enable translation
1957 for_each_drhd_unit(drhd) {
1960 iommu = drhd->iommu;
1961 sprintf (iommu->name, "dmar%d", unit++);
1963 iommu_flush_write_buffer(iommu);
1965 ret = dmar_set_interrupt(iommu);
1969 iommu_set_root_entry(iommu);
1971 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1973 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1975 iommu_disable_protect_mem_regions(iommu);
1977 ret = iommu_enable_translation(iommu);
1984 for_each_drhd_unit(drhd) {
1987 iommu = drhd->iommu;
1994 static inline u64 aligned_size(u64 host_addr, size_t size)
1997 addr = (host_addr & (~PAGE_MASK)) + size;
1998 return PAGE_ALIGN(addr);
2002 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
2006 /* Make sure it's in range */
2007 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
2008 if (!size || (IOVA_START_ADDR + size > end))
2011 piova = alloc_iova(&domain->iovad,
2012 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2016 static struct iova *
2017 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2018 size_t size, u64 dma_mask)
2020 struct pci_dev *pdev = to_pci_dev(dev);
2021 struct iova *iova = NULL;
2023 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
2024 iova = iommu_alloc_iova(domain, size, dma_mask);
2027 * First try to allocate an io virtual address in
2028 * DMA_32BIT_MASK and if that fails then try allocating
2031 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
2033 iova = iommu_alloc_iova(domain, size, dma_mask);
2037 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2044 static struct dmar_domain *
2045 get_valid_domain_for_dev(struct pci_dev *pdev)
2047 struct dmar_domain *domain;
2050 domain = get_domain_for_dev(pdev,
2051 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2054 "Allocating domain for %s failed", pci_name(pdev));
2058 /* make sure context mapping is ok */
2059 if (unlikely(!domain_context_mapped(domain, pdev))) {
2060 ret = domain_context_mapping(domain, pdev);
2063 "Domain context map for %s failed",
2072 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2073 size_t size, int dir, u64 dma_mask)
2075 struct pci_dev *pdev = to_pci_dev(hwdev);
2076 struct dmar_domain *domain;
2077 phys_addr_t start_paddr;
2081 struct intel_iommu *iommu;
2083 BUG_ON(dir == DMA_NONE);
2084 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2087 domain = get_valid_domain_for_dev(pdev);
2091 iommu = domain_get_iommu(domain);
2092 size = aligned_size((u64)paddr, size);
2094 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2098 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2101 * Check if DMAR supports zero-length reads on write only
2104 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2105 !cap_zlr(iommu->cap))
2106 prot |= DMA_PTE_READ;
2107 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2108 prot |= DMA_PTE_WRITE;
2110 * paddr - (paddr + size) might be partial page, we should map the whole
2111 * page. Note: if two part of one page are separately mapped, we
2112 * might have two guest_addr mapping to the same host paddr, but this
2113 * is not a big problem
2115 ret = domain_page_mapping(domain, start_paddr,
2116 ((u64)paddr) & PAGE_MASK, size, prot);
2120 /* it's a non-present to present mapping */
2121 ret = iommu_flush_iotlb_psi(iommu, domain->id,
2122 start_paddr, size >> VTD_PAGE_SHIFT, 1);
2124 iommu_flush_write_buffer(iommu);
2126 return start_paddr + ((u64)paddr & (~PAGE_MASK));
2130 __free_iova(&domain->iovad, iova);
2131 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2132 pci_name(pdev), size, (unsigned long long)paddr, dir);
2136 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
2137 size_t size, int dir)
2139 return __intel_map_single(hwdev, paddr, size, dir,
2140 to_pci_dev(hwdev)->dma_mask);
2143 static void flush_unmaps(void)
2149 /* just flush them all */
2150 for (i = 0; i < g_num_of_iommus; i++) {
2151 struct intel_iommu *iommu = g_iommus[i];
2155 if (deferred_flush[i].next) {
2156 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2157 DMA_TLB_GLOBAL_FLUSH, 0);
2158 for (j = 0; j < deferred_flush[i].next; j++) {
2159 __free_iova(&deferred_flush[i].domain[j]->iovad,
2160 deferred_flush[i].iova[j]);
2162 deferred_flush[i].next = 0;
2169 static void flush_unmaps_timeout(unsigned long data)
2171 unsigned long flags;
2173 spin_lock_irqsave(&async_umap_flush_lock, flags);
2175 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2178 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2180 unsigned long flags;
2182 struct intel_iommu *iommu;
2184 spin_lock_irqsave(&async_umap_flush_lock, flags);
2185 if (list_size == HIGH_WATER_MARK)
2188 iommu = domain_get_iommu(dom);
2189 iommu_id = iommu->seq_id;
2191 next = deferred_flush[iommu_id].next;
2192 deferred_flush[iommu_id].domain[next] = dom;
2193 deferred_flush[iommu_id].iova[next] = iova;
2194 deferred_flush[iommu_id].next++;
2197 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2201 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2204 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2207 struct pci_dev *pdev = to_pci_dev(dev);
2208 struct dmar_domain *domain;
2209 unsigned long start_addr;
2211 struct intel_iommu *iommu;
2213 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2215 domain = find_domain(pdev);
2218 iommu = domain_get_iommu(domain);
2220 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2224 start_addr = iova->pfn_lo << PAGE_SHIFT;
2225 size = aligned_size((u64)dev_addr, size);
2227 pr_debug("Device %s unmapping: %lx@%llx\n",
2228 pci_name(pdev), size, (unsigned long long)start_addr);
2230 /* clear the whole page */
2231 dma_pte_clear_range(domain, start_addr, start_addr + size);
2232 /* free page tables */
2233 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2234 if (intel_iommu_strict) {
2235 if (iommu_flush_iotlb_psi(iommu,
2236 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2237 iommu_flush_write_buffer(iommu);
2239 __free_iova(&domain->iovad, iova);
2241 add_unmap(domain, iova);
2243 * queue up the release of the unmap to save the 1/6th of the
2244 * cpu used up by the iotlb flush operation...
2249 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2250 dma_addr_t *dma_handle, gfp_t flags)
2255 size = PAGE_ALIGN(size);
2256 order = get_order(size);
2257 flags &= ~(GFP_DMA | GFP_DMA32);
2259 vaddr = (void *)__get_free_pages(flags, order);
2262 memset(vaddr, 0, size);
2264 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2266 hwdev->coherent_dma_mask);
2269 free_pages((unsigned long)vaddr, order);
2273 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2274 dma_addr_t dma_handle)
2278 size = PAGE_ALIGN(size);
2279 order = get_order(size);
2281 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2282 free_pages((unsigned long)vaddr, order);
2285 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2287 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2288 int nelems, int dir)
2291 struct pci_dev *pdev = to_pci_dev(hwdev);
2292 struct dmar_domain *domain;
2293 unsigned long start_addr;
2297 struct scatterlist *sg;
2298 struct intel_iommu *iommu;
2300 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2303 domain = find_domain(pdev);
2306 iommu = domain_get_iommu(domain);
2308 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2311 for_each_sg(sglist, sg, nelems, i) {
2312 addr = SG_ENT_VIRT_ADDRESS(sg);
2313 size += aligned_size((u64)addr, sg->length);
2316 start_addr = iova->pfn_lo << PAGE_SHIFT;
2318 /* clear the whole page */
2319 dma_pte_clear_range(domain, start_addr, start_addr + size);
2320 /* free page tables */
2321 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2323 if (iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2324 size >> VTD_PAGE_SHIFT, 0))
2325 iommu_flush_write_buffer(iommu);
2328 __free_iova(&domain->iovad, iova);
2331 static int intel_nontranslate_map_sg(struct device *hddev,
2332 struct scatterlist *sglist, int nelems, int dir)
2335 struct scatterlist *sg;
2337 for_each_sg(sglist, sg, nelems, i) {
2338 BUG_ON(!sg_page(sg));
2339 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2340 sg->dma_length = sg->length;
2345 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2350 struct pci_dev *pdev = to_pci_dev(hwdev);
2351 struct dmar_domain *domain;
2355 struct iova *iova = NULL;
2357 struct scatterlist *sg;
2358 unsigned long start_addr;
2359 struct intel_iommu *iommu;
2361 BUG_ON(dir == DMA_NONE);
2362 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2363 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2365 domain = get_valid_domain_for_dev(pdev);
2369 iommu = domain_get_iommu(domain);
2371 for_each_sg(sglist, sg, nelems, i) {
2372 addr = SG_ENT_VIRT_ADDRESS(sg);
2373 addr = (void *)virt_to_phys(addr);
2374 size += aligned_size((u64)addr, sg->length);
2377 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2379 sglist->dma_length = 0;
2384 * Check if DMAR supports zero-length reads on write only
2387 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2388 !cap_zlr(iommu->cap))
2389 prot |= DMA_PTE_READ;
2390 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2391 prot |= DMA_PTE_WRITE;
2393 start_addr = iova->pfn_lo << PAGE_SHIFT;
2395 for_each_sg(sglist, sg, nelems, i) {
2396 addr = SG_ENT_VIRT_ADDRESS(sg);
2397 addr = (void *)virt_to_phys(addr);
2398 size = aligned_size((u64)addr, sg->length);
2399 ret = domain_page_mapping(domain, start_addr + offset,
2400 ((u64)addr) & PAGE_MASK,
2403 /* clear the page */
2404 dma_pte_clear_range(domain, start_addr,
2405 start_addr + offset);
2406 /* free page tables */
2407 dma_pte_free_pagetable(domain, start_addr,
2408 start_addr + offset);
2410 __free_iova(&domain->iovad, iova);
2413 sg->dma_address = start_addr + offset +
2414 ((u64)addr & (~PAGE_MASK));
2415 sg->dma_length = sg->length;
2419 /* it's a non-present to present mapping */
2420 if (iommu_flush_iotlb_psi(iommu, domain->id,
2421 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2422 iommu_flush_write_buffer(iommu);
2426 static struct dma_mapping_ops intel_dma_ops = {
2427 .alloc_coherent = intel_alloc_coherent,
2428 .free_coherent = intel_free_coherent,
2429 .map_single = intel_map_single,
2430 .unmap_single = intel_unmap_single,
2431 .map_sg = intel_map_sg,
2432 .unmap_sg = intel_unmap_sg,
2435 static inline int iommu_domain_cache_init(void)
2439 iommu_domain_cache = kmem_cache_create("iommu_domain",
2440 sizeof(struct dmar_domain),
2445 if (!iommu_domain_cache) {
2446 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2453 static inline int iommu_devinfo_cache_init(void)
2457 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2458 sizeof(struct device_domain_info),
2462 if (!iommu_devinfo_cache) {
2463 printk(KERN_ERR "Couldn't create devinfo cache\n");
2470 static inline int iommu_iova_cache_init(void)
2474 iommu_iova_cache = kmem_cache_create("iommu_iova",
2475 sizeof(struct iova),
2479 if (!iommu_iova_cache) {
2480 printk(KERN_ERR "Couldn't create iova cache\n");
2487 static int __init iommu_init_mempool(void)
2490 ret = iommu_iova_cache_init();
2494 ret = iommu_domain_cache_init();
2498 ret = iommu_devinfo_cache_init();
2502 kmem_cache_destroy(iommu_domain_cache);
2504 kmem_cache_destroy(iommu_iova_cache);
2509 static void __init iommu_exit_mempool(void)
2511 kmem_cache_destroy(iommu_devinfo_cache);
2512 kmem_cache_destroy(iommu_domain_cache);
2513 kmem_cache_destroy(iommu_iova_cache);
2517 static void __init init_no_remapping_devices(void)
2519 struct dmar_drhd_unit *drhd;
2521 for_each_drhd_unit(drhd) {
2522 if (!drhd->include_all) {
2524 for (i = 0; i < drhd->devices_cnt; i++)
2525 if (drhd->devices[i] != NULL)
2527 /* ignore DMAR unit if no pci devices exist */
2528 if (i == drhd->devices_cnt)
2536 for_each_drhd_unit(drhd) {
2538 if (drhd->ignored || drhd->include_all)
2541 for (i = 0; i < drhd->devices_cnt; i++)
2542 if (drhd->devices[i] &&
2543 !IS_GFX_DEVICE(drhd->devices[i]))
2546 if (i < drhd->devices_cnt)
2549 /* bypass IOMMU if it is just for gfx devices */
2551 for (i = 0; i < drhd->devices_cnt; i++) {
2552 if (!drhd->devices[i])
2554 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2559 int __init intel_iommu_init(void)
2563 if (dmar_table_init())
2566 if (dmar_dev_scope_init())
2570 * Check the need for DMA-remapping initialization now.
2571 * Above initialization will also be used by Interrupt-remapping.
2573 if (no_iommu || swiotlb || dmar_disabled)
2576 iommu_init_mempool();
2577 dmar_init_reserved_ranges();
2579 init_no_remapping_devices();
2583 printk(KERN_ERR "IOMMU: dmar init failed\n");
2584 put_iova_domain(&reserved_iova_list);
2585 iommu_exit_mempool();
2589 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2591 init_timer(&unmap_timer);
2593 dma_ops = &intel_dma_ops;
2597 void intel_iommu_domain_exit(struct dmar_domain *domain)
2601 /* Domain 0 is reserved, so dont process it */
2605 end = DOMAIN_MAX_ADDR(domain->gaw);
2606 end = end & (~VTD_PAGE_MASK);
2609 dma_pte_clear_range(domain, 0, end);
2611 /* free page tables */
2612 dma_pte_free_pagetable(domain, 0, end);
2614 iommu_free_domain(domain);
2615 free_domain_mem(domain);
2617 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2619 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2621 struct dmar_drhd_unit *drhd;
2622 struct dmar_domain *domain;
2623 struct intel_iommu *iommu;
2625 drhd = dmar_find_matched_drhd_unit(pdev);
2627 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2631 iommu = drhd->iommu;
2634 "intel_iommu_domain_alloc: iommu == NULL\n");
2637 domain = iommu_alloc_domain(iommu);
2640 "intel_iommu_domain_alloc: domain == NULL\n");
2643 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2645 "intel_iommu_domain_alloc: domain_init() failed\n");
2646 intel_iommu_domain_exit(domain);
2651 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2653 int intel_iommu_context_mapping(
2654 struct dmar_domain *domain, struct pci_dev *pdev)
2657 rc = domain_context_mapping(domain, pdev);
2660 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2662 int intel_iommu_page_mapping(
2663 struct dmar_domain *domain, dma_addr_t iova,
2664 u64 hpa, size_t size, int prot)
2667 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2670 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2672 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2674 detach_domain_for_dev(domain, bus, devfn);
2676 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2678 struct dmar_domain *
2679 intel_iommu_find_domain(struct pci_dev *pdev)
2681 return find_domain(pdev);
2683 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2685 int intel_iommu_found(void)
2687 return g_num_of_iommus;
2689 EXPORT_SYMBOL_GPL(intel_iommu_found);
2691 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2693 struct dma_pte *pte;
2697 pte = addr_to_dma_pte(domain, iova);
2700 pfn = dma_pte_addr(pte);
2702 return pfn >> VTD_PAGE_SHIFT;
2704 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);