2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/intel-iommu.h>
38 #include <asm/cacheflush.h>
39 #include <asm/iommu.h>
42 #define ROOT_SIZE VTD_PAGE_SIZE
43 #define CONTEXT_SIZE VTD_PAGE_SIZE
45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48 #define IOAPIC_RANGE_START (0xfee00000)
49 #define IOAPIC_RANGE_END (0xfeefffff)
50 #define IOVA_START_ADDR (0x1000)
52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
57 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
58 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
63 * 12-63: Context Ptr (12 - (haw-1))
70 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
71 static inline bool root_present(struct root_entry *root)
73 return (root->val & 1);
75 static inline void set_root_present(struct root_entry *root)
79 static inline void set_root_value(struct root_entry *root, unsigned long value)
81 root->val |= value & VTD_PAGE_MASK;
84 static inline struct context_entry *
85 get_context_addr_from_root(struct root_entry *root)
87 return (struct context_entry *)
88 (root_present(root)?phys_to_virt(
89 root->val & VTD_PAGE_MASK) :
96 * 1: fault processing disable
97 * 2-3: translation type
98 * 12-63: address space root
104 struct context_entry {
109 static inline bool context_present(struct context_entry *context)
111 return (context->lo & 1);
113 static inline void context_set_present(struct context_entry *context)
118 static inline void context_set_fault_enable(struct context_entry *context)
120 context->lo &= (((u64)-1) << 2) | 1;
123 #define CONTEXT_TT_MULTI_LEVEL 0
125 static inline void context_set_translation_type(struct context_entry *context,
128 context->lo &= (((u64)-1) << 4) | 3;
129 context->lo |= (value & 3) << 2;
132 static inline void context_set_address_root(struct context_entry *context,
135 context->lo |= value & VTD_PAGE_MASK;
138 static inline void context_set_address_width(struct context_entry *context,
141 context->hi |= value & 7;
144 static inline void context_set_domain_id(struct context_entry *context,
147 context->hi |= (value & ((1 << 16) - 1)) << 8;
150 static inline void context_clear_entry(struct context_entry *context)
162 * 12-63: Host physcial address
167 #define dma_clear_pte(p) do {(p).val = 0;} while (0)
169 #define dma_set_pte_readable(p) do {(p).val |= DMA_PTE_READ;} while (0)
170 #define dma_set_pte_writable(p) do {(p).val |= DMA_PTE_WRITE;} while (0)
171 #define dma_set_pte_prot(p, prot) \
172 do {(p).val = ((p).val & ~3) | ((prot) & 3); } while (0)
173 #define dma_pte_addr(p) ((p).val & VTD_PAGE_MASK)
174 #define dma_set_pte_addr(p, addr) do {\
175 (p).val |= ((addr) & VTD_PAGE_MASK); } while (0)
176 #define dma_pte_present(p) (((p).val & 3) != 0)
179 int id; /* domain id */
180 struct intel_iommu *iommu; /* back pointer to owning iommu */
182 struct list_head devices; /* all devices' list */
183 struct iova_domain iovad; /* iova's that belong to this domain */
185 struct dma_pte *pgd; /* virtual address */
186 spinlock_t mapping_lock; /* page table lock */
187 int gaw; /* max guest address width */
189 /* adjusted guest address width, 0 is level 2 30-bit */
192 #define DOMAIN_FLAG_MULTIPLE_DEVICES 1
196 /* PCI domain-device relationship */
197 struct device_domain_info {
198 struct list_head link; /* link to domain siblings */
199 struct list_head global; /* link to global list */
200 u8 bus; /* PCI bus numer */
201 u8 devfn; /* PCI devfn number */
202 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
203 struct dmar_domain *domain; /* pointer to domain */
206 static void flush_unmaps_timeout(unsigned long data);
208 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
210 #define HIGH_WATER_MARK 250
211 struct deferred_flush_tables {
213 struct iova *iova[HIGH_WATER_MARK];
214 struct dmar_domain *domain[HIGH_WATER_MARK];
217 static struct deferred_flush_tables *deferred_flush;
219 /* bitmap for indexing intel_iommus */
220 static int g_num_of_iommus;
222 static DEFINE_SPINLOCK(async_umap_flush_lock);
223 static LIST_HEAD(unmaps_to_do);
226 static long list_size;
228 static void domain_remove_dev_info(struct dmar_domain *domain);
231 static int __initdata dmar_map_gfx = 1;
232 static int dmar_forcedac;
233 static int intel_iommu_strict;
235 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
236 static DEFINE_SPINLOCK(device_domain_lock);
237 static LIST_HEAD(device_domain_list);
239 static int __init intel_iommu_setup(char *str)
244 if (!strncmp(str, "off", 3)) {
246 printk(KERN_INFO"Intel-IOMMU: disabled\n");
247 } else if (!strncmp(str, "igfx_off", 8)) {
250 "Intel-IOMMU: disable GFX device mapping\n");
251 } else if (!strncmp(str, "forcedac", 8)) {
253 "Intel-IOMMU: Forcing DAC for PCI devices\n");
255 } else if (!strncmp(str, "strict", 6)) {
257 "Intel-IOMMU: disable batched IOTLB flush\n");
258 intel_iommu_strict = 1;
261 str += strcspn(str, ",");
267 __setup("intel_iommu=", intel_iommu_setup);
269 static struct kmem_cache *iommu_domain_cache;
270 static struct kmem_cache *iommu_devinfo_cache;
271 static struct kmem_cache *iommu_iova_cache;
273 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
278 /* trying to avoid low memory issues */
279 flags = current->flags & PF_MEMALLOC;
280 current->flags |= PF_MEMALLOC;
281 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
282 current->flags &= (~PF_MEMALLOC | flags);
287 static inline void *alloc_pgtable_page(void)
292 /* trying to avoid low memory issues */
293 flags = current->flags & PF_MEMALLOC;
294 current->flags |= PF_MEMALLOC;
295 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
296 current->flags &= (~PF_MEMALLOC | flags);
300 static inline void free_pgtable_page(void *vaddr)
302 free_page((unsigned long)vaddr);
305 static inline void *alloc_domain_mem(void)
307 return iommu_kmem_cache_alloc(iommu_domain_cache);
310 static void free_domain_mem(void *vaddr)
312 kmem_cache_free(iommu_domain_cache, vaddr);
315 static inline void * alloc_devinfo_mem(void)
317 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
320 static inline void free_devinfo_mem(void *vaddr)
322 kmem_cache_free(iommu_devinfo_cache, vaddr);
325 struct iova *alloc_iova_mem(void)
327 return iommu_kmem_cache_alloc(iommu_iova_cache);
330 void free_iova_mem(struct iova *iova)
332 kmem_cache_free(iommu_iova_cache, iova);
335 /* Gets context entry for a given bus and devfn */
336 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
339 struct root_entry *root;
340 struct context_entry *context;
341 unsigned long phy_addr;
344 spin_lock_irqsave(&iommu->lock, flags);
345 root = &iommu->root_entry[bus];
346 context = get_context_addr_from_root(root);
348 context = (struct context_entry *)alloc_pgtable_page();
350 spin_unlock_irqrestore(&iommu->lock, flags);
353 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
354 phy_addr = virt_to_phys((void *)context);
355 set_root_value(root, phy_addr);
356 set_root_present(root);
357 __iommu_flush_cache(iommu, root, sizeof(*root));
359 spin_unlock_irqrestore(&iommu->lock, flags);
360 return &context[devfn];
363 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
365 struct root_entry *root;
366 struct context_entry *context;
370 spin_lock_irqsave(&iommu->lock, flags);
371 root = &iommu->root_entry[bus];
372 context = get_context_addr_from_root(root);
377 ret = context_present(&context[devfn]);
379 spin_unlock_irqrestore(&iommu->lock, flags);
383 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
385 struct root_entry *root;
386 struct context_entry *context;
389 spin_lock_irqsave(&iommu->lock, flags);
390 root = &iommu->root_entry[bus];
391 context = get_context_addr_from_root(root);
393 context_clear_entry(&context[devfn]);
394 __iommu_flush_cache(iommu, &context[devfn], \
397 spin_unlock_irqrestore(&iommu->lock, flags);
400 static void free_context_table(struct intel_iommu *iommu)
402 struct root_entry *root;
405 struct context_entry *context;
407 spin_lock_irqsave(&iommu->lock, flags);
408 if (!iommu->root_entry) {
411 for (i = 0; i < ROOT_ENTRY_NR; i++) {
412 root = &iommu->root_entry[i];
413 context = get_context_addr_from_root(root);
415 free_pgtable_page(context);
417 free_pgtable_page(iommu->root_entry);
418 iommu->root_entry = NULL;
420 spin_unlock_irqrestore(&iommu->lock, flags);
423 /* page table handling */
424 #define LEVEL_STRIDE (9)
425 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
427 static inline int agaw_to_level(int agaw)
432 static inline int agaw_to_width(int agaw)
434 return 30 + agaw * LEVEL_STRIDE;
438 static inline int width_to_agaw(int width)
440 return (width - 30) / LEVEL_STRIDE;
443 static inline unsigned int level_to_offset_bits(int level)
445 return (12 + (level - 1) * LEVEL_STRIDE);
448 static inline int address_level_offset(u64 addr, int level)
450 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
453 static inline u64 level_mask(int level)
455 return ((u64)-1 << level_to_offset_bits(level));
458 static inline u64 level_size(int level)
460 return ((u64)1 << level_to_offset_bits(level));
463 static inline u64 align_to_level(u64 addr, int level)
465 return ((addr + level_size(level) - 1) & level_mask(level));
468 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
470 int addr_width = agaw_to_width(domain->agaw);
471 struct dma_pte *parent, *pte = NULL;
472 int level = agaw_to_level(domain->agaw);
476 BUG_ON(!domain->pgd);
478 addr &= (((u64)1) << addr_width) - 1;
479 parent = domain->pgd;
481 spin_lock_irqsave(&domain->mapping_lock, flags);
485 offset = address_level_offset(addr, level);
486 pte = &parent[offset];
490 if (!dma_pte_present(*pte)) {
491 tmp_page = alloc_pgtable_page();
494 spin_unlock_irqrestore(&domain->mapping_lock,
498 __iommu_flush_cache(domain->iommu, tmp_page,
500 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
502 * high level table always sets r/w, last level page
503 * table control read/write
505 dma_set_pte_readable(*pte);
506 dma_set_pte_writable(*pte);
507 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
509 parent = phys_to_virt(dma_pte_addr(*pte));
513 spin_unlock_irqrestore(&domain->mapping_lock, flags);
517 /* return address's pte at specific level */
518 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
521 struct dma_pte *parent, *pte = NULL;
522 int total = agaw_to_level(domain->agaw);
525 parent = domain->pgd;
526 while (level <= total) {
527 offset = address_level_offset(addr, total);
528 pte = &parent[offset];
532 if (!dma_pte_present(*pte))
534 parent = phys_to_virt(dma_pte_addr(*pte));
540 /* clear one page's page table */
541 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
543 struct dma_pte *pte = NULL;
545 /* get last level pte */
546 pte = dma_addr_level_pte(domain, addr, 1);
550 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
554 /* clear last level pte, a tlb flush should be followed */
555 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
557 int addr_width = agaw_to_width(domain->agaw);
559 start &= (((u64)1) << addr_width) - 1;
560 end &= (((u64)1) << addr_width) - 1;
561 /* in case it's partial page */
562 start = PAGE_ALIGN(start);
565 /* we don't need lock here, nobody else touches the iova range */
566 while (start < end) {
567 dma_pte_clear_one(domain, start);
568 start += VTD_PAGE_SIZE;
572 /* free page table pages. last level pte should already be cleared */
573 static void dma_pte_free_pagetable(struct dmar_domain *domain,
576 int addr_width = agaw_to_width(domain->agaw);
578 int total = agaw_to_level(domain->agaw);
582 start &= (((u64)1) << addr_width) - 1;
583 end &= (((u64)1) << addr_width) - 1;
585 /* we don't need lock here, nobody else touches the iova range */
587 while (level <= total) {
588 tmp = align_to_level(start, level);
589 if (tmp >= end || (tmp + level_size(level) > end))
593 pte = dma_addr_level_pte(domain, tmp, level);
596 phys_to_virt(dma_pte_addr(*pte)));
598 __iommu_flush_cache(domain->iommu,
601 tmp += level_size(level);
606 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
607 free_pgtable_page(domain->pgd);
613 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
615 struct root_entry *root;
618 root = (struct root_entry *)alloc_pgtable_page();
622 __iommu_flush_cache(iommu, root, ROOT_SIZE);
624 spin_lock_irqsave(&iommu->lock, flags);
625 iommu->root_entry = root;
626 spin_unlock_irqrestore(&iommu->lock, flags);
631 static void iommu_set_root_entry(struct intel_iommu *iommu)
637 addr = iommu->root_entry;
639 spin_lock_irqsave(&iommu->register_lock, flag);
640 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
642 cmd = iommu->gcmd | DMA_GCMD_SRTP;
643 writel(cmd, iommu->reg + DMAR_GCMD_REG);
645 /* Make sure hardware complete it */
646 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
647 readl, (sts & DMA_GSTS_RTPS), sts);
649 spin_unlock_irqrestore(&iommu->register_lock, flag);
652 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
657 if (!cap_rwbf(iommu->cap))
659 val = iommu->gcmd | DMA_GCMD_WBF;
661 spin_lock_irqsave(&iommu->register_lock, flag);
662 writel(val, iommu->reg + DMAR_GCMD_REG);
664 /* Make sure hardware complete it */
665 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
666 readl, (!(val & DMA_GSTS_WBFS)), val);
668 spin_unlock_irqrestore(&iommu->register_lock, flag);
671 /* return value determine if we need a write buffer flush */
672 static int __iommu_flush_context(struct intel_iommu *iommu,
673 u16 did, u16 source_id, u8 function_mask, u64 type,
674 int non_present_entry_flush)
680 * In the non-present entry flush case, if hardware doesn't cache
681 * non-present entry we do nothing and if hardware cache non-present
682 * entry, we flush entries of domain 0 (the domain id is used to cache
683 * any non-present entries)
685 if (non_present_entry_flush) {
686 if (!cap_caching_mode(iommu->cap))
693 case DMA_CCMD_GLOBAL_INVL:
694 val = DMA_CCMD_GLOBAL_INVL;
696 case DMA_CCMD_DOMAIN_INVL:
697 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
699 case DMA_CCMD_DEVICE_INVL:
700 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
701 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
708 spin_lock_irqsave(&iommu->register_lock, flag);
709 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
711 /* Make sure hardware complete it */
712 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
713 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
715 spin_unlock_irqrestore(&iommu->register_lock, flag);
717 /* flush context entry will implicitly flush write buffer */
721 /* return value determine if we need a write buffer flush */
722 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
723 u64 addr, unsigned int size_order, u64 type,
724 int non_present_entry_flush)
726 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
727 u64 val = 0, val_iva = 0;
731 * In the non-present entry flush case, if hardware doesn't cache
732 * non-present entry we do nothing and if hardware cache non-present
733 * entry, we flush entries of domain 0 (the domain id is used to cache
734 * any non-present entries)
736 if (non_present_entry_flush) {
737 if (!cap_caching_mode(iommu->cap))
744 case DMA_TLB_GLOBAL_FLUSH:
745 /* global flush doesn't need set IVA_REG */
746 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
748 case DMA_TLB_DSI_FLUSH:
749 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
751 case DMA_TLB_PSI_FLUSH:
752 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
753 /* Note: always flush non-leaf currently */
754 val_iva = size_order | addr;
759 /* Note: set drain read/write */
762 * This is probably to be super secure.. Looks like we can
763 * ignore it without any impact.
765 if (cap_read_drain(iommu->cap))
766 val |= DMA_TLB_READ_DRAIN;
768 if (cap_write_drain(iommu->cap))
769 val |= DMA_TLB_WRITE_DRAIN;
771 spin_lock_irqsave(&iommu->register_lock, flag);
772 /* Note: Only uses first TLB reg currently */
774 dmar_writeq(iommu->reg + tlb_offset, val_iva);
775 dmar_writeq(iommu->reg + tlb_offset + 8, val);
777 /* Make sure hardware complete it */
778 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
779 dmar_readq, (!(val & DMA_TLB_IVT)), val);
781 spin_unlock_irqrestore(&iommu->register_lock, flag);
783 /* check IOTLB invalidation granularity */
784 if (DMA_TLB_IAIG(val) == 0)
785 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
786 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
787 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
788 (unsigned long long)DMA_TLB_IIRG(type),
789 (unsigned long long)DMA_TLB_IAIG(val));
790 /* flush iotlb entry will implicitly flush write buffer */
794 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
795 u64 addr, unsigned int pages, int non_present_entry_flush)
799 BUG_ON(addr & (~VTD_PAGE_MASK));
802 /* Fallback to domain selective flush if no PSI support */
803 if (!cap_pgsel_inv(iommu->cap))
804 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
806 non_present_entry_flush);
809 * PSI requires page size to be 2 ^ x, and the base address is naturally
810 * aligned to the size
812 mask = ilog2(__roundup_pow_of_two(pages));
813 /* Fallback to domain selective flush if size is too big */
814 if (mask > cap_max_amask_val(iommu->cap))
815 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
816 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
818 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
820 non_present_entry_flush);
823 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
828 spin_lock_irqsave(&iommu->register_lock, flags);
829 pmen = readl(iommu->reg + DMAR_PMEN_REG);
830 pmen &= ~DMA_PMEN_EPM;
831 writel(pmen, iommu->reg + DMAR_PMEN_REG);
833 /* wait for the protected region status bit to clear */
834 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
835 readl, !(pmen & DMA_PMEN_PRS), pmen);
837 spin_unlock_irqrestore(&iommu->register_lock, flags);
840 static int iommu_enable_translation(struct intel_iommu *iommu)
845 spin_lock_irqsave(&iommu->register_lock, flags);
846 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
848 /* Make sure hardware complete it */
849 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
850 readl, (sts & DMA_GSTS_TES), sts);
852 iommu->gcmd |= DMA_GCMD_TE;
853 spin_unlock_irqrestore(&iommu->register_lock, flags);
857 static int iommu_disable_translation(struct intel_iommu *iommu)
862 spin_lock_irqsave(&iommu->register_lock, flag);
863 iommu->gcmd &= ~DMA_GCMD_TE;
864 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
866 /* Make sure hardware complete it */
867 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
868 readl, (!(sts & DMA_GSTS_TES)), sts);
870 spin_unlock_irqrestore(&iommu->register_lock, flag);
874 /* iommu interrupt handling. Most stuff are MSI-like. */
876 static const char *fault_reason_strings[] =
879 "Present bit in root entry is clear",
880 "Present bit in context entry is clear",
881 "Invalid context entry",
882 "Access beyond MGAW",
883 "PTE Write access is not set",
884 "PTE Read access is not set",
885 "Next page table ptr is invalid",
886 "Root table address invalid",
887 "Context table ptr is invalid",
888 "non-zero reserved fields in RTP",
889 "non-zero reserved fields in CTP",
890 "non-zero reserved fields in PTE",
892 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
894 const char *dmar_get_fault_reason(u8 fault_reason)
896 if (fault_reason > MAX_FAULT_REASON_IDX)
899 return fault_reason_strings[fault_reason];
902 void dmar_msi_unmask(unsigned int irq)
904 struct intel_iommu *iommu = get_irq_data(irq);
908 spin_lock_irqsave(&iommu->register_lock, flag);
909 writel(0, iommu->reg + DMAR_FECTL_REG);
910 /* Read a reg to force flush the post write */
911 readl(iommu->reg + DMAR_FECTL_REG);
912 spin_unlock_irqrestore(&iommu->register_lock, flag);
915 void dmar_msi_mask(unsigned int irq)
918 struct intel_iommu *iommu = get_irq_data(irq);
921 spin_lock_irqsave(&iommu->register_lock, flag);
922 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
923 /* Read a reg to force flush the post write */
924 readl(iommu->reg + DMAR_FECTL_REG);
925 spin_unlock_irqrestore(&iommu->register_lock, flag);
928 void dmar_msi_write(int irq, struct msi_msg *msg)
930 struct intel_iommu *iommu = get_irq_data(irq);
933 spin_lock_irqsave(&iommu->register_lock, flag);
934 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
935 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
936 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
937 spin_unlock_irqrestore(&iommu->register_lock, flag);
940 void dmar_msi_read(int irq, struct msi_msg *msg)
942 struct intel_iommu *iommu = get_irq_data(irq);
945 spin_lock_irqsave(&iommu->register_lock, flag);
946 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
947 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
948 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
949 spin_unlock_irqrestore(&iommu->register_lock, flag);
952 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
953 u8 fault_reason, u16 source_id, unsigned long long addr)
957 reason = dmar_get_fault_reason(fault_reason);
960 "DMAR:[%s] Request device [%02x:%02x.%d] "
962 "DMAR:[fault reason %02d] %s\n",
963 (type ? "DMA Read" : "DMA Write"),
964 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
965 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
969 #define PRIMARY_FAULT_REG_LEN (16)
970 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
972 struct intel_iommu *iommu = dev_id;
973 int reg, fault_index;
977 spin_lock_irqsave(&iommu->register_lock, flag);
978 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
980 /* TBD: ignore advanced fault log currently */
981 if (!(fault_status & DMA_FSTS_PPF))
984 fault_index = dma_fsts_fault_record_index(fault_status);
985 reg = cap_fault_reg_offset(iommu->cap);
993 /* highest 32 bits */
994 data = readl(iommu->reg + reg +
995 fault_index * PRIMARY_FAULT_REG_LEN + 12);
996 if (!(data & DMA_FRCD_F))
999 fault_reason = dma_frcd_fault_reason(data);
1000 type = dma_frcd_type(data);
1002 data = readl(iommu->reg + reg +
1003 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1004 source_id = dma_frcd_source_id(data);
1006 guest_addr = dmar_readq(iommu->reg + reg +
1007 fault_index * PRIMARY_FAULT_REG_LEN);
1008 guest_addr = dma_frcd_page_addr(guest_addr);
1009 /* clear the fault */
1010 writel(DMA_FRCD_F, iommu->reg + reg +
1011 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1013 spin_unlock_irqrestore(&iommu->register_lock, flag);
1015 iommu_page_fault_do_one(iommu, type, fault_reason,
1016 source_id, guest_addr);
1019 if (fault_index > cap_num_fault_regs(iommu->cap))
1021 spin_lock_irqsave(&iommu->register_lock, flag);
1024 /* clear primary fault overflow */
1025 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1026 if (fault_status & DMA_FSTS_PFO)
1027 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
1029 spin_unlock_irqrestore(&iommu->register_lock, flag);
1033 int dmar_set_interrupt(struct intel_iommu *iommu)
1039 printk(KERN_ERR "IOMMU: no free vectors\n");
1043 set_irq_data(irq, iommu);
1046 ret = arch_setup_dmar_msi(irq);
1048 set_irq_data(irq, NULL);
1054 /* Force fault register is cleared */
1055 iommu_page_fault(irq, iommu);
1057 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
1059 printk(KERN_ERR "IOMMU: can't request irq\n");
1063 static int iommu_init_domains(struct intel_iommu *iommu)
1065 unsigned long ndomains;
1066 unsigned long nlongs;
1068 ndomains = cap_ndoms(iommu->cap);
1069 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1070 nlongs = BITS_TO_LONGS(ndomains);
1072 /* TBD: there might be 64K domains,
1073 * consider other allocation for future chip
1075 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1076 if (!iommu->domain_ids) {
1077 printk(KERN_ERR "Allocating domain id array failed\n");
1080 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1082 if (!iommu->domains) {
1083 printk(KERN_ERR "Allocating domain array failed\n");
1084 kfree(iommu->domain_ids);
1088 spin_lock_init(&iommu->lock);
1091 * if Caching mode is set, then invalid translations are tagged
1092 * with domainid 0. Hence we need to pre-allocate it.
1094 if (cap_caching_mode(iommu->cap))
1095 set_bit(0, iommu->domain_ids);
1100 static void domain_exit(struct dmar_domain *domain);
1102 void free_dmar_iommu(struct intel_iommu *iommu)
1104 struct dmar_domain *domain;
1107 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1108 for (; i < cap_ndoms(iommu->cap); ) {
1109 domain = iommu->domains[i];
1110 clear_bit(i, iommu->domain_ids);
1111 domain_exit(domain);
1112 i = find_next_bit(iommu->domain_ids,
1113 cap_ndoms(iommu->cap), i+1);
1116 if (iommu->gcmd & DMA_GCMD_TE)
1117 iommu_disable_translation(iommu);
1120 set_irq_data(iommu->irq, NULL);
1121 /* This will mask the irq */
1122 free_irq(iommu->irq, iommu);
1123 destroy_irq(iommu->irq);
1126 kfree(iommu->domains);
1127 kfree(iommu->domain_ids);
1129 /* free context mapping */
1130 free_context_table(iommu);
1133 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1136 unsigned long ndomains;
1137 struct dmar_domain *domain;
1138 unsigned long flags;
1140 domain = alloc_domain_mem();
1144 ndomains = cap_ndoms(iommu->cap);
1146 spin_lock_irqsave(&iommu->lock, flags);
1147 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1148 if (num >= ndomains) {
1149 spin_unlock_irqrestore(&iommu->lock, flags);
1150 free_domain_mem(domain);
1151 printk(KERN_ERR "IOMMU: no free domain ids\n");
1155 set_bit(num, iommu->domain_ids);
1157 domain->iommu = iommu;
1158 iommu->domains[num] = domain;
1159 spin_unlock_irqrestore(&iommu->lock, flags);
1164 static void iommu_free_domain(struct dmar_domain *domain)
1166 unsigned long flags;
1168 spin_lock_irqsave(&domain->iommu->lock, flags);
1169 clear_bit(domain->id, domain->iommu->domain_ids);
1170 spin_unlock_irqrestore(&domain->iommu->lock, flags);
1173 static struct iova_domain reserved_iova_list;
1174 static struct lock_class_key reserved_alloc_key;
1175 static struct lock_class_key reserved_rbtree_key;
1177 static void dmar_init_reserved_ranges(void)
1179 struct pci_dev *pdev = NULL;
1184 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1186 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1187 &reserved_alloc_key);
1188 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1189 &reserved_rbtree_key);
1191 /* IOAPIC ranges shouldn't be accessed by DMA */
1192 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1193 IOVA_PFN(IOAPIC_RANGE_END));
1195 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1197 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1198 for_each_pci_dev(pdev) {
1201 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1202 r = &pdev->resource[i];
1203 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1207 size = r->end - addr;
1208 size = PAGE_ALIGN(size);
1209 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1210 IOVA_PFN(size + addr) - 1);
1212 printk(KERN_ERR "Reserve iova failed\n");
1218 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1220 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1223 static inline int guestwidth_to_adjustwidth(int gaw)
1226 int r = (gaw - 12) % 9;
1237 static int domain_init(struct dmar_domain *domain, int guest_width)
1239 struct intel_iommu *iommu;
1240 int adjust_width, agaw;
1241 unsigned long sagaw;
1243 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1244 spin_lock_init(&domain->mapping_lock);
1246 domain_reserve_special_ranges(domain);
1248 /* calculate AGAW */
1249 iommu = domain->iommu;
1250 if (guest_width > cap_mgaw(iommu->cap))
1251 guest_width = cap_mgaw(iommu->cap);
1252 domain->gaw = guest_width;
1253 adjust_width = guestwidth_to_adjustwidth(guest_width);
1254 agaw = width_to_agaw(adjust_width);
1255 sagaw = cap_sagaw(iommu->cap);
1256 if (!test_bit(agaw, &sagaw)) {
1257 /* hardware doesn't support it, choose a bigger one */
1258 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1259 agaw = find_next_bit(&sagaw, 5, agaw);
1263 domain->agaw = agaw;
1264 INIT_LIST_HEAD(&domain->devices);
1266 /* always allocate the top pgd */
1267 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1270 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1274 static void domain_exit(struct dmar_domain *domain)
1278 /* Domain 0 is reserved, so dont process it */
1282 domain_remove_dev_info(domain);
1284 put_iova_domain(&domain->iovad);
1285 end = DOMAIN_MAX_ADDR(domain->gaw);
1286 end = end & (~PAGE_MASK);
1289 dma_pte_clear_range(domain, 0, end);
1291 /* free page tables */
1292 dma_pte_free_pagetable(domain, 0, end);
1294 iommu_free_domain(domain);
1295 free_domain_mem(domain);
1298 static int domain_context_mapping_one(struct dmar_domain *domain,
1301 struct context_entry *context;
1302 struct intel_iommu *iommu = domain->iommu;
1303 unsigned long flags;
1305 pr_debug("Set context mapping for %02x:%02x.%d\n",
1306 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1307 BUG_ON(!domain->pgd);
1308 context = device_to_context_entry(iommu, bus, devfn);
1311 spin_lock_irqsave(&iommu->lock, flags);
1312 if (context_present(context)) {
1313 spin_unlock_irqrestore(&iommu->lock, flags);
1317 context_set_domain_id(context, domain->id);
1318 context_set_address_width(context, domain->agaw);
1319 context_set_address_root(context, virt_to_phys(domain->pgd));
1320 context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1321 context_set_fault_enable(context);
1322 context_set_present(context);
1323 __iommu_flush_cache(iommu, context, sizeof(*context));
1325 /* it's a non-present to present mapping */
1326 if (iommu->flush.flush_context(iommu, domain->id,
1327 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1328 DMA_CCMD_DEVICE_INVL, 1))
1329 iommu_flush_write_buffer(iommu);
1331 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1333 spin_unlock_irqrestore(&iommu->lock, flags);
1338 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1341 struct pci_dev *tmp, *parent;
1343 ret = domain_context_mapping_one(domain, pdev->bus->number,
1348 /* dependent device mapping */
1349 tmp = pci_find_upstream_pcie_bridge(pdev);
1352 /* Secondary interface's bus number and devfn 0 */
1353 parent = pdev->bus->self;
1354 while (parent != tmp) {
1355 ret = domain_context_mapping_one(domain, parent->bus->number,
1359 parent = parent->bus->self;
1361 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1362 return domain_context_mapping_one(domain,
1363 tmp->subordinate->number, 0);
1364 else /* this is a legacy PCI bridge */
1365 return domain_context_mapping_one(domain,
1366 tmp->bus->number, tmp->devfn);
1369 static int domain_context_mapped(struct dmar_domain *domain,
1370 struct pci_dev *pdev)
1373 struct pci_dev *tmp, *parent;
1375 ret = device_context_mapped(domain->iommu,
1376 pdev->bus->number, pdev->devfn);
1379 /* dependent device mapping */
1380 tmp = pci_find_upstream_pcie_bridge(pdev);
1383 /* Secondary interface's bus number and devfn 0 */
1384 parent = pdev->bus->self;
1385 while (parent != tmp) {
1386 ret = device_context_mapped(domain->iommu, parent->bus->number,
1390 parent = parent->bus->self;
1393 return device_context_mapped(domain->iommu,
1394 tmp->subordinate->number, 0);
1396 return device_context_mapped(domain->iommu,
1397 tmp->bus->number, tmp->devfn);
1401 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1402 u64 hpa, size_t size, int prot)
1404 u64 start_pfn, end_pfn;
1405 struct dma_pte *pte;
1407 int addr_width = agaw_to_width(domain->agaw);
1409 hpa &= (((u64)1) << addr_width) - 1;
1411 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1414 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1415 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1417 while (start_pfn < end_pfn) {
1418 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1421 /* We don't need lock here, nobody else
1422 * touches the iova range
1424 BUG_ON(dma_pte_addr(*pte));
1425 dma_set_pte_addr(*pte, start_pfn << VTD_PAGE_SHIFT);
1426 dma_set_pte_prot(*pte, prot);
1427 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1434 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1436 clear_context_table(domain->iommu, bus, devfn);
1437 domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1438 DMA_CCMD_GLOBAL_INVL, 0);
1439 domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1440 DMA_TLB_GLOBAL_FLUSH, 0);
1443 static void domain_remove_dev_info(struct dmar_domain *domain)
1445 struct device_domain_info *info;
1446 unsigned long flags;
1448 spin_lock_irqsave(&device_domain_lock, flags);
1449 while (!list_empty(&domain->devices)) {
1450 info = list_entry(domain->devices.next,
1451 struct device_domain_info, link);
1452 list_del(&info->link);
1453 list_del(&info->global);
1455 info->dev->dev.archdata.iommu = NULL;
1456 spin_unlock_irqrestore(&device_domain_lock, flags);
1458 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1459 free_devinfo_mem(info);
1461 spin_lock_irqsave(&device_domain_lock, flags);
1463 spin_unlock_irqrestore(&device_domain_lock, flags);
1468 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1470 static struct dmar_domain *
1471 find_domain(struct pci_dev *pdev)
1473 struct device_domain_info *info;
1475 /* No lock here, assumes no domain exit in normal case */
1476 info = pdev->dev.archdata.iommu;
1478 return info->domain;
1482 /* domain is initialized */
1483 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1485 struct dmar_domain *domain, *found = NULL;
1486 struct intel_iommu *iommu;
1487 struct dmar_drhd_unit *drhd;
1488 struct device_domain_info *info, *tmp;
1489 struct pci_dev *dev_tmp;
1490 unsigned long flags;
1491 int bus = 0, devfn = 0;
1493 domain = find_domain(pdev);
1497 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1499 if (dev_tmp->is_pcie) {
1500 bus = dev_tmp->subordinate->number;
1503 bus = dev_tmp->bus->number;
1504 devfn = dev_tmp->devfn;
1506 spin_lock_irqsave(&device_domain_lock, flags);
1507 list_for_each_entry(info, &device_domain_list, global) {
1508 if (info->bus == bus && info->devfn == devfn) {
1509 found = info->domain;
1513 spin_unlock_irqrestore(&device_domain_lock, flags);
1514 /* pcie-pci bridge already has a domain, uses it */
1521 /* Allocate new domain for the device */
1522 drhd = dmar_find_matched_drhd_unit(pdev);
1524 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1528 iommu = drhd->iommu;
1530 domain = iommu_alloc_domain(iommu);
1534 if (domain_init(domain, gaw)) {
1535 domain_exit(domain);
1539 /* register pcie-to-pci device */
1541 info = alloc_devinfo_mem();
1543 domain_exit(domain);
1547 info->devfn = devfn;
1549 info->domain = domain;
1550 /* This domain is shared by devices under p2p bridge */
1551 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1553 /* pcie-to-pci bridge already has a domain, uses it */
1555 spin_lock_irqsave(&device_domain_lock, flags);
1556 list_for_each_entry(tmp, &device_domain_list, global) {
1557 if (tmp->bus == bus && tmp->devfn == devfn) {
1558 found = tmp->domain;
1563 free_devinfo_mem(info);
1564 domain_exit(domain);
1567 list_add(&info->link, &domain->devices);
1568 list_add(&info->global, &device_domain_list);
1570 spin_unlock_irqrestore(&device_domain_lock, flags);
1574 info = alloc_devinfo_mem();
1577 info->bus = pdev->bus->number;
1578 info->devfn = pdev->devfn;
1580 info->domain = domain;
1581 spin_lock_irqsave(&device_domain_lock, flags);
1582 /* somebody is fast */
1583 found = find_domain(pdev);
1584 if (found != NULL) {
1585 spin_unlock_irqrestore(&device_domain_lock, flags);
1586 if (found != domain) {
1587 domain_exit(domain);
1590 free_devinfo_mem(info);
1593 list_add(&info->link, &domain->devices);
1594 list_add(&info->global, &device_domain_list);
1595 pdev->dev.archdata.iommu = info;
1596 spin_unlock_irqrestore(&device_domain_lock, flags);
1599 /* recheck it here, maybe others set it */
1600 return find_domain(pdev);
1603 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1604 unsigned long long start,
1605 unsigned long long end)
1607 struct dmar_domain *domain;
1609 unsigned long long base;
1613 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1614 pci_name(pdev), start, end);
1615 /* page table init */
1616 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1620 /* The address might not be aligned */
1621 base = start & PAGE_MASK;
1623 size = PAGE_ALIGN(size);
1624 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1625 IOVA_PFN(base + size) - 1)) {
1626 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1631 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1632 size, base, pci_name(pdev));
1634 * RMRR range might have overlap with physical memory range,
1637 dma_pte_clear_range(domain, base, base + size);
1639 ret = domain_page_mapping(domain, base, base, size,
1640 DMA_PTE_READ|DMA_PTE_WRITE);
1644 /* context entry init */
1645 ret = domain_context_mapping(domain, pdev);
1649 domain_exit(domain);
1654 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1655 struct pci_dev *pdev)
1657 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1659 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1660 rmrr->end_address + 1);
1663 #ifdef CONFIG_DMAR_GFX_WA
1664 struct iommu_prepare_data {
1665 struct pci_dev *pdev;
1669 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1670 unsigned long end_pfn, void *datax)
1672 struct iommu_prepare_data *data;
1674 data = (struct iommu_prepare_data *)datax;
1676 data->ret = iommu_prepare_identity_map(data->pdev,
1677 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1682 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1685 struct iommu_prepare_data data;
1690 for_each_online_node(nid) {
1691 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1698 static void __init iommu_prepare_gfx_mapping(void)
1700 struct pci_dev *pdev = NULL;
1703 for_each_pci_dev(pdev) {
1704 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1705 !IS_GFX_DEVICE(pdev))
1707 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1709 ret = iommu_prepare_with_active_regions(pdev);
1711 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1714 #else /* !CONFIG_DMAR_GFX_WA */
1715 static inline void iommu_prepare_gfx_mapping(void)
1721 #ifdef CONFIG_DMAR_FLOPPY_WA
1722 static inline void iommu_prepare_isa(void)
1724 struct pci_dev *pdev;
1727 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1731 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1732 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1735 printk("IOMMU: Failed to create 0-64M identity map, "
1736 "floppy might not work\n");
1740 static inline void iommu_prepare_isa(void)
1744 #endif /* !CONFIG_DMAR_FLPY_WA */
1746 static int __init init_dmars(void)
1748 struct dmar_drhd_unit *drhd;
1749 struct dmar_rmrr_unit *rmrr;
1750 struct pci_dev *pdev;
1751 struct intel_iommu *iommu;
1752 int i, ret, unit = 0;
1757 * initialize and program root entry to not present
1760 for_each_drhd_unit(drhd) {
1763 * lock not needed as this is only incremented in the single
1764 * threaded kernel __init code path all other access are read
1769 deferred_flush = kzalloc(g_num_of_iommus *
1770 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1771 if (!deferred_flush) {
1776 for_each_drhd_unit(drhd) {
1780 iommu = drhd->iommu;
1782 ret = iommu_init_domains(iommu);
1788 * we could share the same root & context tables
1789 * amoung all IOMMU's. Need to Split it later.
1791 ret = iommu_alloc_root_entry(iommu);
1793 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1798 for_each_drhd_unit(drhd) {
1802 iommu = drhd->iommu;
1803 if (dmar_enable_qi(iommu)) {
1805 * Queued Invalidate not enabled, use Register Based
1808 iommu->flush.flush_context = __iommu_flush_context;
1809 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1810 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1812 (unsigned long long)drhd->reg_base_addr);
1814 iommu->flush.flush_context = qi_flush_context;
1815 iommu->flush.flush_iotlb = qi_flush_iotlb;
1816 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1818 (unsigned long long)drhd->reg_base_addr);
1824 * for each dev attached to rmrr
1826 * locate drhd for dev, alloc domain for dev
1827 * allocate free domain
1828 * allocate page table entries for rmrr
1829 * if context not allocated for bus
1830 * allocate and init context
1831 * set present in root table for this bus
1832 * init context with domain, translation etc
1836 for_each_rmrr_units(rmrr) {
1837 for (i = 0; i < rmrr->devices_cnt; i++) {
1838 pdev = rmrr->devices[i];
1839 /* some BIOS lists non-exist devices in DMAR table */
1842 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1845 "IOMMU: mapping reserved region failed\n");
1849 iommu_prepare_gfx_mapping();
1851 iommu_prepare_isa();
1856 * global invalidate context cache
1857 * global invalidate iotlb
1858 * enable translation
1860 for_each_drhd_unit(drhd) {
1863 iommu = drhd->iommu;
1864 sprintf (iommu->name, "dmar%d", unit++);
1866 iommu_flush_write_buffer(iommu);
1868 ret = dmar_set_interrupt(iommu);
1872 iommu_set_root_entry(iommu);
1874 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1876 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1878 iommu_disable_protect_mem_regions(iommu);
1880 ret = iommu_enable_translation(iommu);
1887 for_each_drhd_unit(drhd) {
1890 iommu = drhd->iommu;
1896 static inline u64 aligned_size(u64 host_addr, size_t size)
1899 addr = (host_addr & (~PAGE_MASK)) + size;
1900 return PAGE_ALIGN(addr);
1904 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1908 /* Make sure it's in range */
1909 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1910 if (!size || (IOVA_START_ADDR + size > end))
1913 piova = alloc_iova(&domain->iovad,
1914 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1918 static struct iova *
1919 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1920 size_t size, u64 dma_mask)
1922 struct pci_dev *pdev = to_pci_dev(dev);
1923 struct iova *iova = NULL;
1925 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1926 iova = iommu_alloc_iova(domain, size, dma_mask);
1929 * First try to allocate an io virtual address in
1930 * DMA_32BIT_MASK and if that fails then try allocating
1933 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1935 iova = iommu_alloc_iova(domain, size, dma_mask);
1939 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1946 static struct dmar_domain *
1947 get_valid_domain_for_dev(struct pci_dev *pdev)
1949 struct dmar_domain *domain;
1952 domain = get_domain_for_dev(pdev,
1953 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1956 "Allocating domain for %s failed", pci_name(pdev));
1960 /* make sure context mapping is ok */
1961 if (unlikely(!domain_context_mapped(domain, pdev))) {
1962 ret = domain_context_mapping(domain, pdev);
1965 "Domain context map for %s failed",
1974 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
1975 size_t size, int dir, u64 dma_mask)
1977 struct pci_dev *pdev = to_pci_dev(hwdev);
1978 struct dmar_domain *domain;
1979 phys_addr_t start_paddr;
1984 BUG_ON(dir == DMA_NONE);
1985 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1988 domain = get_valid_domain_for_dev(pdev);
1992 size = aligned_size((u64)paddr, size);
1994 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
1998 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2001 * Check if DMAR supports zero-length reads on write only
2004 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2005 !cap_zlr(domain->iommu->cap))
2006 prot |= DMA_PTE_READ;
2007 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2008 prot |= DMA_PTE_WRITE;
2010 * paddr - (paddr + size) might be partial page, we should map the whole
2011 * page. Note: if two part of one page are separately mapped, we
2012 * might have two guest_addr mapping to the same host paddr, but this
2013 * is not a big problem
2015 ret = domain_page_mapping(domain, start_paddr,
2016 ((u64)paddr) & PAGE_MASK, size, prot);
2020 /* it's a non-present to present mapping */
2021 ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
2022 start_paddr, size >> VTD_PAGE_SHIFT, 1);
2024 iommu_flush_write_buffer(domain->iommu);
2026 return start_paddr + ((u64)paddr & (~PAGE_MASK));
2030 __free_iova(&domain->iovad, iova);
2031 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2032 pci_name(pdev), size, (unsigned long long)paddr, dir);
2036 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
2037 size_t size, int dir)
2039 return __intel_map_single(hwdev, paddr, size, dir,
2040 to_pci_dev(hwdev)->dma_mask);
2043 static void flush_unmaps(void)
2049 /* just flush them all */
2050 for (i = 0; i < g_num_of_iommus; i++) {
2051 if (deferred_flush[i].next) {
2052 struct intel_iommu *iommu =
2053 deferred_flush[i].domain[0]->iommu;
2055 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2056 DMA_TLB_GLOBAL_FLUSH, 0);
2057 for (j = 0; j < deferred_flush[i].next; j++) {
2058 __free_iova(&deferred_flush[i].domain[j]->iovad,
2059 deferred_flush[i].iova[j]);
2061 deferred_flush[i].next = 0;
2068 static void flush_unmaps_timeout(unsigned long data)
2070 unsigned long flags;
2072 spin_lock_irqsave(&async_umap_flush_lock, flags);
2074 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2077 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2079 unsigned long flags;
2082 spin_lock_irqsave(&async_umap_flush_lock, flags);
2083 if (list_size == HIGH_WATER_MARK)
2086 iommu_id = dom->iommu->seq_id;
2088 next = deferred_flush[iommu_id].next;
2089 deferred_flush[iommu_id].domain[next] = dom;
2090 deferred_flush[iommu_id].iova[next] = iova;
2091 deferred_flush[iommu_id].next++;
2094 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2098 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2101 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2104 struct pci_dev *pdev = to_pci_dev(dev);
2105 struct dmar_domain *domain;
2106 unsigned long start_addr;
2109 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2111 domain = find_domain(pdev);
2114 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2118 start_addr = iova->pfn_lo << PAGE_SHIFT;
2119 size = aligned_size((u64)dev_addr, size);
2121 pr_debug("Device %s unmapping: %lx@%llx\n",
2122 pci_name(pdev), size, (unsigned long long)start_addr);
2124 /* clear the whole page */
2125 dma_pte_clear_range(domain, start_addr, start_addr + size);
2126 /* free page tables */
2127 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2128 if (intel_iommu_strict) {
2129 if (iommu_flush_iotlb_psi(domain->iommu,
2130 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2131 iommu_flush_write_buffer(domain->iommu);
2133 __free_iova(&domain->iovad, iova);
2135 add_unmap(domain, iova);
2137 * queue up the release of the unmap to save the 1/6th of the
2138 * cpu used up by the iotlb flush operation...
2143 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2144 dma_addr_t *dma_handle, gfp_t flags)
2149 size = PAGE_ALIGN(size);
2150 order = get_order(size);
2151 flags &= ~(GFP_DMA | GFP_DMA32);
2153 vaddr = (void *)__get_free_pages(flags, order);
2156 memset(vaddr, 0, size);
2158 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2160 hwdev->coherent_dma_mask);
2163 free_pages((unsigned long)vaddr, order);
2167 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2168 dma_addr_t dma_handle)
2172 size = PAGE_ALIGN(size);
2173 order = get_order(size);
2175 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2176 free_pages((unsigned long)vaddr, order);
2179 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2181 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2182 int nelems, int dir)
2185 struct pci_dev *pdev = to_pci_dev(hwdev);
2186 struct dmar_domain *domain;
2187 unsigned long start_addr;
2191 struct scatterlist *sg;
2193 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2196 domain = find_domain(pdev);
2198 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2201 for_each_sg(sglist, sg, nelems, i) {
2202 addr = SG_ENT_VIRT_ADDRESS(sg);
2203 size += aligned_size((u64)addr, sg->length);
2206 start_addr = iova->pfn_lo << PAGE_SHIFT;
2208 /* clear the whole page */
2209 dma_pte_clear_range(domain, start_addr, start_addr + size);
2210 /* free page tables */
2211 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2213 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2214 size >> VTD_PAGE_SHIFT, 0))
2215 iommu_flush_write_buffer(domain->iommu);
2218 __free_iova(&domain->iovad, iova);
2221 static int intel_nontranslate_map_sg(struct device *hddev,
2222 struct scatterlist *sglist, int nelems, int dir)
2225 struct scatterlist *sg;
2227 for_each_sg(sglist, sg, nelems, i) {
2228 BUG_ON(!sg_page(sg));
2229 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2230 sg->dma_length = sg->length;
2235 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2240 struct pci_dev *pdev = to_pci_dev(hwdev);
2241 struct dmar_domain *domain;
2245 struct iova *iova = NULL;
2247 struct scatterlist *sg;
2248 unsigned long start_addr;
2250 BUG_ON(dir == DMA_NONE);
2251 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2252 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2254 domain = get_valid_domain_for_dev(pdev);
2258 for_each_sg(sglist, sg, nelems, i) {
2259 addr = SG_ENT_VIRT_ADDRESS(sg);
2260 addr = (void *)virt_to_phys(addr);
2261 size += aligned_size((u64)addr, sg->length);
2264 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2266 sglist->dma_length = 0;
2271 * Check if DMAR supports zero-length reads on write only
2274 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2275 !cap_zlr(domain->iommu->cap))
2276 prot |= DMA_PTE_READ;
2277 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2278 prot |= DMA_PTE_WRITE;
2280 start_addr = iova->pfn_lo << PAGE_SHIFT;
2282 for_each_sg(sglist, sg, nelems, i) {
2283 addr = SG_ENT_VIRT_ADDRESS(sg);
2284 addr = (void *)virt_to_phys(addr);
2285 size = aligned_size((u64)addr, sg->length);
2286 ret = domain_page_mapping(domain, start_addr + offset,
2287 ((u64)addr) & PAGE_MASK,
2290 /* clear the page */
2291 dma_pte_clear_range(domain, start_addr,
2292 start_addr + offset);
2293 /* free page tables */
2294 dma_pte_free_pagetable(domain, start_addr,
2295 start_addr + offset);
2297 __free_iova(&domain->iovad, iova);
2300 sg->dma_address = start_addr + offset +
2301 ((u64)addr & (~PAGE_MASK));
2302 sg->dma_length = sg->length;
2306 /* it's a non-present to present mapping */
2307 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2308 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2309 iommu_flush_write_buffer(domain->iommu);
2313 static struct dma_mapping_ops intel_dma_ops = {
2314 .alloc_coherent = intel_alloc_coherent,
2315 .free_coherent = intel_free_coherent,
2316 .map_single = intel_map_single,
2317 .unmap_single = intel_unmap_single,
2318 .map_sg = intel_map_sg,
2319 .unmap_sg = intel_unmap_sg,
2322 static inline int iommu_domain_cache_init(void)
2326 iommu_domain_cache = kmem_cache_create("iommu_domain",
2327 sizeof(struct dmar_domain),
2332 if (!iommu_domain_cache) {
2333 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2340 static inline int iommu_devinfo_cache_init(void)
2344 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2345 sizeof(struct device_domain_info),
2349 if (!iommu_devinfo_cache) {
2350 printk(KERN_ERR "Couldn't create devinfo cache\n");
2357 static inline int iommu_iova_cache_init(void)
2361 iommu_iova_cache = kmem_cache_create("iommu_iova",
2362 sizeof(struct iova),
2366 if (!iommu_iova_cache) {
2367 printk(KERN_ERR "Couldn't create iova cache\n");
2374 static int __init iommu_init_mempool(void)
2377 ret = iommu_iova_cache_init();
2381 ret = iommu_domain_cache_init();
2385 ret = iommu_devinfo_cache_init();
2389 kmem_cache_destroy(iommu_domain_cache);
2391 kmem_cache_destroy(iommu_iova_cache);
2396 static void __init iommu_exit_mempool(void)
2398 kmem_cache_destroy(iommu_devinfo_cache);
2399 kmem_cache_destroy(iommu_domain_cache);
2400 kmem_cache_destroy(iommu_iova_cache);
2404 static void __init init_no_remapping_devices(void)
2406 struct dmar_drhd_unit *drhd;
2408 for_each_drhd_unit(drhd) {
2409 if (!drhd->include_all) {
2411 for (i = 0; i < drhd->devices_cnt; i++)
2412 if (drhd->devices[i] != NULL)
2414 /* ignore DMAR unit if no pci devices exist */
2415 if (i == drhd->devices_cnt)
2423 for_each_drhd_unit(drhd) {
2425 if (drhd->ignored || drhd->include_all)
2428 for (i = 0; i < drhd->devices_cnt; i++)
2429 if (drhd->devices[i] &&
2430 !IS_GFX_DEVICE(drhd->devices[i]))
2433 if (i < drhd->devices_cnt)
2436 /* bypass IOMMU if it is just for gfx devices */
2438 for (i = 0; i < drhd->devices_cnt; i++) {
2439 if (!drhd->devices[i])
2441 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2446 int __init intel_iommu_init(void)
2450 if (dmar_table_init())
2453 if (dmar_dev_scope_init())
2457 * Check the need for DMA-remapping initialization now.
2458 * Above initialization will also be used by Interrupt-remapping.
2460 if (no_iommu || swiotlb || dmar_disabled)
2463 iommu_init_mempool();
2464 dmar_init_reserved_ranges();
2466 init_no_remapping_devices();
2470 printk(KERN_ERR "IOMMU: dmar init failed\n");
2471 put_iova_domain(&reserved_iova_list);
2472 iommu_exit_mempool();
2476 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2478 init_timer(&unmap_timer);
2480 dma_ops = &intel_dma_ops;
2484 void intel_iommu_domain_exit(struct dmar_domain *domain)
2488 /* Domain 0 is reserved, so dont process it */
2492 end = DOMAIN_MAX_ADDR(domain->gaw);
2493 end = end & (~VTD_PAGE_MASK);
2496 dma_pte_clear_range(domain, 0, end);
2498 /* free page tables */
2499 dma_pte_free_pagetable(domain, 0, end);
2501 iommu_free_domain(domain);
2502 free_domain_mem(domain);
2504 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2506 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2508 struct dmar_drhd_unit *drhd;
2509 struct dmar_domain *domain;
2510 struct intel_iommu *iommu;
2512 drhd = dmar_find_matched_drhd_unit(pdev);
2514 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2518 iommu = drhd->iommu;
2521 "intel_iommu_domain_alloc: iommu == NULL\n");
2524 domain = iommu_alloc_domain(iommu);
2527 "intel_iommu_domain_alloc: domain == NULL\n");
2530 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2532 "intel_iommu_domain_alloc: domain_init() failed\n");
2533 intel_iommu_domain_exit(domain);
2538 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2540 int intel_iommu_context_mapping(
2541 struct dmar_domain *domain, struct pci_dev *pdev)
2544 rc = domain_context_mapping(domain, pdev);
2547 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2549 int intel_iommu_page_mapping(
2550 struct dmar_domain *domain, dma_addr_t iova,
2551 u64 hpa, size_t size, int prot)
2554 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2557 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2559 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2561 detach_domain_for_dev(domain, bus, devfn);
2563 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2565 struct dmar_domain *
2566 intel_iommu_find_domain(struct pci_dev *pdev)
2568 return find_domain(pdev);
2570 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2572 int intel_iommu_found(void)
2574 return g_num_of_iommus;
2576 EXPORT_SYMBOL_GPL(intel_iommu_found);
2578 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2580 struct dma_pte *pte;
2584 pte = addr_to_dma_pte(domain, iova);
2587 pfn = dma_pte_addr(*pte);
2589 return pfn >> VTD_PAGE_SHIFT;
2591 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);