2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/intel-iommu.h>
38 #include <asm/cacheflush.h>
39 #include <asm/iommu.h>
42 #define ROOT_SIZE VTD_PAGE_SIZE
43 #define CONTEXT_SIZE VTD_PAGE_SIZE
45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48 #define IOAPIC_RANGE_START (0xfee00000)
49 #define IOAPIC_RANGE_END (0xfeefffff)
50 #define IOVA_START_ADDR (0x1000)
52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
57 static void flush_unmaps_timeout(unsigned long data);
59 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
61 #define HIGH_WATER_MARK 250
62 struct deferred_flush_tables {
64 struct iova *iova[HIGH_WATER_MARK];
65 struct dmar_domain *domain[HIGH_WATER_MARK];
68 static struct deferred_flush_tables *deferred_flush;
70 /* bitmap for indexing intel_iommus */
71 static int g_num_of_iommus;
73 static DEFINE_SPINLOCK(async_umap_flush_lock);
74 static LIST_HEAD(unmaps_to_do);
77 static long list_size;
79 static void domain_remove_dev_info(struct dmar_domain *domain);
82 static int __initdata dmar_map_gfx = 1;
83 static int dmar_forcedac;
84 static int intel_iommu_strict;
86 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
87 static DEFINE_SPINLOCK(device_domain_lock);
88 static LIST_HEAD(device_domain_list);
90 static int __init intel_iommu_setup(char *str)
95 if (!strncmp(str, "off", 3)) {
97 printk(KERN_INFO"Intel-IOMMU: disabled\n");
98 } else if (!strncmp(str, "igfx_off", 8)) {
101 "Intel-IOMMU: disable GFX device mapping\n");
102 } else if (!strncmp(str, "forcedac", 8)) {
104 "Intel-IOMMU: Forcing DAC for PCI devices\n");
106 } else if (!strncmp(str, "strict", 6)) {
108 "Intel-IOMMU: disable batched IOTLB flush\n");
109 intel_iommu_strict = 1;
112 str += strcspn(str, ",");
118 __setup("intel_iommu=", intel_iommu_setup);
120 static struct kmem_cache *iommu_domain_cache;
121 static struct kmem_cache *iommu_devinfo_cache;
122 static struct kmem_cache *iommu_iova_cache;
124 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
129 /* trying to avoid low memory issues */
130 flags = current->flags & PF_MEMALLOC;
131 current->flags |= PF_MEMALLOC;
132 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
133 current->flags &= (~PF_MEMALLOC | flags);
138 static inline void *alloc_pgtable_page(void)
143 /* trying to avoid low memory issues */
144 flags = current->flags & PF_MEMALLOC;
145 current->flags |= PF_MEMALLOC;
146 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
147 current->flags &= (~PF_MEMALLOC | flags);
151 static inline void free_pgtable_page(void *vaddr)
153 free_page((unsigned long)vaddr);
156 static inline void *alloc_domain_mem(void)
158 return iommu_kmem_cache_alloc(iommu_domain_cache);
161 static void free_domain_mem(void *vaddr)
163 kmem_cache_free(iommu_domain_cache, vaddr);
166 static inline void * alloc_devinfo_mem(void)
168 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
171 static inline void free_devinfo_mem(void *vaddr)
173 kmem_cache_free(iommu_devinfo_cache, vaddr);
176 struct iova *alloc_iova_mem(void)
178 return iommu_kmem_cache_alloc(iommu_iova_cache);
181 void free_iova_mem(struct iova *iova)
183 kmem_cache_free(iommu_iova_cache, iova);
186 /* Gets context entry for a given bus and devfn */
187 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
190 struct root_entry *root;
191 struct context_entry *context;
192 unsigned long phy_addr;
195 spin_lock_irqsave(&iommu->lock, flags);
196 root = &iommu->root_entry[bus];
197 context = get_context_addr_from_root(root);
199 context = (struct context_entry *)alloc_pgtable_page();
201 spin_unlock_irqrestore(&iommu->lock, flags);
204 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
205 phy_addr = virt_to_phys((void *)context);
206 set_root_value(root, phy_addr);
207 set_root_present(root);
208 __iommu_flush_cache(iommu, root, sizeof(*root));
210 spin_unlock_irqrestore(&iommu->lock, flags);
211 return &context[devfn];
214 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
216 struct root_entry *root;
217 struct context_entry *context;
221 spin_lock_irqsave(&iommu->lock, flags);
222 root = &iommu->root_entry[bus];
223 context = get_context_addr_from_root(root);
228 ret = context_present(context[devfn]);
230 spin_unlock_irqrestore(&iommu->lock, flags);
234 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
236 struct root_entry *root;
237 struct context_entry *context;
240 spin_lock_irqsave(&iommu->lock, flags);
241 root = &iommu->root_entry[bus];
242 context = get_context_addr_from_root(root);
244 context_clear_entry(context[devfn]);
245 __iommu_flush_cache(iommu, &context[devfn], \
248 spin_unlock_irqrestore(&iommu->lock, flags);
251 static void free_context_table(struct intel_iommu *iommu)
253 struct root_entry *root;
256 struct context_entry *context;
258 spin_lock_irqsave(&iommu->lock, flags);
259 if (!iommu->root_entry) {
262 for (i = 0; i < ROOT_ENTRY_NR; i++) {
263 root = &iommu->root_entry[i];
264 context = get_context_addr_from_root(root);
266 free_pgtable_page(context);
268 free_pgtable_page(iommu->root_entry);
269 iommu->root_entry = NULL;
271 spin_unlock_irqrestore(&iommu->lock, flags);
274 /* page table handling */
275 #define LEVEL_STRIDE (9)
276 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
278 static inline int agaw_to_level(int agaw)
283 static inline int agaw_to_width(int agaw)
285 return 30 + agaw * LEVEL_STRIDE;
289 static inline int width_to_agaw(int width)
291 return (width - 30) / LEVEL_STRIDE;
294 static inline unsigned int level_to_offset_bits(int level)
296 return (12 + (level - 1) * LEVEL_STRIDE);
299 static inline int address_level_offset(u64 addr, int level)
301 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
304 static inline u64 level_mask(int level)
306 return ((u64)-1 << level_to_offset_bits(level));
309 static inline u64 level_size(int level)
311 return ((u64)1 << level_to_offset_bits(level));
314 static inline u64 align_to_level(u64 addr, int level)
316 return ((addr + level_size(level) - 1) & level_mask(level));
319 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
321 int addr_width = agaw_to_width(domain->agaw);
322 struct dma_pte *parent, *pte = NULL;
323 int level = agaw_to_level(domain->agaw);
327 BUG_ON(!domain->pgd);
329 addr &= (((u64)1) << addr_width) - 1;
330 parent = domain->pgd;
332 spin_lock_irqsave(&domain->mapping_lock, flags);
336 offset = address_level_offset(addr, level);
337 pte = &parent[offset];
341 if (!dma_pte_present(*pte)) {
342 tmp_page = alloc_pgtable_page();
345 spin_unlock_irqrestore(&domain->mapping_lock,
349 __iommu_flush_cache(domain->iommu, tmp_page,
351 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
353 * high level table always sets r/w, last level page
354 * table control read/write
356 dma_set_pte_readable(*pte);
357 dma_set_pte_writable(*pte);
358 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
360 parent = phys_to_virt(dma_pte_addr(*pte));
364 spin_unlock_irqrestore(&domain->mapping_lock, flags);
368 /* return address's pte at specific level */
369 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
372 struct dma_pte *parent, *pte = NULL;
373 int total = agaw_to_level(domain->agaw);
376 parent = domain->pgd;
377 while (level <= total) {
378 offset = address_level_offset(addr, total);
379 pte = &parent[offset];
383 if (!dma_pte_present(*pte))
385 parent = phys_to_virt(dma_pte_addr(*pte));
391 /* clear one page's page table */
392 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
394 struct dma_pte *pte = NULL;
396 /* get last level pte */
397 pte = dma_addr_level_pte(domain, addr, 1);
401 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
405 /* clear last level pte, a tlb flush should be followed */
406 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
408 int addr_width = agaw_to_width(domain->agaw);
410 start &= (((u64)1) << addr_width) - 1;
411 end &= (((u64)1) << addr_width) - 1;
412 /* in case it's partial page */
413 start = PAGE_ALIGN(start);
416 /* we don't need lock here, nobody else touches the iova range */
417 while (start < end) {
418 dma_pte_clear_one(domain, start);
419 start += VTD_PAGE_SIZE;
423 /* free page table pages. last level pte should already be cleared */
424 static void dma_pte_free_pagetable(struct dmar_domain *domain,
427 int addr_width = agaw_to_width(domain->agaw);
429 int total = agaw_to_level(domain->agaw);
433 start &= (((u64)1) << addr_width) - 1;
434 end &= (((u64)1) << addr_width) - 1;
436 /* we don't need lock here, nobody else touches the iova range */
438 while (level <= total) {
439 tmp = align_to_level(start, level);
440 if (tmp >= end || (tmp + level_size(level) > end))
444 pte = dma_addr_level_pte(domain, tmp, level);
447 phys_to_virt(dma_pte_addr(*pte)));
449 __iommu_flush_cache(domain->iommu,
452 tmp += level_size(level);
457 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
458 free_pgtable_page(domain->pgd);
464 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
466 struct root_entry *root;
469 root = (struct root_entry *)alloc_pgtable_page();
473 __iommu_flush_cache(iommu, root, ROOT_SIZE);
475 spin_lock_irqsave(&iommu->lock, flags);
476 iommu->root_entry = root;
477 spin_unlock_irqrestore(&iommu->lock, flags);
482 static void iommu_set_root_entry(struct intel_iommu *iommu)
488 addr = iommu->root_entry;
490 spin_lock_irqsave(&iommu->register_lock, flag);
491 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
493 cmd = iommu->gcmd | DMA_GCMD_SRTP;
494 writel(cmd, iommu->reg + DMAR_GCMD_REG);
496 /* Make sure hardware complete it */
497 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
498 readl, (sts & DMA_GSTS_RTPS), sts);
500 spin_unlock_irqrestore(&iommu->register_lock, flag);
503 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
508 if (!cap_rwbf(iommu->cap))
510 val = iommu->gcmd | DMA_GCMD_WBF;
512 spin_lock_irqsave(&iommu->register_lock, flag);
513 writel(val, iommu->reg + DMAR_GCMD_REG);
515 /* Make sure hardware complete it */
516 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
517 readl, (!(val & DMA_GSTS_WBFS)), val);
519 spin_unlock_irqrestore(&iommu->register_lock, flag);
522 /* return value determine if we need a write buffer flush */
523 static int __iommu_flush_context(struct intel_iommu *iommu,
524 u16 did, u16 source_id, u8 function_mask, u64 type,
525 int non_present_entry_flush)
531 * In the non-present entry flush case, if hardware doesn't cache
532 * non-present entry we do nothing and if hardware cache non-present
533 * entry, we flush entries of domain 0 (the domain id is used to cache
534 * any non-present entries)
536 if (non_present_entry_flush) {
537 if (!cap_caching_mode(iommu->cap))
544 case DMA_CCMD_GLOBAL_INVL:
545 val = DMA_CCMD_GLOBAL_INVL;
547 case DMA_CCMD_DOMAIN_INVL:
548 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
550 case DMA_CCMD_DEVICE_INVL:
551 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
552 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
559 spin_lock_irqsave(&iommu->register_lock, flag);
560 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
562 /* Make sure hardware complete it */
563 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
564 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
566 spin_unlock_irqrestore(&iommu->register_lock, flag);
568 /* flush context entry will implicitly flush write buffer */
572 /* return value determine if we need a write buffer flush */
573 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
574 u64 addr, unsigned int size_order, u64 type,
575 int non_present_entry_flush)
577 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
578 u64 val = 0, val_iva = 0;
582 * In the non-present entry flush case, if hardware doesn't cache
583 * non-present entry we do nothing and if hardware cache non-present
584 * entry, we flush entries of domain 0 (the domain id is used to cache
585 * any non-present entries)
587 if (non_present_entry_flush) {
588 if (!cap_caching_mode(iommu->cap))
595 case DMA_TLB_GLOBAL_FLUSH:
596 /* global flush doesn't need set IVA_REG */
597 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
599 case DMA_TLB_DSI_FLUSH:
600 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
602 case DMA_TLB_PSI_FLUSH:
603 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
604 /* Note: always flush non-leaf currently */
605 val_iva = size_order | addr;
610 /* Note: set drain read/write */
613 * This is probably to be super secure.. Looks like we can
614 * ignore it without any impact.
616 if (cap_read_drain(iommu->cap))
617 val |= DMA_TLB_READ_DRAIN;
619 if (cap_write_drain(iommu->cap))
620 val |= DMA_TLB_WRITE_DRAIN;
622 spin_lock_irqsave(&iommu->register_lock, flag);
623 /* Note: Only uses first TLB reg currently */
625 dmar_writeq(iommu->reg + tlb_offset, val_iva);
626 dmar_writeq(iommu->reg + tlb_offset + 8, val);
628 /* Make sure hardware complete it */
629 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
630 dmar_readq, (!(val & DMA_TLB_IVT)), val);
632 spin_unlock_irqrestore(&iommu->register_lock, flag);
634 /* check IOTLB invalidation granularity */
635 if (DMA_TLB_IAIG(val) == 0)
636 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
637 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
638 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
639 (unsigned long long)DMA_TLB_IIRG(type),
640 (unsigned long long)DMA_TLB_IAIG(val));
641 /* flush iotlb entry will implicitly flush write buffer */
645 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
646 u64 addr, unsigned int pages, int non_present_entry_flush)
650 BUG_ON(addr & (~VTD_PAGE_MASK));
653 /* Fallback to domain selective flush if no PSI support */
654 if (!cap_pgsel_inv(iommu->cap))
655 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
657 non_present_entry_flush);
660 * PSI requires page size to be 2 ^ x, and the base address is naturally
661 * aligned to the size
663 mask = ilog2(__roundup_pow_of_two(pages));
664 /* Fallback to domain selective flush if size is too big */
665 if (mask > cap_max_amask_val(iommu->cap))
666 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
667 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
669 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
671 non_present_entry_flush);
674 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
679 spin_lock_irqsave(&iommu->register_lock, flags);
680 pmen = readl(iommu->reg + DMAR_PMEN_REG);
681 pmen &= ~DMA_PMEN_EPM;
682 writel(pmen, iommu->reg + DMAR_PMEN_REG);
684 /* wait for the protected region status bit to clear */
685 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
686 readl, !(pmen & DMA_PMEN_PRS), pmen);
688 spin_unlock_irqrestore(&iommu->register_lock, flags);
691 static int iommu_enable_translation(struct intel_iommu *iommu)
696 spin_lock_irqsave(&iommu->register_lock, flags);
697 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
699 /* Make sure hardware complete it */
700 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
701 readl, (sts & DMA_GSTS_TES), sts);
703 iommu->gcmd |= DMA_GCMD_TE;
704 spin_unlock_irqrestore(&iommu->register_lock, flags);
708 static int iommu_disable_translation(struct intel_iommu *iommu)
713 spin_lock_irqsave(&iommu->register_lock, flag);
714 iommu->gcmd &= ~DMA_GCMD_TE;
715 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
717 /* Make sure hardware complete it */
718 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
719 readl, (!(sts & DMA_GSTS_TES)), sts);
721 spin_unlock_irqrestore(&iommu->register_lock, flag);
725 /* iommu interrupt handling. Most stuff are MSI-like. */
727 static const char *fault_reason_strings[] =
730 "Present bit in root entry is clear",
731 "Present bit in context entry is clear",
732 "Invalid context entry",
733 "Access beyond MGAW",
734 "PTE Write access is not set",
735 "PTE Read access is not set",
736 "Next page table ptr is invalid",
737 "Root table address invalid",
738 "Context table ptr is invalid",
739 "non-zero reserved fields in RTP",
740 "non-zero reserved fields in CTP",
741 "non-zero reserved fields in PTE",
743 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
745 const char *dmar_get_fault_reason(u8 fault_reason)
747 if (fault_reason > MAX_FAULT_REASON_IDX)
750 return fault_reason_strings[fault_reason];
753 void dmar_msi_unmask(unsigned int irq)
755 struct intel_iommu *iommu = get_irq_data(irq);
759 spin_lock_irqsave(&iommu->register_lock, flag);
760 writel(0, iommu->reg + DMAR_FECTL_REG);
761 /* Read a reg to force flush the post write */
762 readl(iommu->reg + DMAR_FECTL_REG);
763 spin_unlock_irqrestore(&iommu->register_lock, flag);
766 void dmar_msi_mask(unsigned int irq)
769 struct intel_iommu *iommu = get_irq_data(irq);
772 spin_lock_irqsave(&iommu->register_lock, flag);
773 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
774 /* Read a reg to force flush the post write */
775 readl(iommu->reg + DMAR_FECTL_REG);
776 spin_unlock_irqrestore(&iommu->register_lock, flag);
779 void dmar_msi_write(int irq, struct msi_msg *msg)
781 struct intel_iommu *iommu = get_irq_data(irq);
784 spin_lock_irqsave(&iommu->register_lock, flag);
785 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
786 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
787 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
788 spin_unlock_irqrestore(&iommu->register_lock, flag);
791 void dmar_msi_read(int irq, struct msi_msg *msg)
793 struct intel_iommu *iommu = get_irq_data(irq);
796 spin_lock_irqsave(&iommu->register_lock, flag);
797 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
798 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
799 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
800 spin_unlock_irqrestore(&iommu->register_lock, flag);
803 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
804 u8 fault_reason, u16 source_id, unsigned long long addr)
808 reason = dmar_get_fault_reason(fault_reason);
811 "DMAR:[%s] Request device [%02x:%02x.%d] "
813 "DMAR:[fault reason %02d] %s\n",
814 (type ? "DMA Read" : "DMA Write"),
815 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
816 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
820 #define PRIMARY_FAULT_REG_LEN (16)
821 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
823 struct intel_iommu *iommu = dev_id;
824 int reg, fault_index;
828 spin_lock_irqsave(&iommu->register_lock, flag);
829 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
831 /* TBD: ignore advanced fault log currently */
832 if (!(fault_status & DMA_FSTS_PPF))
835 fault_index = dma_fsts_fault_record_index(fault_status);
836 reg = cap_fault_reg_offset(iommu->cap);
844 /* highest 32 bits */
845 data = readl(iommu->reg + reg +
846 fault_index * PRIMARY_FAULT_REG_LEN + 12);
847 if (!(data & DMA_FRCD_F))
850 fault_reason = dma_frcd_fault_reason(data);
851 type = dma_frcd_type(data);
853 data = readl(iommu->reg + reg +
854 fault_index * PRIMARY_FAULT_REG_LEN + 8);
855 source_id = dma_frcd_source_id(data);
857 guest_addr = dmar_readq(iommu->reg + reg +
858 fault_index * PRIMARY_FAULT_REG_LEN);
859 guest_addr = dma_frcd_page_addr(guest_addr);
860 /* clear the fault */
861 writel(DMA_FRCD_F, iommu->reg + reg +
862 fault_index * PRIMARY_FAULT_REG_LEN + 12);
864 spin_unlock_irqrestore(&iommu->register_lock, flag);
866 iommu_page_fault_do_one(iommu, type, fault_reason,
867 source_id, guest_addr);
870 if (fault_index > cap_num_fault_regs(iommu->cap))
872 spin_lock_irqsave(&iommu->register_lock, flag);
875 /* clear primary fault overflow */
876 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
877 if (fault_status & DMA_FSTS_PFO)
878 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
880 spin_unlock_irqrestore(&iommu->register_lock, flag);
884 int dmar_set_interrupt(struct intel_iommu *iommu)
890 printk(KERN_ERR "IOMMU: no free vectors\n");
894 set_irq_data(irq, iommu);
897 ret = arch_setup_dmar_msi(irq);
899 set_irq_data(irq, NULL);
905 /* Force fault register is cleared */
906 iommu_page_fault(irq, iommu);
908 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
910 printk(KERN_ERR "IOMMU: can't request irq\n");
914 static int iommu_init_domains(struct intel_iommu *iommu)
916 unsigned long ndomains;
917 unsigned long nlongs;
919 ndomains = cap_ndoms(iommu->cap);
920 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
921 nlongs = BITS_TO_LONGS(ndomains);
923 /* TBD: there might be 64K domains,
924 * consider other allocation for future chip
926 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
927 if (!iommu->domain_ids) {
928 printk(KERN_ERR "Allocating domain id array failed\n");
931 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
933 if (!iommu->domains) {
934 printk(KERN_ERR "Allocating domain array failed\n");
935 kfree(iommu->domain_ids);
939 spin_lock_init(&iommu->lock);
942 * if Caching mode is set, then invalid translations are tagged
943 * with domainid 0. Hence we need to pre-allocate it.
945 if (cap_caching_mode(iommu->cap))
946 set_bit(0, iommu->domain_ids);
951 static void domain_exit(struct dmar_domain *domain);
953 void free_dmar_iommu(struct intel_iommu *iommu)
955 struct dmar_domain *domain;
958 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
959 for (; i < cap_ndoms(iommu->cap); ) {
960 domain = iommu->domains[i];
961 clear_bit(i, iommu->domain_ids);
963 i = find_next_bit(iommu->domain_ids,
964 cap_ndoms(iommu->cap), i+1);
967 if (iommu->gcmd & DMA_GCMD_TE)
968 iommu_disable_translation(iommu);
971 set_irq_data(iommu->irq, NULL);
972 /* This will mask the irq */
973 free_irq(iommu->irq, iommu);
974 destroy_irq(iommu->irq);
977 kfree(iommu->domains);
978 kfree(iommu->domain_ids);
980 /* free context mapping */
981 free_context_table(iommu);
984 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
987 unsigned long ndomains;
988 struct dmar_domain *domain;
991 domain = alloc_domain_mem();
995 ndomains = cap_ndoms(iommu->cap);
997 spin_lock_irqsave(&iommu->lock, flags);
998 num = find_first_zero_bit(iommu->domain_ids, ndomains);
999 if (num >= ndomains) {
1000 spin_unlock_irqrestore(&iommu->lock, flags);
1001 free_domain_mem(domain);
1002 printk(KERN_ERR "IOMMU: no free domain ids\n");
1006 set_bit(num, iommu->domain_ids);
1008 domain->iommu = iommu;
1009 iommu->domains[num] = domain;
1010 spin_unlock_irqrestore(&iommu->lock, flags);
1015 static void iommu_free_domain(struct dmar_domain *domain)
1017 unsigned long flags;
1019 spin_lock_irqsave(&domain->iommu->lock, flags);
1020 clear_bit(domain->id, domain->iommu->domain_ids);
1021 spin_unlock_irqrestore(&domain->iommu->lock, flags);
1024 static struct iova_domain reserved_iova_list;
1025 static struct lock_class_key reserved_alloc_key;
1026 static struct lock_class_key reserved_rbtree_key;
1028 static void dmar_init_reserved_ranges(void)
1030 struct pci_dev *pdev = NULL;
1035 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1037 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1038 &reserved_alloc_key);
1039 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1040 &reserved_rbtree_key);
1042 /* IOAPIC ranges shouldn't be accessed by DMA */
1043 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1044 IOVA_PFN(IOAPIC_RANGE_END));
1046 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1048 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1049 for_each_pci_dev(pdev) {
1052 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1053 r = &pdev->resource[i];
1054 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1058 size = r->end - addr;
1059 size = PAGE_ALIGN(size);
1060 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1061 IOVA_PFN(size + addr) - 1);
1063 printk(KERN_ERR "Reserve iova failed\n");
1069 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1071 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1074 static inline int guestwidth_to_adjustwidth(int gaw)
1077 int r = (gaw - 12) % 9;
1088 static int domain_init(struct dmar_domain *domain, int guest_width)
1090 struct intel_iommu *iommu;
1091 int adjust_width, agaw;
1092 unsigned long sagaw;
1094 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1095 spin_lock_init(&domain->mapping_lock);
1097 domain_reserve_special_ranges(domain);
1099 /* calculate AGAW */
1100 iommu = domain->iommu;
1101 if (guest_width > cap_mgaw(iommu->cap))
1102 guest_width = cap_mgaw(iommu->cap);
1103 domain->gaw = guest_width;
1104 adjust_width = guestwidth_to_adjustwidth(guest_width);
1105 agaw = width_to_agaw(adjust_width);
1106 sagaw = cap_sagaw(iommu->cap);
1107 if (!test_bit(agaw, &sagaw)) {
1108 /* hardware doesn't support it, choose a bigger one */
1109 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1110 agaw = find_next_bit(&sagaw, 5, agaw);
1114 domain->agaw = agaw;
1115 INIT_LIST_HEAD(&domain->devices);
1117 /* always allocate the top pgd */
1118 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1121 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1125 static void domain_exit(struct dmar_domain *domain)
1129 /* Domain 0 is reserved, so dont process it */
1133 domain_remove_dev_info(domain);
1135 put_iova_domain(&domain->iovad);
1136 end = DOMAIN_MAX_ADDR(domain->gaw);
1137 end = end & (~PAGE_MASK);
1140 dma_pte_clear_range(domain, 0, end);
1142 /* free page tables */
1143 dma_pte_free_pagetable(domain, 0, end);
1145 iommu_free_domain(domain);
1146 free_domain_mem(domain);
1149 static int domain_context_mapping_one(struct dmar_domain *domain,
1152 struct context_entry *context;
1153 struct intel_iommu *iommu = domain->iommu;
1154 unsigned long flags;
1156 pr_debug("Set context mapping for %02x:%02x.%d\n",
1157 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1158 BUG_ON(!domain->pgd);
1159 context = device_to_context_entry(iommu, bus, devfn);
1162 spin_lock_irqsave(&iommu->lock, flags);
1163 if (context_present(*context)) {
1164 spin_unlock_irqrestore(&iommu->lock, flags);
1168 context_set_domain_id(*context, domain->id);
1169 context_set_address_width(*context, domain->agaw);
1170 context_set_address_root(*context, virt_to_phys(domain->pgd));
1171 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1172 context_set_fault_enable(*context);
1173 context_set_present(*context);
1174 __iommu_flush_cache(iommu, context, sizeof(*context));
1176 /* it's a non-present to present mapping */
1177 if (iommu->flush.flush_context(iommu, domain->id,
1178 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1179 DMA_CCMD_DEVICE_INVL, 1))
1180 iommu_flush_write_buffer(iommu);
1182 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1184 spin_unlock_irqrestore(&iommu->lock, flags);
1189 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1192 struct pci_dev *tmp, *parent;
1194 ret = domain_context_mapping_one(domain, pdev->bus->number,
1199 /* dependent device mapping */
1200 tmp = pci_find_upstream_pcie_bridge(pdev);
1203 /* Secondary interface's bus number and devfn 0 */
1204 parent = pdev->bus->self;
1205 while (parent != tmp) {
1206 ret = domain_context_mapping_one(domain, parent->bus->number,
1210 parent = parent->bus->self;
1212 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1213 return domain_context_mapping_one(domain,
1214 tmp->subordinate->number, 0);
1215 else /* this is a legacy PCI bridge */
1216 return domain_context_mapping_one(domain,
1217 tmp->bus->number, tmp->devfn);
1220 static int domain_context_mapped(struct dmar_domain *domain,
1221 struct pci_dev *pdev)
1224 struct pci_dev *tmp, *parent;
1226 ret = device_context_mapped(domain->iommu,
1227 pdev->bus->number, pdev->devfn);
1230 /* dependent device mapping */
1231 tmp = pci_find_upstream_pcie_bridge(pdev);
1234 /* Secondary interface's bus number and devfn 0 */
1235 parent = pdev->bus->self;
1236 while (parent != tmp) {
1237 ret = device_context_mapped(domain->iommu, parent->bus->number,
1241 parent = parent->bus->self;
1244 return device_context_mapped(domain->iommu,
1245 tmp->subordinate->number, 0);
1247 return device_context_mapped(domain->iommu,
1248 tmp->bus->number, tmp->devfn);
1252 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1253 u64 hpa, size_t size, int prot)
1255 u64 start_pfn, end_pfn;
1256 struct dma_pte *pte;
1258 int addr_width = agaw_to_width(domain->agaw);
1260 hpa &= (((u64)1) << addr_width) - 1;
1262 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1265 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1266 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1268 while (start_pfn < end_pfn) {
1269 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1272 /* We don't need lock here, nobody else
1273 * touches the iova range
1275 BUG_ON(dma_pte_addr(*pte));
1276 dma_set_pte_addr(*pte, start_pfn << VTD_PAGE_SHIFT);
1277 dma_set_pte_prot(*pte, prot);
1278 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1285 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1287 clear_context_table(domain->iommu, bus, devfn);
1288 domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1289 DMA_CCMD_GLOBAL_INVL, 0);
1290 domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1291 DMA_TLB_GLOBAL_FLUSH, 0);
1294 static void domain_remove_dev_info(struct dmar_domain *domain)
1296 struct device_domain_info *info;
1297 unsigned long flags;
1299 spin_lock_irqsave(&device_domain_lock, flags);
1300 while (!list_empty(&domain->devices)) {
1301 info = list_entry(domain->devices.next,
1302 struct device_domain_info, link);
1303 list_del(&info->link);
1304 list_del(&info->global);
1306 info->dev->dev.archdata.iommu = NULL;
1307 spin_unlock_irqrestore(&device_domain_lock, flags);
1309 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1310 free_devinfo_mem(info);
1312 spin_lock_irqsave(&device_domain_lock, flags);
1314 spin_unlock_irqrestore(&device_domain_lock, flags);
1319 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1321 static struct dmar_domain *
1322 find_domain(struct pci_dev *pdev)
1324 struct device_domain_info *info;
1326 /* No lock here, assumes no domain exit in normal case */
1327 info = pdev->dev.archdata.iommu;
1329 return info->domain;
1333 /* domain is initialized */
1334 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1336 struct dmar_domain *domain, *found = NULL;
1337 struct intel_iommu *iommu;
1338 struct dmar_drhd_unit *drhd;
1339 struct device_domain_info *info, *tmp;
1340 struct pci_dev *dev_tmp;
1341 unsigned long flags;
1342 int bus = 0, devfn = 0;
1344 domain = find_domain(pdev);
1348 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1350 if (dev_tmp->is_pcie) {
1351 bus = dev_tmp->subordinate->number;
1354 bus = dev_tmp->bus->number;
1355 devfn = dev_tmp->devfn;
1357 spin_lock_irqsave(&device_domain_lock, flags);
1358 list_for_each_entry(info, &device_domain_list, global) {
1359 if (info->bus == bus && info->devfn == devfn) {
1360 found = info->domain;
1364 spin_unlock_irqrestore(&device_domain_lock, flags);
1365 /* pcie-pci bridge already has a domain, uses it */
1372 /* Allocate new domain for the device */
1373 drhd = dmar_find_matched_drhd_unit(pdev);
1375 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1379 iommu = drhd->iommu;
1381 domain = iommu_alloc_domain(iommu);
1385 if (domain_init(domain, gaw)) {
1386 domain_exit(domain);
1390 /* register pcie-to-pci device */
1392 info = alloc_devinfo_mem();
1394 domain_exit(domain);
1398 info->devfn = devfn;
1400 info->domain = domain;
1401 /* This domain is shared by devices under p2p bridge */
1402 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1404 /* pcie-to-pci bridge already has a domain, uses it */
1406 spin_lock_irqsave(&device_domain_lock, flags);
1407 list_for_each_entry(tmp, &device_domain_list, global) {
1408 if (tmp->bus == bus && tmp->devfn == devfn) {
1409 found = tmp->domain;
1414 free_devinfo_mem(info);
1415 domain_exit(domain);
1418 list_add(&info->link, &domain->devices);
1419 list_add(&info->global, &device_domain_list);
1421 spin_unlock_irqrestore(&device_domain_lock, flags);
1425 info = alloc_devinfo_mem();
1428 info->bus = pdev->bus->number;
1429 info->devfn = pdev->devfn;
1431 info->domain = domain;
1432 spin_lock_irqsave(&device_domain_lock, flags);
1433 /* somebody is fast */
1434 found = find_domain(pdev);
1435 if (found != NULL) {
1436 spin_unlock_irqrestore(&device_domain_lock, flags);
1437 if (found != domain) {
1438 domain_exit(domain);
1441 free_devinfo_mem(info);
1444 list_add(&info->link, &domain->devices);
1445 list_add(&info->global, &device_domain_list);
1446 pdev->dev.archdata.iommu = info;
1447 spin_unlock_irqrestore(&device_domain_lock, flags);
1450 /* recheck it here, maybe others set it */
1451 return find_domain(pdev);
1454 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1455 unsigned long long start,
1456 unsigned long long end)
1458 struct dmar_domain *domain;
1460 unsigned long long base;
1464 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1465 pci_name(pdev), start, end);
1466 /* page table init */
1467 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1471 /* The address might not be aligned */
1472 base = start & PAGE_MASK;
1474 size = PAGE_ALIGN(size);
1475 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1476 IOVA_PFN(base + size) - 1)) {
1477 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1482 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1483 size, base, pci_name(pdev));
1485 * RMRR range might have overlap with physical memory range,
1488 dma_pte_clear_range(domain, base, base + size);
1490 ret = domain_page_mapping(domain, base, base, size,
1491 DMA_PTE_READ|DMA_PTE_WRITE);
1495 /* context entry init */
1496 ret = domain_context_mapping(domain, pdev);
1500 domain_exit(domain);
1505 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1506 struct pci_dev *pdev)
1508 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1510 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1511 rmrr->end_address + 1);
1514 #ifdef CONFIG_DMAR_GFX_WA
1515 struct iommu_prepare_data {
1516 struct pci_dev *pdev;
1520 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1521 unsigned long end_pfn, void *datax)
1523 struct iommu_prepare_data *data;
1525 data = (struct iommu_prepare_data *)datax;
1527 data->ret = iommu_prepare_identity_map(data->pdev,
1528 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1533 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1536 struct iommu_prepare_data data;
1541 for_each_online_node(nid) {
1542 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1549 static void __init iommu_prepare_gfx_mapping(void)
1551 struct pci_dev *pdev = NULL;
1554 for_each_pci_dev(pdev) {
1555 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1556 !IS_GFX_DEVICE(pdev))
1558 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1560 ret = iommu_prepare_with_active_regions(pdev);
1562 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1567 #ifdef CONFIG_DMAR_FLOPPY_WA
1568 static inline void iommu_prepare_isa(void)
1570 struct pci_dev *pdev;
1573 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1577 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1578 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1581 printk("IOMMU: Failed to create 0-64M identity map, "
1582 "floppy might not work\n");
1586 static inline void iommu_prepare_isa(void)
1590 #endif /* !CONFIG_DMAR_FLPY_WA */
1592 static int __init init_dmars(void)
1594 struct dmar_drhd_unit *drhd;
1595 struct dmar_rmrr_unit *rmrr;
1596 struct pci_dev *pdev;
1597 struct intel_iommu *iommu;
1598 int i, ret, unit = 0;
1603 * initialize and program root entry to not present
1606 for_each_drhd_unit(drhd) {
1609 * lock not needed as this is only incremented in the single
1610 * threaded kernel __init code path all other access are read
1615 deferred_flush = kzalloc(g_num_of_iommus *
1616 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1617 if (!deferred_flush) {
1622 for_each_drhd_unit(drhd) {
1626 iommu = drhd->iommu;
1628 ret = iommu_init_domains(iommu);
1634 * we could share the same root & context tables
1635 * amoung all IOMMU's. Need to Split it later.
1637 ret = iommu_alloc_root_entry(iommu);
1639 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1644 for_each_drhd_unit(drhd) {
1648 iommu = drhd->iommu;
1649 if (dmar_enable_qi(iommu)) {
1651 * Queued Invalidate not enabled, use Register Based
1654 iommu->flush.flush_context = __iommu_flush_context;
1655 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1656 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1658 (unsigned long long)drhd->reg_base_addr);
1660 iommu->flush.flush_context = qi_flush_context;
1661 iommu->flush.flush_iotlb = qi_flush_iotlb;
1662 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1664 (unsigned long long)drhd->reg_base_addr);
1670 * for each dev attached to rmrr
1672 * locate drhd for dev, alloc domain for dev
1673 * allocate free domain
1674 * allocate page table entries for rmrr
1675 * if context not allocated for bus
1676 * allocate and init context
1677 * set present in root table for this bus
1678 * init context with domain, translation etc
1682 for_each_rmrr_units(rmrr) {
1683 for (i = 0; i < rmrr->devices_cnt; i++) {
1684 pdev = rmrr->devices[i];
1685 /* some BIOS lists non-exist devices in DMAR table */
1688 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1691 "IOMMU: mapping reserved region failed\n");
1695 iommu_prepare_gfx_mapping();
1697 iommu_prepare_isa();
1702 * global invalidate context cache
1703 * global invalidate iotlb
1704 * enable translation
1706 for_each_drhd_unit(drhd) {
1709 iommu = drhd->iommu;
1710 sprintf (iommu->name, "dmar%d", unit++);
1712 iommu_flush_write_buffer(iommu);
1714 ret = dmar_set_interrupt(iommu);
1718 iommu_set_root_entry(iommu);
1720 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1722 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1724 iommu_disable_protect_mem_regions(iommu);
1726 ret = iommu_enable_translation(iommu);
1733 for_each_drhd_unit(drhd) {
1736 iommu = drhd->iommu;
1742 static inline u64 aligned_size(u64 host_addr, size_t size)
1745 addr = (host_addr & (~PAGE_MASK)) + size;
1746 return PAGE_ALIGN(addr);
1750 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1754 /* Make sure it's in range */
1755 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1756 if (!size || (IOVA_START_ADDR + size > end))
1759 piova = alloc_iova(&domain->iovad,
1760 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1764 static struct iova *
1765 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1766 size_t size, u64 dma_mask)
1768 struct pci_dev *pdev = to_pci_dev(dev);
1769 struct iova *iova = NULL;
1771 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1772 iova = iommu_alloc_iova(domain, size, dma_mask);
1775 * First try to allocate an io virtual address in
1776 * DMA_32BIT_MASK and if that fails then try allocating
1779 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1781 iova = iommu_alloc_iova(domain, size, dma_mask);
1785 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1792 static struct dmar_domain *
1793 get_valid_domain_for_dev(struct pci_dev *pdev)
1795 struct dmar_domain *domain;
1798 domain = get_domain_for_dev(pdev,
1799 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1802 "Allocating domain for %s failed", pci_name(pdev));
1806 /* make sure context mapping is ok */
1807 if (unlikely(!domain_context_mapped(domain, pdev))) {
1808 ret = domain_context_mapping(domain, pdev);
1811 "Domain context map for %s failed",
1820 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
1821 size_t size, int dir, u64 dma_mask)
1823 struct pci_dev *pdev = to_pci_dev(hwdev);
1824 struct dmar_domain *domain;
1825 phys_addr_t start_paddr;
1830 BUG_ON(dir == DMA_NONE);
1831 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1834 domain = get_valid_domain_for_dev(pdev);
1838 size = aligned_size((u64)paddr, size);
1840 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
1844 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
1847 * Check if DMAR supports zero-length reads on write only
1850 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1851 !cap_zlr(domain->iommu->cap))
1852 prot |= DMA_PTE_READ;
1853 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1854 prot |= DMA_PTE_WRITE;
1856 * paddr - (paddr + size) might be partial page, we should map the whole
1857 * page. Note: if two part of one page are separately mapped, we
1858 * might have two guest_addr mapping to the same host paddr, but this
1859 * is not a big problem
1861 ret = domain_page_mapping(domain, start_paddr,
1862 ((u64)paddr) & PAGE_MASK, size, prot);
1866 /* it's a non-present to present mapping */
1867 ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
1868 start_paddr, size >> VTD_PAGE_SHIFT, 1);
1870 iommu_flush_write_buffer(domain->iommu);
1872 return start_paddr + ((u64)paddr & (~PAGE_MASK));
1876 __free_iova(&domain->iovad, iova);
1877 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1878 pci_name(pdev), size, (unsigned long long)paddr, dir);
1882 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
1883 size_t size, int dir)
1885 return __intel_map_single(hwdev, paddr, size, dir,
1886 to_pci_dev(hwdev)->dma_mask);
1889 static void flush_unmaps(void)
1895 /* just flush them all */
1896 for (i = 0; i < g_num_of_iommus; i++) {
1897 if (deferred_flush[i].next) {
1898 struct intel_iommu *iommu =
1899 deferred_flush[i].domain[0]->iommu;
1901 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1902 DMA_TLB_GLOBAL_FLUSH, 0);
1903 for (j = 0; j < deferred_flush[i].next; j++) {
1904 __free_iova(&deferred_flush[i].domain[j]->iovad,
1905 deferred_flush[i].iova[j]);
1907 deferred_flush[i].next = 0;
1914 static void flush_unmaps_timeout(unsigned long data)
1916 unsigned long flags;
1918 spin_lock_irqsave(&async_umap_flush_lock, flags);
1920 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1923 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
1925 unsigned long flags;
1928 spin_lock_irqsave(&async_umap_flush_lock, flags);
1929 if (list_size == HIGH_WATER_MARK)
1932 iommu_id = dom->iommu->seq_id;
1934 next = deferred_flush[iommu_id].next;
1935 deferred_flush[iommu_id].domain[next] = dom;
1936 deferred_flush[iommu_id].iova[next] = iova;
1937 deferred_flush[iommu_id].next++;
1940 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
1944 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1947 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
1950 struct pci_dev *pdev = to_pci_dev(dev);
1951 struct dmar_domain *domain;
1952 unsigned long start_addr;
1955 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1957 domain = find_domain(pdev);
1960 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1964 start_addr = iova->pfn_lo << PAGE_SHIFT;
1965 size = aligned_size((u64)dev_addr, size);
1967 pr_debug("Device %s unmapping: %lx@%llx\n",
1968 pci_name(pdev), size, (unsigned long long)start_addr);
1970 /* clear the whole page */
1971 dma_pte_clear_range(domain, start_addr, start_addr + size);
1972 /* free page tables */
1973 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
1974 if (intel_iommu_strict) {
1975 if (iommu_flush_iotlb_psi(domain->iommu,
1976 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
1977 iommu_flush_write_buffer(domain->iommu);
1979 __free_iova(&domain->iovad, iova);
1981 add_unmap(domain, iova);
1983 * queue up the release of the unmap to save the 1/6th of the
1984 * cpu used up by the iotlb flush operation...
1989 void *intel_alloc_coherent(struct device *hwdev, size_t size,
1990 dma_addr_t *dma_handle, gfp_t flags)
1995 size = PAGE_ALIGN(size);
1996 order = get_order(size);
1997 flags &= ~(GFP_DMA | GFP_DMA32);
1999 vaddr = (void *)__get_free_pages(flags, order);
2002 memset(vaddr, 0, size);
2004 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2006 hwdev->coherent_dma_mask);
2009 free_pages((unsigned long)vaddr, order);
2013 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2014 dma_addr_t dma_handle)
2018 size = PAGE_ALIGN(size);
2019 order = get_order(size);
2021 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2022 free_pages((unsigned long)vaddr, order);
2025 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2027 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2028 int nelems, int dir)
2031 struct pci_dev *pdev = to_pci_dev(hwdev);
2032 struct dmar_domain *domain;
2033 unsigned long start_addr;
2037 struct scatterlist *sg;
2039 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2042 domain = find_domain(pdev);
2044 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2047 for_each_sg(sglist, sg, nelems, i) {
2048 addr = SG_ENT_VIRT_ADDRESS(sg);
2049 size += aligned_size((u64)addr, sg->length);
2052 start_addr = iova->pfn_lo << PAGE_SHIFT;
2054 /* clear the whole page */
2055 dma_pte_clear_range(domain, start_addr, start_addr + size);
2056 /* free page tables */
2057 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2059 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2060 size >> VTD_PAGE_SHIFT, 0))
2061 iommu_flush_write_buffer(domain->iommu);
2064 __free_iova(&domain->iovad, iova);
2067 static int intel_nontranslate_map_sg(struct device *hddev,
2068 struct scatterlist *sglist, int nelems, int dir)
2071 struct scatterlist *sg;
2073 for_each_sg(sglist, sg, nelems, i) {
2074 BUG_ON(!sg_page(sg));
2075 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2076 sg->dma_length = sg->length;
2081 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2086 struct pci_dev *pdev = to_pci_dev(hwdev);
2087 struct dmar_domain *domain;
2091 struct iova *iova = NULL;
2093 struct scatterlist *sg;
2094 unsigned long start_addr;
2096 BUG_ON(dir == DMA_NONE);
2097 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2098 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2100 domain = get_valid_domain_for_dev(pdev);
2104 for_each_sg(sglist, sg, nelems, i) {
2105 addr = SG_ENT_VIRT_ADDRESS(sg);
2106 addr = (void *)virt_to_phys(addr);
2107 size += aligned_size((u64)addr, sg->length);
2110 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2112 sglist->dma_length = 0;
2117 * Check if DMAR supports zero-length reads on write only
2120 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2121 !cap_zlr(domain->iommu->cap))
2122 prot |= DMA_PTE_READ;
2123 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2124 prot |= DMA_PTE_WRITE;
2126 start_addr = iova->pfn_lo << PAGE_SHIFT;
2128 for_each_sg(sglist, sg, nelems, i) {
2129 addr = SG_ENT_VIRT_ADDRESS(sg);
2130 addr = (void *)virt_to_phys(addr);
2131 size = aligned_size((u64)addr, sg->length);
2132 ret = domain_page_mapping(domain, start_addr + offset,
2133 ((u64)addr) & PAGE_MASK,
2136 /* clear the page */
2137 dma_pte_clear_range(domain, start_addr,
2138 start_addr + offset);
2139 /* free page tables */
2140 dma_pte_free_pagetable(domain, start_addr,
2141 start_addr + offset);
2143 __free_iova(&domain->iovad, iova);
2146 sg->dma_address = start_addr + offset +
2147 ((u64)addr & (~PAGE_MASK));
2148 sg->dma_length = sg->length;
2152 /* it's a non-present to present mapping */
2153 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2154 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2155 iommu_flush_write_buffer(domain->iommu);
2159 static struct dma_mapping_ops intel_dma_ops = {
2160 .alloc_coherent = intel_alloc_coherent,
2161 .free_coherent = intel_free_coherent,
2162 .map_single = intel_map_single,
2163 .unmap_single = intel_unmap_single,
2164 .map_sg = intel_map_sg,
2165 .unmap_sg = intel_unmap_sg,
2168 static inline int iommu_domain_cache_init(void)
2172 iommu_domain_cache = kmem_cache_create("iommu_domain",
2173 sizeof(struct dmar_domain),
2178 if (!iommu_domain_cache) {
2179 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2186 static inline int iommu_devinfo_cache_init(void)
2190 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2191 sizeof(struct device_domain_info),
2195 if (!iommu_devinfo_cache) {
2196 printk(KERN_ERR "Couldn't create devinfo cache\n");
2203 static inline int iommu_iova_cache_init(void)
2207 iommu_iova_cache = kmem_cache_create("iommu_iova",
2208 sizeof(struct iova),
2212 if (!iommu_iova_cache) {
2213 printk(KERN_ERR "Couldn't create iova cache\n");
2220 static int __init iommu_init_mempool(void)
2223 ret = iommu_iova_cache_init();
2227 ret = iommu_domain_cache_init();
2231 ret = iommu_devinfo_cache_init();
2235 kmem_cache_destroy(iommu_domain_cache);
2237 kmem_cache_destroy(iommu_iova_cache);
2242 static void __init iommu_exit_mempool(void)
2244 kmem_cache_destroy(iommu_devinfo_cache);
2245 kmem_cache_destroy(iommu_domain_cache);
2246 kmem_cache_destroy(iommu_iova_cache);
2250 static void __init init_no_remapping_devices(void)
2252 struct dmar_drhd_unit *drhd;
2254 for_each_drhd_unit(drhd) {
2255 if (!drhd->include_all) {
2257 for (i = 0; i < drhd->devices_cnt; i++)
2258 if (drhd->devices[i] != NULL)
2260 /* ignore DMAR unit if no pci devices exist */
2261 if (i == drhd->devices_cnt)
2269 for_each_drhd_unit(drhd) {
2271 if (drhd->ignored || drhd->include_all)
2274 for (i = 0; i < drhd->devices_cnt; i++)
2275 if (drhd->devices[i] &&
2276 !IS_GFX_DEVICE(drhd->devices[i]))
2279 if (i < drhd->devices_cnt)
2282 /* bypass IOMMU if it is just for gfx devices */
2284 for (i = 0; i < drhd->devices_cnt; i++) {
2285 if (!drhd->devices[i])
2287 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2292 int __init intel_iommu_init(void)
2296 if (dmar_table_init())
2299 if (dmar_dev_scope_init())
2303 * Check the need for DMA-remapping initialization now.
2304 * Above initialization will also be used by Interrupt-remapping.
2306 if (no_iommu || swiotlb || dmar_disabled)
2309 iommu_init_mempool();
2310 dmar_init_reserved_ranges();
2312 init_no_remapping_devices();
2316 printk(KERN_ERR "IOMMU: dmar init failed\n");
2317 put_iova_domain(&reserved_iova_list);
2318 iommu_exit_mempool();
2322 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2324 init_timer(&unmap_timer);
2326 dma_ops = &intel_dma_ops;
2330 void intel_iommu_domain_exit(struct dmar_domain *domain)
2334 /* Domain 0 is reserved, so dont process it */
2338 end = DOMAIN_MAX_ADDR(domain->gaw);
2339 end = end & (~VTD_PAGE_MASK);
2342 dma_pte_clear_range(domain, 0, end);
2344 /* free page tables */
2345 dma_pte_free_pagetable(domain, 0, end);
2347 iommu_free_domain(domain);
2348 free_domain_mem(domain);
2350 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2352 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2354 struct dmar_drhd_unit *drhd;
2355 struct dmar_domain *domain;
2356 struct intel_iommu *iommu;
2358 drhd = dmar_find_matched_drhd_unit(pdev);
2360 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2364 iommu = drhd->iommu;
2367 "intel_iommu_domain_alloc: iommu == NULL\n");
2370 domain = iommu_alloc_domain(iommu);
2373 "intel_iommu_domain_alloc: domain == NULL\n");
2376 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2378 "intel_iommu_domain_alloc: domain_init() failed\n");
2379 intel_iommu_domain_exit(domain);
2384 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2386 int intel_iommu_context_mapping(
2387 struct dmar_domain *domain, struct pci_dev *pdev)
2390 rc = domain_context_mapping(domain, pdev);
2393 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2395 int intel_iommu_page_mapping(
2396 struct dmar_domain *domain, dma_addr_t iova,
2397 u64 hpa, size_t size, int prot)
2400 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2403 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2405 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2407 detach_domain_for_dev(domain, bus, devfn);
2409 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2411 struct dmar_domain *
2412 intel_iommu_find_domain(struct pci_dev *pdev)
2414 return find_domain(pdev);
2416 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2418 int intel_iommu_found(void)
2420 return g_num_of_iommus;
2422 EXPORT_SYMBOL_GPL(intel_iommu_found);
2424 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2426 struct dma_pte *pte;
2430 pte = addr_to_dma_pte(domain, iova);
2433 pfn = dma_pte_addr(*pte);
2435 return pfn >> VTD_PAGE_SHIFT;
2437 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);