drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) Ashok Raj <ashok.raj@intel.com>
  18  * Copyright (C) Shaohua Li <shaohua.li@intel.com>
  19  * Copyright (C) Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  20  */
  21
  22 #include <linux/init.h>
  23 #include <linux/bitmap.h>
  24 #include <linux/slab.h>
  25 #include <linux/irq.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/sysdev.h>
  28 #include <linux/spinlock.h>
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/dma-mapping.h>
  32 #include <linux/mempool.h>
  33 #include "iova.h"
  34 #include "intel-iommu.h"
  35 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
  36 #include <asm/cacheflush.h>
  37 #include <asm/iommu.h>
  38 #include "pci.h"
  39
  40 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  41 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  42
  43 #define IOAPIC_RANGE_START      (0xfee00000)
  44 #define IOAPIC_RANGE_END        (0xfeefffff)
  45 #define IOVA_START_ADDR         (0x1000)
  46
  47 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  48
  49 #define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
  50
  51 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  52
  53 static void domain_remove_dev_info(struct dmar_domain *domain);
  54
  55 static int dmar_disabled;
  56 static int __initdata dmar_map_gfx = 1;
  57
  58 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
  59 static DEFINE_SPINLOCK(device_domain_lock);
  60 static LIST_HEAD(device_domain_list);
  61
  62 static int __init intel_iommu_setup(char *str)
  63 {
  64         if (!str)
  65                 return -EINVAL;
  66         while (*str) {
  67                 if (!strncmp(str, "off", 3)) {
  68                         dmar_disabled = 1;
  69                         printk(KERN_INFO"Intel-IOMMU: disabled\n");
  70                 } else if (!strncmp(str, "igfx_off", 8)) {
  71                         dmar_map_gfx = 0;
  72                         printk(KERN_INFO
  73                                 "Intel-IOMMU: disable GFX device mapping\n");
  74                 }
  75
  76                 str += strcspn(str, ",");
  77                 while (*str == ',')
  78                         str++;
  79         }
  80         return 0;
  81 }
  82 __setup("intel_iommu=", intel_iommu_setup);
  83
  84 static struct kmem_cache *iommu_domain_cache;
  85 static struct kmem_cache *iommu_devinfo_cache;
  86 static struct kmem_cache *iommu_iova_cache;
  87
  88 static inline void *alloc_pgtable_page(void)
  89 {
  90         return (void *)get_zeroed_page(GFP_ATOMIC);
  91 }
  92
  93 static inline void free_pgtable_page(void *vaddr)
  94 {
  95         free_page((unsigned long)vaddr);
  96 }
  97
  98 static inline void *alloc_domain_mem(void)
  99 {
 100         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 101 }
 102
 103 static inline void free_domain_mem(void *vaddr)
 104 {
 105         kmem_cache_free(iommu_domain_cache, vaddr);
 106 }
 107
 108 static inline void * alloc_devinfo_mem(void)
 109 {
 110         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 111 }
 112
 113 static inline void free_devinfo_mem(void *vaddr)
 114 {
 115         kmem_cache_free(iommu_devinfo_cache, vaddr);
 116 }
 117
 118 struct iova *alloc_iova_mem(void)
 119 {
 120         return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 121 }
 122
 123 void free_iova_mem(struct iova *iova)
 124 {
 125         kmem_cache_free(iommu_iova_cache, iova);
 126 }
 127
 128 static inline void __iommu_flush_cache(
 129         struct intel_iommu *iommu, void *addr, int size)
 130 {
 131         if (!ecap_coherent(iommu->ecap))
 132                 clflush_cache_range(addr, size);
 133 }
 134
 135 /* Gets context entry for a given bus and devfn */
 136 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 137                 u8 bus, u8 devfn)
 138 {
 139         struct root_entry *root;
 140         struct context_entry *context;
 141         unsigned long phy_addr;
 142         unsigned long flags;
 143
 144         spin_lock_irqsave(&iommu->lock, flags);
 145         root = &iommu->root_entry[bus];
 146         context = get_context_addr_from_root(root);
 147         if (!context) {
 148                 context = (struct context_entry *)alloc_pgtable_page();
 149                 if (!context) {
 150                         spin_unlock_irqrestore(&iommu->lock, flags);
 151                         return NULL;
 152                 }
 153                 __iommu_flush_cache(iommu, (void *)context, PAGE_SIZE_4K);
 154                 phy_addr = virt_to_phys((void *)context);
 155                 set_root_value(root, phy_addr);
 156                 set_root_present(root);
 157                 __iommu_flush_cache(iommu, root, sizeof(*root));
 158         }
 159         spin_unlock_irqrestore(&iommu->lock, flags);
 160         return &context[devfn];
 161 }
 162
 163 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 164 {
 165         struct root_entry *root;
 166         struct context_entry *context;
 167         int ret;
 168         unsigned long flags;
 169
 170         spin_lock_irqsave(&iommu->lock, flags);
 171         root = &iommu->root_entry[bus];
 172         context = get_context_addr_from_root(root);
 173         if (!context) {
 174                 ret = 0;
 175                 goto out;
 176         }
 177         ret = context_present(context[devfn]);
 178 out:
 179         spin_unlock_irqrestore(&iommu->lock, flags);
 180         return ret;
 181 }
 182
 183 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 184 {
 185         struct root_entry *root;
 186         struct context_entry *context;
 187         unsigned long flags;
 188
 189         spin_lock_irqsave(&iommu->lock, flags);
 190         root = &iommu->root_entry[bus];
 191         context = get_context_addr_from_root(root);
 192         if (context) {
 193                 context_clear_entry(context[devfn]);
 194                 __iommu_flush_cache(iommu, &context[devfn], \
 195                         sizeof(*context));
 196         }
 197         spin_unlock_irqrestore(&iommu->lock, flags);
 198 }
 199
 200 static void free_context_table(struct intel_iommu *iommu)
 201 {
 202         struct root_entry *root;
 203         int i;
 204         unsigned long flags;
 205         struct context_entry *context;
 206
 207         spin_lock_irqsave(&iommu->lock, flags);
 208         if (!iommu->root_entry) {
 209                 goto out;
 210         }
 211         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 212                 root = &iommu->root_entry[i];
 213                 context = get_context_addr_from_root(root);
 214                 if (context)
 215                         free_pgtable_page(context);
 216         }
 217         free_pgtable_page(iommu->root_entry);
 218         iommu->root_entry = NULL;
 219 out:
 220         spin_unlock_irqrestore(&iommu->lock, flags);
 221 }
 222
 223 /* page table handling */
 224 #define LEVEL_STRIDE            (9)
 225 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 226
 227 static inline int agaw_to_level(int agaw)
 228 {
 229         return agaw + 2;
 230 }
 231
 232 static inline int agaw_to_width(int agaw)
 233 {
 234         return 30 + agaw * LEVEL_STRIDE;
 235
 236 }
 237
 238 static inline int width_to_agaw(int width)
 239 {
 240         return (width - 30) / LEVEL_STRIDE;
 241 }
 242
 243 static inline unsigned int level_to_offset_bits(int level)
 244 {
 245         return (12 + (level - 1) * LEVEL_STRIDE);
 246 }
 247
 248 static inline int address_level_offset(u64 addr, int level)
 249 {
 250         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 251 }
 252
 253 static inline u64 level_mask(int level)
 254 {
 255         return ((u64)-1 << level_to_offset_bits(level));
 256 }
 257
 258 static inline u64 level_size(int level)
 259 {
 260         return ((u64)1 << level_to_offset_bits(level));
 261 }
 262
 263 static inline u64 align_to_level(u64 addr, int level)
 264 {
 265         return ((addr + level_size(level) - 1) & level_mask(level));
 266 }
 267
 268 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 269 {
 270         int addr_width = agaw_to_width(domain->agaw);
 271         struct dma_pte *parent, *pte = NULL;
 272         int level = agaw_to_level(domain->agaw);
 273         int offset;
 274         unsigned long flags;
 275
 276         BUG_ON(!domain->pgd);
 277
 278         addr &= (((u64)1) << addr_width) - 1;
 279         parent = domain->pgd;
 280
 281         spin_lock_irqsave(&domain->mapping_lock, flags);
 282         while (level > 0) {
 283                 void *tmp_page;
 284
 285                 offset = address_level_offset(addr, level);
 286                 pte = &parent[offset];
 287                 if (level == 1)
 288                         break;
 289
 290                 if (!dma_pte_present(*pte)) {
 291                         tmp_page = alloc_pgtable_page();
 292
 293                         if (!tmp_page) {
 294                                 spin_unlock_irqrestore(&domain->mapping_lock,
 295                                         flags);
 296                                 return NULL;
 297                         }
 298                         __iommu_flush_cache(domain->iommu, tmp_page,
 299                                         PAGE_SIZE_4K);
 300                         dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
 301                         /*
 302                          * high level table always sets r/w, last level page
 303                          * table control read/write
 304                          */
 305                         dma_set_pte_readable(*pte);
 306                         dma_set_pte_writable(*pte);
 307                         __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 308                 }
 309                 parent = phys_to_virt(dma_pte_addr(*pte));
 310                 level--;
 311         }
 312
 313         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 314         return pte;
 315 }
 316
 317 /* return address's pte at specific level */
 318 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 319                 int level)
 320 {
 321         struct dma_pte *parent, *pte = NULL;
 322         int total = agaw_to_level(domain->agaw);
 323         int offset;
 324
 325         parent = domain->pgd;
 326         while (level <= total) {
 327                 offset = address_level_offset(addr, total);
 328                 pte = &parent[offset];
 329                 if (level == total)
 330                         return pte;
 331
 332                 if (!dma_pte_present(*pte))
 333                         break;
 334                 parent = phys_to_virt(dma_pte_addr(*pte));
 335                 total--;
 336         }
 337         return NULL;
 338 }
 339
 340 /* clear one page's page table */
 341 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 342 {
 343         struct dma_pte *pte = NULL;
 344
 345         /* get last level pte */
 346         pte = dma_addr_level_pte(domain, addr, 1);
 347
 348         if (pte) {
 349                 dma_clear_pte(*pte);
 350                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 351         }
 352 }
 353
 354 /* clear last level pte, a tlb flush should be followed */
 355 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 356 {
 357         int addr_width = agaw_to_width(domain->agaw);
 358
 359         start &= (((u64)1) << addr_width) - 1;
 360         end &= (((u64)1) << addr_width) - 1;
 361         /* in case it's partial page */
 362         start = PAGE_ALIGN_4K(start);
 363         end &= PAGE_MASK_4K;
 364
 365         /* we don't need lock here, nobody else touches the iova range */
 366         while (start < end) {
 367                 dma_pte_clear_one(domain, start);
 368                 start += PAGE_SIZE_4K;
 369         }
 370 }
 371
 372 /* free page table pages. last level pte should already be cleared */
 373 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 374         u64 start, u64 end)
 375 {
 376         int addr_width = agaw_to_width(domain->agaw);
 377         struct dma_pte *pte;
 378         int total = agaw_to_level(domain->agaw);
 379         int level;
 380         u64 tmp;
 381
 382         start &= (((u64)1) << addr_width) - 1;
 383         end &= (((u64)1) << addr_width) - 1;
 384
 385         /* we don't need lock here, nobody else touches the iova range */
 386         level = 2;
 387         while (level <= total) {
 388                 tmp = align_to_level(start, level);
 389                 if (tmp >= end || (tmp + level_size(level) > end))
 390                         return;
 391
 392                 while (tmp < end) {
 393                         pte = dma_addr_level_pte(domain, tmp, level);
 394                         if (pte) {
 395                                 free_pgtable_page(
 396                                         phys_to_virt(dma_pte_addr(*pte)));
 397                                 dma_clear_pte(*pte);
 398                                 __iommu_flush_cache(domain->iommu,
 399                                                 pte, sizeof(*pte));
 400                         }
 401                         tmp += level_size(level);
 402                 }
 403                 level++;
 404         }
 405         /* free pgd */
 406         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 407                 free_pgtable_page(domain->pgd);
 408                 domain->pgd = NULL;
 409         }
 410 }
 411
 412 /* iommu handling */
 413 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 414 {
 415         struct root_entry *root;
 416         unsigned long flags;
 417
 418         root = (struct root_entry *)alloc_pgtable_page();
 419         if (!root)
 420                 return -ENOMEM;
 421
 422         __iommu_flush_cache(iommu, root, PAGE_SIZE_4K);
 423
 424         spin_lock_irqsave(&iommu->lock, flags);
 425         iommu->root_entry = root;
 426         spin_unlock_irqrestore(&iommu->lock, flags);
 427
 428         return 0;
 429 }
 430
 431 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
 432 {\
 433         unsigned long start_time = jiffies;\
 434         while (1) {\
 435                 sts = op (iommu->reg + offset);\
 436                 if (cond)\
 437                         break;\
 438                 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
 439                         panic("DMAR hardware is malfunctioning\n");\
 440                 cpu_relax();\
 441         }\
 442 }
 443
 444 static void iommu_set_root_entry(struct intel_iommu *iommu)
 445 {
 446         void *addr;
 447         u32 cmd, sts;
 448         unsigned long flag;
 449
 450         addr = iommu->root_entry;
 451
 452         spin_lock_irqsave(&iommu->register_lock, flag);
 453         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 454
 455         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 456         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 457
 458         /* Make sure hardware complete it */
 459         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 460                 readl, (sts & DMA_GSTS_RTPS), sts);
 461
 462         spin_unlock_irqrestore(&iommu->register_lock, flag);
 463 }
 464
 465 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 466 {
 467         u32 val;
 468         unsigned long flag;
 469
 470         if (!cap_rwbf(iommu->cap))
 471                 return;
 472         val = iommu->gcmd | DMA_GCMD_WBF;
 473
 474         spin_lock_irqsave(&iommu->register_lock, flag);
 475         writel(val, iommu->reg + DMAR_GCMD_REG);
 476
 477         /* Make sure hardware complete it */
 478         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 479                         readl, (!(val & DMA_GSTS_WBFS)), val);
 480
 481         spin_unlock_irqrestore(&iommu->register_lock, flag);
 482 }
 483
 484 /* return value determine if we need a write buffer flush */
 485 static int __iommu_flush_context(struct intel_iommu *iommu,
 486         u16 did, u16 source_id, u8 function_mask, u64 type,
 487         int non_present_entry_flush)
 488 {
 489         u64 val = 0;
 490         unsigned long flag;
 491
 492         /*
 493          * In the non-present entry flush case, if hardware doesn't cache
 494          * non-present entry we do nothing and if hardware cache non-present
 495          * entry, we flush entries of domain 0 (the domain id is used to cache
 496          * any non-present entries)
 497          */
 498         if (non_present_entry_flush) {
 499                 if (!cap_caching_mode(iommu->cap))
 500                         return 1;
 501                 else
 502                         did = 0;
 503         }
 504
 505         switch (type) {
 506         case DMA_CCMD_GLOBAL_INVL:
 507                 val = DMA_CCMD_GLOBAL_INVL;
 508                 break;
 509         case DMA_CCMD_DOMAIN_INVL:
 510                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 511                 break;
 512         case DMA_CCMD_DEVICE_INVL:
 513                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 514                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 515                 break;
 516         default:
 517                 BUG();
 518         }
 519         val |= DMA_CCMD_ICC;
 520
 521         spin_lock_irqsave(&iommu->register_lock, flag);
 522         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 523
 524         /* Make sure hardware complete it */
 525         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 526                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 527
 528         spin_unlock_irqrestore(&iommu->register_lock, flag);
 529
 530         /* flush context entry will implictly flush write buffer */
 531         return 0;
 532 }
 533
 534 static int inline iommu_flush_context_global(struct intel_iommu *iommu,
 535         int non_present_entry_flush)
 536 {
 537         return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
 538                 non_present_entry_flush);
 539 }
 540
 541 static int inline iommu_flush_context_domain(struct intel_iommu *iommu, u16 did,
 542         int non_present_entry_flush)
 543 {
 544         return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
 545                 non_present_entry_flush);
 546 }
 547
 548 static int inline iommu_flush_context_device(struct intel_iommu *iommu,
 549         u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
 550 {
 551         return __iommu_flush_context(iommu, did, source_id, function_mask,
 552                 DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
 553 }
 554
 555 /* return value determine if we need a write buffer flush */
 556 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 557         u64 addr, unsigned int size_order, u64 type,
 558         int non_present_entry_flush)
 559 {
 560         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 561         u64 val = 0, val_iva = 0;
 562         unsigned long flag;
 563
 564         /*
 565          * In the non-present entry flush case, if hardware doesn't cache
 566          * non-present entry we do nothing and if hardware cache non-present
 567          * entry, we flush entries of domain 0 (the domain id is used to cache
 568          * any non-present entries)
 569          */
 570         if (non_present_entry_flush) {
 571                 if (!cap_caching_mode(iommu->cap))
 572                         return 1;
 573                 else
 574                         did = 0;
 575         }
 576
 577         switch (type) {
 578         case DMA_TLB_GLOBAL_FLUSH:
 579                 /* global flush doesn't need set IVA_REG */
 580                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 581                 break;
 582         case DMA_TLB_DSI_FLUSH:
 583                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 584                 break;
 585         case DMA_TLB_PSI_FLUSH:
 586                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 587                 /* Note: always flush non-leaf currently */
 588                 val_iva = size_order | addr;
 589                 break;
 590         default:
 591                 BUG();
 592         }
 593         /* Note: set drain read/write */
 594 #if 0
 595         /*
 596          * This is probably to be super secure.. Looks like we can
 597          * ignore it without any impact.
 598          */
 599         if (cap_read_drain(iommu->cap))
 600                 val |= DMA_TLB_READ_DRAIN;
 601 #endif
 602         if (cap_write_drain(iommu->cap))
 603                 val |= DMA_TLB_WRITE_DRAIN;
 604
 605         spin_lock_irqsave(&iommu->register_lock, flag);
 606         /* Note: Only uses first TLB reg currently */
 607         if (val_iva)
 608                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 609         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 610
 611         /* Make sure hardware complete it */
 612         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 613                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 614
 615         spin_unlock_irqrestore(&iommu->register_lock, flag);
 616
 617         /* check IOTLB invalidation granularity */
 618         if (DMA_TLB_IAIG(val) == 0)
 619                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 620         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 621                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 622                         DMA_TLB_IIRG(type), DMA_TLB_IAIG(val));
 623         /* flush context entry will implictly flush write buffer */
 624         return 0;
 625 }
 626
 627 static int inline iommu_flush_iotlb_global(struct intel_iommu *iommu,
 628         int non_present_entry_flush)
 629 {
 630         return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
 631                 non_present_entry_flush);
 632 }
 633
 634 static int inline iommu_flush_iotlb_dsi(struct intel_iommu *iommu, u16 did,
 635         int non_present_entry_flush)
 636 {
 637         return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
 638                 non_present_entry_flush);
 639 }
 640
 641 static int iommu_get_alignment(u64 base, unsigned int size)
 642 {
 643         int t = 0;
 644         u64 end;
 645
 646         end = base + size - 1;
 647         while (base != end) {
 648                 t++;
 649                 base >>= 1;
 650                 end >>= 1;
 651         }
 652         return t;
 653 }
 654
 655 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 656         u64 addr, unsigned int pages, int non_present_entry_flush)
 657 {
 658         unsigned int align;
 659
 660         BUG_ON(addr & (~PAGE_MASK_4K));
 661         BUG_ON(pages == 0);
 662
 663         /* Fallback to domain selective flush if no PSI support */
 664         if (!cap_pgsel_inv(iommu->cap))
 665                 return iommu_flush_iotlb_dsi(iommu, did,
 666                         non_present_entry_flush);
 667
 668         /*
 669          * PSI requires page size to be 2 ^ x, and the base address is naturally
 670          * aligned to the size
 671          */
 672         align = iommu_get_alignment(addr >> PAGE_SHIFT_4K, pages);
 673         /* Fallback to domain selective flush if size is too big */
 674         if (align > cap_max_amask_val(iommu->cap))
 675                 return iommu_flush_iotlb_dsi(iommu, did,
 676                         non_present_entry_flush);
 677
 678         addr >>= PAGE_SHIFT_4K + align;
 679         addr <<= PAGE_SHIFT_4K + align;
 680
 681         return __iommu_flush_iotlb(iommu, did, addr, align,
 682                 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
 683 }
 684
 685 static int iommu_enable_translation(struct intel_iommu *iommu)
 686 {
 687         u32 sts;
 688         unsigned long flags;
 689
 690         spin_lock_irqsave(&iommu->register_lock, flags);
 691         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
 692
 693         /* Make sure hardware complete it */
 694         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 695                 readl, (sts & DMA_GSTS_TES), sts);
 696
 697         iommu->gcmd |= DMA_GCMD_TE;
 698         spin_unlock_irqrestore(&iommu->register_lock, flags);
 699         return 0;
 700 }
 701
 702 static int iommu_disable_translation(struct intel_iommu *iommu)
 703 {
 704         u32 sts;
 705         unsigned long flag;
 706
 707         spin_lock_irqsave(&iommu->register_lock, flag);
 708         iommu->gcmd &= ~DMA_GCMD_TE;
 709         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 710
 711         /* Make sure hardware complete it */
 712         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 713                 readl, (!(sts & DMA_GSTS_TES)), sts);
 714
 715         spin_unlock_irqrestore(&iommu->register_lock, flag);
 716         return 0;
 717 }
 718
 719 static int iommu_init_domains(struct intel_iommu *iommu)
 720 {
 721         unsigned long ndomains;
 722         unsigned long nlongs;
 723
 724         ndomains = cap_ndoms(iommu->cap);
 725         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
 726         nlongs = BITS_TO_LONGS(ndomains);
 727
 728         /* TBD: there might be 64K domains,
 729          * consider other allocation for future chip
 730          */
 731         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
 732         if (!iommu->domain_ids) {
 733                 printk(KERN_ERR "Allocating domain id array failed\n");
 734                 return -ENOMEM;
 735         }
 736         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
 737                         GFP_KERNEL);
 738         if (!iommu->domains) {
 739                 printk(KERN_ERR "Allocating domain array failed\n");
 740                 kfree(iommu->domain_ids);
 741                 return -ENOMEM;
 742         }
 743
 744         /*
 745          * if Caching mode is set, then invalid translations are tagged
 746          * with domainid 0. Hence we need to pre-allocate it.
 747          */
 748         if (cap_caching_mode(iommu->cap))
 749                 set_bit(0, iommu->domain_ids);
 750         return 0;
 751 }
 752
 753 static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
 754 {
 755         struct intel_iommu *iommu;
 756         int ret;
 757         int map_size;
 758         u32 ver;
 759
 760         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 761         if (!iommu)
 762                 return NULL;
 763         iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
 764         if (!iommu->reg) {
 765                 printk(KERN_ERR "IOMMU: can't map the region\n");
 766                 goto error;
 767         }
 768         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 769         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 770
 771         /* the registers might be more than one page */
 772         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 773                 cap_max_fault_reg_offset(iommu->cap));
 774         map_size = PAGE_ALIGN_4K(map_size);
 775         if (map_size > PAGE_SIZE_4K) {
 776                 iounmap(iommu->reg);
 777                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 778                 if (!iommu->reg) {
 779                         printk(KERN_ERR "IOMMU: can't map the region\n");
 780                         goto error;
 781                 }
 782         }
 783
 784         ver = readl(iommu->reg + DMAR_VER_REG);
 785         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 786                 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 787                 iommu->cap, iommu->ecap);
 788         ret = iommu_init_domains(iommu);
 789         if (ret)
 790                 goto error_unmap;
 791         spin_lock_init(&iommu->lock);
 792         spin_lock_init(&iommu->register_lock);
 793
 794         drhd->iommu = iommu;
 795         return iommu;
 796 error_unmap:
 797         iounmap(iommu->reg);
 798         iommu->reg = 0;
 799 error:
 800         kfree(iommu);
 801         return NULL;
 802 }
 803
 804 static void domain_exit(struct dmar_domain *domain);
 805 static void free_iommu(struct intel_iommu *iommu)
 806 {
 807         struct dmar_domain *domain;
 808         int i;
 809
 810         if (!iommu)
 811                 return;
 812
 813         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
 814         for (; i < cap_ndoms(iommu->cap); ) {
 815                 domain = iommu->domains[i];
 816                 clear_bit(i, iommu->domain_ids);
 817                 domain_exit(domain);
 818                 i = find_next_bit(iommu->domain_ids,
 819                         cap_ndoms(iommu->cap), i+1);
 820         }
 821
 822         if (iommu->gcmd & DMA_GCMD_TE)
 823                 iommu_disable_translation(iommu);
 824
 825         if (iommu->irq) {
 826                 set_irq_data(iommu->irq, NULL);
 827                 /* This will mask the irq */
 828                 free_irq(iommu->irq, iommu);
 829                 destroy_irq(iommu->irq);
 830         }
 831
 832         kfree(iommu->domains);
 833         kfree(iommu->domain_ids);
 834
 835         /* free context mapping */
 836         free_context_table(iommu);
 837
 838         if (iommu->reg)
 839                 iounmap(iommu->reg);
 840         kfree(iommu);
 841 }
 842
 843 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
 844 {
 845         unsigned long num;
 846         unsigned long ndomains;
 847         struct dmar_domain *domain;
 848         unsigned long flags;
 849
 850         domain = alloc_domain_mem();
 851         if (!domain)
 852                 return NULL;
 853
 854         ndomains = cap_ndoms(iommu->cap);
 855
 856         spin_lock_irqsave(&iommu->lock, flags);
 857         num = find_first_zero_bit(iommu->domain_ids, ndomains);
 858         if (num >= ndomains) {
 859                 spin_unlock_irqrestore(&iommu->lock, flags);
 860                 free_domain_mem(domain);
 861                 printk(KERN_ERR "IOMMU: no free domain ids\n");
 862                 return NULL;
 863         }
 864
 865         set_bit(num, iommu->domain_ids);
 866         domain->id = num;
 867         domain->iommu = iommu;
 868         iommu->domains[num] = domain;
 869         spin_unlock_irqrestore(&iommu->lock, flags);
 870
 871         return domain;
 872 }
 873
 874 static void iommu_free_domain(struct dmar_domain *domain)
 875 {
 876         unsigned long flags;
 877
 878         spin_lock_irqsave(&domain->iommu->lock, flags);
 879         clear_bit(domain->id, domain->iommu->domain_ids);
 880         spin_unlock_irqrestore(&domain->iommu->lock, flags);
 881 }
 882
 883 static struct iova_domain reserved_iova_list;
 884
 885 static void dmar_init_reserved_ranges(void)
 886 {
 887         struct pci_dev *pdev = NULL;
 888         struct iova *iova;
 889         int i;
 890         u64 addr, size;
 891
 892         init_iova_domain(&reserved_iova_list);
 893
 894         /* IOAPIC ranges shouldn't be accessed by DMA */
 895         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
 896                 IOVA_PFN(IOAPIC_RANGE_END));
 897         if (!iova)
 898                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
 899
 900         /* Reserve all PCI MMIO to avoid peer-to-peer access */
 901         for_each_pci_dev(pdev) {
 902                 struct resource *r;
 903
 904                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
 905                         r = &pdev->resource[i];
 906                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
 907                                 continue;
 908                         addr = r->start;
 909                         addr &= PAGE_MASK_4K;
 910                         size = r->end - addr;
 911                         size = PAGE_ALIGN_4K(size);
 912                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
 913                                 IOVA_PFN(size + addr) - 1);
 914                         if (!iova)
 915                                 printk(KERN_ERR "Reserve iova failed\n");
 916                 }
 917         }
 918
 919 }
 920
 921 static void domain_reserve_special_ranges(struct dmar_domain *domain)
 922 {
 923         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
 924 }
 925
 926 static inline int guestwidth_to_adjustwidth(int gaw)
 927 {
 928         int agaw;
 929         int r = (gaw - 12) % 9;
 930
 931         if (r == 0)
 932                 agaw = gaw;
 933         else
 934                 agaw = gaw + 9 - r;
 935         if (agaw > 64)
 936                 agaw = 64;
 937         return agaw;
 938 }
 939
 940 static int domain_init(struct dmar_domain *domain, int guest_width)
 941 {
 942         struct intel_iommu *iommu;
 943         int adjust_width, agaw;
 944         unsigned long sagaw;
 945
 946         init_iova_domain(&domain->iovad);
 947         spin_lock_init(&domain->mapping_lock);
 948
 949         domain_reserve_special_ranges(domain);
 950
 951         /* calculate AGAW */
 952         iommu = domain->iommu;
 953         if (guest_width > cap_mgaw(iommu->cap))
 954                 guest_width = cap_mgaw(iommu->cap);
 955         domain->gaw = guest_width;
 956         adjust_width = guestwidth_to_adjustwidth(guest_width);
 957         agaw = width_to_agaw(adjust_width);
 958         sagaw = cap_sagaw(iommu->cap);
 959         if (!test_bit(agaw, &sagaw)) {
 960                 /* hardware doesn't support it, choose a bigger one */
 961                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
 962                 agaw = find_next_bit(&sagaw, 5, agaw);
 963                 if (agaw >= 5)
 964                         return -ENODEV;
 965         }
 966         domain->agaw = agaw;
 967         INIT_LIST_HEAD(&domain->devices);
 968
 969         /* always allocate the top pgd */
 970         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
 971         if (!domain->pgd)
 972                 return -ENOMEM;
 973         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE_4K);
 974         return 0;
 975 }
 976
 977 static void domain_exit(struct dmar_domain *domain)
 978 {
 979         u64 end;
 980
 981         /* Domain 0 is reserved, so dont process it */
 982         if (!domain)
 983                 return;
 984
 985         domain_remove_dev_info(domain);
 986         /* destroy iovas */
 987         put_iova_domain(&domain->iovad);
 988         end = DOMAIN_MAX_ADDR(domain->gaw);
 989         end = end & (~PAGE_MASK_4K);
 990
 991         /* clear ptes */
 992         dma_pte_clear_range(domain, 0, end);
 993
 994         /* free page tables */
 995         dma_pte_free_pagetable(domain, 0, end);
 996
 997         iommu_free_domain(domain);
 998         free_domain_mem(domain);
 999 }
1000
1001 static int domain_context_mapping_one(struct dmar_domain *domain,
1002                 u8 bus, u8 devfn)
1003 {
1004         struct context_entry *context;
1005         struct intel_iommu *iommu = domain->iommu;
1006         unsigned long flags;
1007
1008         pr_debug("Set context mapping for %02x:%02x.%d\n",
1009                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1010         BUG_ON(!domain->pgd);
1011         context = device_to_context_entry(iommu, bus, devfn);
1012         if (!context)
1013                 return -ENOMEM;
1014         spin_lock_irqsave(&iommu->lock, flags);
1015         if (context_present(*context)) {
1016                 spin_unlock_irqrestore(&iommu->lock, flags);
1017                 return 0;
1018         }
1019
1020         context_set_domain_id(*context, domain->id);
1021         context_set_address_width(*context, domain->agaw);
1022         context_set_address_root(*context, virt_to_phys(domain->pgd));
1023         context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1024         context_set_fault_enable(*context);
1025         context_set_present(*context);
1026         __iommu_flush_cache(iommu, context, sizeof(*context));
1027
1028         /* it's a non-present to present mapping */
1029         if (iommu_flush_context_device(iommu, domain->id,
1030                         (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, 1))
1031                 iommu_flush_write_buffer(iommu);
1032         else
1033                 iommu_flush_iotlb_dsi(iommu, 0, 0);
1034         spin_unlock_irqrestore(&iommu->lock, flags);
1035         return 0;
1036 }
1037
1038 static int
1039 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1040 {
1041         int ret;
1042         struct pci_dev *tmp, *parent;
1043
1044         ret = domain_context_mapping_one(domain, pdev->bus->number,
1045                 pdev->devfn);
1046         if (ret)
1047                 return ret;
1048
1049         /* dependent device mapping */
1050         tmp = pci_find_upstream_pcie_bridge(pdev);
1051         if (!tmp)
1052                 return 0;
1053         /* Secondary interface's bus number and devfn 0 */
1054         parent = pdev->bus->self;
1055         while (parent != tmp) {
1056                 ret = domain_context_mapping_one(domain, parent->bus->number,
1057                         parent->devfn);
1058                 if (ret)
1059                         return ret;
1060                 parent = parent->bus->self;
1061         }
1062         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1063                 return domain_context_mapping_one(domain,
1064                         tmp->subordinate->number, 0);
1065         else /* this is a legacy PCI bridge */
1066                 return domain_context_mapping_one(domain,
1067                         tmp->bus->number, tmp->devfn);
1068 }
1069
1070 static int domain_context_mapped(struct dmar_domain *domain,
1071         struct pci_dev *pdev)
1072 {
1073         int ret;
1074         struct pci_dev *tmp, *parent;
1075
1076         ret = device_context_mapped(domain->iommu,
1077                 pdev->bus->number, pdev->devfn);
1078         if (!ret)
1079                 return ret;
1080         /* dependent device mapping */
1081         tmp = pci_find_upstream_pcie_bridge(pdev);
1082         if (!tmp)
1083                 return ret;
1084         /* Secondary interface's bus number and devfn 0 */
1085         parent = pdev->bus->self;
1086         while (parent != tmp) {
1087                 ret = device_context_mapped(domain->iommu, parent->bus->number,
1088                         parent->devfn);
1089                 if (!ret)
1090                         return ret;
1091                 parent = parent->bus->self;
1092         }
1093         if (tmp->is_pcie)
1094                 return device_context_mapped(domain->iommu,
1095                         tmp->subordinate->number, 0);
1096         else
1097                 return device_context_mapped(domain->iommu,
1098                         tmp->bus->number, tmp->devfn);
1099 }
1100
1101 static int
1102 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1103                         u64 hpa, size_t size, int prot)
1104 {
1105         u64 start_pfn, end_pfn;
1106         struct dma_pte *pte;
1107         int index;
1108
1109         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1110                 return -EINVAL;
1111         iova &= PAGE_MASK_4K;
1112         start_pfn = ((u64)hpa) >> PAGE_SHIFT_4K;
1113         end_pfn = (PAGE_ALIGN_4K(((u64)hpa) + size)) >> PAGE_SHIFT_4K;
1114         index = 0;
1115         while (start_pfn < end_pfn) {
1116                 pte = addr_to_dma_pte(domain, iova + PAGE_SIZE_4K * index);
1117                 if (!pte)
1118                         return -ENOMEM;
1119                 /* We don't need lock here, nobody else
1120                  * touches the iova range
1121                  */
1122                 BUG_ON(dma_pte_addr(*pte));
1123                 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1124                 dma_set_pte_prot(*pte, prot);
1125                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1126                 start_pfn++;
1127                 index++;
1128         }
1129         return 0;
1130 }
1131
1132 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1133 {
1134         clear_context_table(domain->iommu, bus, devfn);
1135         iommu_flush_context_global(domain->iommu, 0);
1136         iommu_flush_iotlb_global(domain->iommu, 0);
1137 }
1138
1139 static void domain_remove_dev_info(struct dmar_domain *domain)
1140 {
1141         struct device_domain_info *info;
1142         unsigned long flags;
1143
1144         spin_lock_irqsave(&device_domain_lock, flags);
1145         while (!list_empty(&domain->devices)) {
1146                 info = list_entry(domain->devices.next,
1147                         struct device_domain_info, link);
1148                 list_del(&info->link);
1149                 list_del(&info->global);
1150                 if (info->dev)
1151                         info->dev->sysdata = NULL;
1152                 spin_unlock_irqrestore(&device_domain_lock, flags);
1153
1154                 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1155                 free_devinfo_mem(info);
1156
1157                 spin_lock_irqsave(&device_domain_lock, flags);
1158         }
1159         spin_unlock_irqrestore(&device_domain_lock, flags);
1160 }
1161
1162 /*
1163  * find_domain
1164  * Note: we use struct pci_dev->sysdata stores the info
1165  */
1166 struct dmar_domain *
1167 find_domain(struct pci_dev *pdev)
1168 {
1169         struct device_domain_info *info;
1170
1171         /* No lock here, assumes no domain exit in normal case */
1172         info = pdev->sysdata;
1173         if (info)
1174                 return info->domain;
1175         return NULL;
1176 }
1177
1178 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
1179      struct pci_dev *dev)
1180 {
1181         int index;
1182
1183         while (dev) {
1184                 for (index = 0; index < cnt; index ++)
1185                         if (dev == devices[index])
1186                                 return 1;
1187
1188                 /* Check our parent */
1189                 dev = dev->bus->self;
1190         }
1191
1192         return 0;
1193 }
1194
1195 static struct dmar_drhd_unit *
1196 dmar_find_matched_drhd_unit(struct pci_dev *dev)
1197 {
1198         struct dmar_drhd_unit *drhd = NULL;
1199
1200         list_for_each_entry(drhd, &dmar_drhd_units, list) {
1201                 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
1202                                                 drhd->devices_cnt, dev))
1203                         return drhd;
1204         }
1205
1206         return NULL;
1207 }
1208
1209 /* domain is initialized */
1210 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1211 {
1212         struct dmar_domain *domain, *found = NULL;
1213         struct intel_iommu *iommu;
1214         struct dmar_drhd_unit *drhd;
1215         struct device_domain_info *info, *tmp;
1216         struct pci_dev *dev_tmp;
1217         unsigned long flags;
1218         int bus = 0, devfn = 0;
1219
1220         domain = find_domain(pdev);
1221         if (domain)
1222                 return domain;
1223
1224         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1225         if (dev_tmp) {
1226                 if (dev_tmp->is_pcie) {
1227                         bus = dev_tmp->subordinate->number;
1228                         devfn = 0;
1229                 } else {
1230                         bus = dev_tmp->bus->number;
1231                         devfn = dev_tmp->devfn;
1232                 }
1233                 spin_lock_irqsave(&device_domain_lock, flags);
1234                 list_for_each_entry(info, &device_domain_list, global) {
1235                         if (info->bus == bus && info->devfn == devfn) {
1236                                 found = info->domain;
1237                                 break;
1238                         }
1239                 }
1240                 spin_unlock_irqrestore(&device_domain_lock, flags);
1241                 /* pcie-pci bridge already has a domain, uses it */
1242                 if (found) {
1243                         domain = found;
1244                         goto found_domain;
1245                 }
1246         }
1247
1248         /* Allocate new domain for the device */
1249         drhd = dmar_find_matched_drhd_unit(pdev);
1250         if (!drhd) {
1251                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1252                         pci_name(pdev));
1253                 return NULL;
1254         }
1255         iommu = drhd->iommu;
1256
1257         domain = iommu_alloc_domain(iommu);
1258         if (!domain)
1259                 goto error;
1260
1261         if (domain_init(domain, gaw)) {
1262                 domain_exit(domain);
1263                 goto error;
1264         }
1265
1266         /* register pcie-to-pci device */
1267         if (dev_tmp) {
1268                 info = alloc_devinfo_mem();
1269                 if (!info) {
1270                         domain_exit(domain);
1271                         goto error;
1272                 }
1273                 info->bus = bus;
1274                 info->devfn = devfn;
1275                 info->dev = NULL;
1276                 info->domain = domain;
1277                 /* This domain is shared by devices under p2p bridge */
1278                 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1279
1280                 /* pcie-to-pci bridge already has a domain, uses it */
1281                 found = NULL;
1282                 spin_lock_irqsave(&device_domain_lock, flags);
1283                 list_for_each_entry(tmp, &device_domain_list, global) {
1284                         if (tmp->bus == bus && tmp->devfn == devfn) {
1285                                 found = tmp->domain;
1286                                 break;
1287                         }
1288                 }
1289                 if (found) {
1290                         free_devinfo_mem(info);
1291                         domain_exit(domain);
1292                         domain = found;
1293                 } else {
1294                         list_add(&info->link, &domain->devices);
1295                         list_add(&info->global, &device_domain_list);
1296                 }
1297                 spin_unlock_irqrestore(&device_domain_lock, flags);
1298         }
1299
1300 found_domain:
1301         info = alloc_devinfo_mem();
1302         if (!info)
1303                 goto error;
1304         info->bus = pdev->bus->number;
1305         info->devfn = pdev->devfn;
1306         info->dev = pdev;
1307         info->domain = domain;
1308         spin_lock_irqsave(&device_domain_lock, flags);
1309         /* somebody is fast */
1310         found = find_domain(pdev);
1311         if (found != NULL) {
1312                 spin_unlock_irqrestore(&device_domain_lock, flags);
1313                 if (found != domain) {
1314                         domain_exit(domain);
1315                         domain = found;
1316                 }
1317                 free_devinfo_mem(info);
1318                 return domain;
1319         }
1320         list_add(&info->link, &domain->devices);
1321         list_add(&info->global, &device_domain_list);
1322         pdev->sysdata = info;
1323         spin_unlock_irqrestore(&device_domain_lock, flags);
1324         return domain;
1325 error:
1326         /* recheck it here, maybe others set it */
1327         return find_domain(pdev);
1328 }
1329
1330 static int iommu_prepare_identity_map(struct pci_dev *pdev, u64 start, u64 end)
1331 {
1332         struct dmar_domain *domain;
1333         unsigned long size;
1334         u64 base;
1335         int ret;
1336
1337         printk(KERN_INFO
1338                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1339                 pci_name(pdev), start, end);
1340         /* page table init */
1341         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1342         if (!domain)
1343                 return -ENOMEM;
1344
1345         /* The address might not be aligned */
1346         base = start & PAGE_MASK_4K;
1347         size = end - base;
1348         size = PAGE_ALIGN_4K(size);
1349         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1350                         IOVA_PFN(base + size) - 1)) {
1351                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1352                 ret = -ENOMEM;
1353                 goto error;
1354         }
1355
1356         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1357                 size, base, pci_name(pdev));
1358         /*
1359          * RMRR range might have overlap with physical memory range,
1360          * clear it first
1361          */
1362         dma_pte_clear_range(domain, base, base + size);
1363
1364         ret = domain_page_mapping(domain, base, base, size,
1365                 DMA_PTE_READ|DMA_PTE_WRITE);
1366         if (ret)
1367                 goto error;
1368
1369         /* context entry init */
1370         ret = domain_context_mapping(domain, pdev);
1371         if (!ret)
1372                 return 0;
1373 error:
1374         domain_exit(domain);
1375         return ret;
1376
1377 }
1378
1379 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1380         struct pci_dev *pdev)
1381 {
1382         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1383                 return 0;
1384         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1385                 rmrr->end_address + 1);
1386 }
1387
1388 int __init init_dmars(void)
1389 {
1390         struct dmar_drhd_unit *drhd;
1391         struct dmar_rmrr_unit *rmrr;
1392         struct pci_dev *pdev;
1393         struct intel_iommu *iommu;
1394         int ret, unit = 0;
1395
1396         /*
1397          * for each drhd
1398          *    allocate root
1399          *    initialize and program root entry to not present
1400          * endfor
1401          */
1402         for_each_drhd_unit(drhd) {
1403                 if (drhd->ignored)
1404                         continue;
1405                 iommu = alloc_iommu(drhd);
1406                 if (!iommu) {
1407                         ret = -ENOMEM;
1408                         goto error;
1409                 }
1410
1411                 /*
1412                  * TBD:
1413                  * we could share the same root & context tables
1414                  * amoung all IOMMU's. Need to Split it later.
1415                  */
1416                 ret = iommu_alloc_root_entry(iommu);
1417                 if (ret) {
1418                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1419                         goto error;
1420                 }
1421         }
1422
1423         /*
1424          * For each rmrr
1425          *   for each dev attached to rmrr
1426          *   do
1427          *     locate drhd for dev, alloc domain for dev
1428          *     allocate free domain
1429          *     allocate page table entries for rmrr
1430          *     if context not allocated for bus
1431          *           allocate and init context
1432          *           set present in root table for this bus
1433          *     init context with domain, translation etc
1434          *    endfor
1435          * endfor
1436          */
1437         for_each_rmrr_units(rmrr) {
1438                 int i;
1439                 for (i = 0; i < rmrr->devices_cnt; i++) {
1440                         pdev = rmrr->devices[i];
1441                         /* some BIOS lists non-exist devices in DMAR table */
1442                         if (!pdev)
1443                                 continue;
1444                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1445                         if (ret)
1446                                 printk(KERN_ERR
1447                                  "IOMMU: mapping reserved region failed\n");
1448                 }
1449         }
1450
1451         /*
1452          * for each drhd
1453          *   enable fault log
1454          *   global invalidate context cache
1455          *   global invalidate iotlb
1456          *   enable translation
1457          */
1458         for_each_drhd_unit(drhd) {
1459                 if (drhd->ignored)
1460                         continue;
1461                 iommu = drhd->iommu;
1462                 sprintf (iommu->name, "dmar%d", unit++);
1463
1464                 iommu_flush_write_buffer(iommu);
1465
1466                 iommu_set_root_entry(iommu);
1467
1468                 iommu_flush_context_global(iommu, 0);
1469                 iommu_flush_iotlb_global(iommu, 0);
1470
1471                 ret = iommu_enable_translation(iommu);
1472                 if (ret)
1473                         goto error;
1474         }
1475
1476         return 0;
1477 error:
1478         for_each_drhd_unit(drhd) {
1479                 if (drhd->ignored)
1480                         continue;
1481                 iommu = drhd->iommu;
1482                 free_iommu(iommu);
1483         }
1484         return ret;
1485 }
1486
1487 static inline u64 aligned_size(u64 host_addr, size_t size)
1488 {
1489         u64 addr;
1490         addr = (host_addr & (~PAGE_MASK_4K)) + size;
1491         return PAGE_ALIGN_4K(addr);
1492 }
1493
1494 struct iova *
1495 iommu_alloc_iova(struct dmar_domain *domain, void *host_addr, size_t size,
1496                 u64 start, u64 end)
1497 {
1498         u64 start_addr;
1499         struct iova *piova;
1500
1501         /* Make sure it's in range */
1502         if ((start > DOMAIN_MAX_ADDR(domain->gaw)) || end < start)
1503                 return NULL;
1504
1505         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1506         start_addr = PAGE_ALIGN_4K(start);
1507         size = aligned_size((u64)host_addr, size);
1508         if (!size || (start_addr + size > end))
1509                 return NULL;
1510
1511         piova = alloc_iova(&domain->iovad,
1512                         size >> PAGE_SHIFT_4K, IOVA_PFN(end));
1513
1514         return piova;
1515 }
1516
1517 static dma_addr_t __intel_map_single(struct device *dev, void *addr,
1518         size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1519 {
1520         struct dmar_domain *domain;
1521         struct pci_dev *pdev = to_pci_dev(dev);
1522         int ret;
1523         int prot = 0;
1524         struct iova *iova = NULL;
1525         u64 start_addr;
1526
1527         addr = (void *)virt_to_phys(addr);
1528
1529         domain = get_domain_for_dev(pdev,
1530                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
1531         if (!domain) {
1532                 printk(KERN_ERR
1533                         "Allocating domain for %s failed", pci_name(pdev));
1534                 return 0;
1535         }
1536
1537         start_addr = IOVA_START_ADDR;
1538
1539         if (pdev->dma_mask <= DMA_32BIT_MASK) {
1540                 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1541                         pdev->dma_mask);
1542         } else  {
1543                 /*
1544                  * First try to allocate an io virtual address in
1545                  * DMA_32BIT_MASK and if that fails then try allocating
1546                  * from higer range
1547                  */
1548                 iova = iommu_alloc_iova(domain, addr, size, start_addr,
1549                         DMA_32BIT_MASK);
1550                 if (!iova)
1551                         iova = iommu_alloc_iova(domain, addr, size, start_addr,
1552                         pdev->dma_mask);
1553         }
1554
1555         if (!iova) {
1556                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1557                 return 0;
1558         }
1559
1560         /* make sure context mapping is ok */
1561         if (unlikely(!domain_context_mapped(domain, pdev))) {
1562                 ret = domain_context_mapping(domain, pdev);
1563                 if (ret)
1564                         goto error;
1565         }
1566
1567         /*
1568          * Check if DMAR supports zero-length reads on write only
1569          * mappings..
1570          */
1571         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1572                         !cap_zlr(domain->iommu->cap))
1573                 prot |= DMA_PTE_READ;
1574         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1575                 prot |= DMA_PTE_WRITE;
1576         /*
1577          * addr - (addr + size) might be partial page, we should map the whole
1578          * page.  Note: if two part of one page are separately mapped, we
1579          * might have two guest_addr mapping to the same host addr, but this
1580          * is not a big problem
1581          */
1582         ret = domain_page_mapping(domain, iova->pfn_lo << PAGE_SHIFT_4K,
1583                 ((u64)addr) & PAGE_MASK_4K,
1584                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K, prot);
1585         if (ret)
1586                 goto error;
1587
1588         pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
1589                 pci_name(pdev), size, (u64)addr,
1590                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1591                 (u64)(iova->pfn_lo << PAGE_SHIFT_4K), dir);
1592
1593         *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1594         *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1595         return (iova->pfn_lo << PAGE_SHIFT_4K) + ((u64)addr & (~PAGE_MASK_4K));
1596 error:
1597         __free_iova(&domain->iovad, iova);
1598         printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1599                 pci_name(pdev), size, (u64)addr, dir);
1600         return 0;
1601 }
1602
1603 static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
1604         size_t size, int dir)
1605 {
1606         struct pci_dev *pdev = to_pci_dev(hwdev);
1607         dma_addr_t ret;
1608         struct dmar_domain *domain;
1609         u64 flush_addr;
1610         unsigned int flush_size;
1611
1612         BUG_ON(dir == DMA_NONE);
1613         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1614                 return virt_to_bus(addr);
1615
1616         ret = __intel_map_single(hwdev, addr, size,
1617                         dir, &flush_addr, &flush_size);
1618         if (ret) {
1619                 domain = find_domain(pdev);
1620                 /* it's a non-present to present mapping */
1621                 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
1622                                 flush_addr, flush_size >> PAGE_SHIFT_4K, 1))
1623                         iommu_flush_write_buffer(domain->iommu);
1624         }
1625         return ret;
1626 }
1627
1628 static void __intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1629         size_t size, int dir, u64 *flush_addr, unsigned int *flush_size)
1630 {
1631         struct dmar_domain *domain;
1632         struct pci_dev *pdev = to_pci_dev(dev);
1633         struct iova *iova;
1634
1635         domain = find_domain(pdev);
1636         BUG_ON(!domain);
1637
1638         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1639         if (!iova) {
1640                 *flush_size = 0;
1641                 return;
1642         }
1643         pr_debug("Device %s unmapping: %lx@%llx\n",
1644                 pci_name(pdev),
1645                 (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K,
1646                 (u64)(iova->pfn_lo << PAGE_SHIFT_4K));
1647
1648         *flush_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1649         *flush_size = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT_4K;
1650         /*  clear the whole page, not just dev_addr - (dev_addr + size) */
1651         dma_pte_clear_range(domain, *flush_addr, *flush_addr + *flush_size);
1652         /* free page tables */
1653         dma_pte_free_pagetable(domain, *flush_addr, *flush_addr + *flush_size);
1654         /* free iova */
1655         __free_iova(&domain->iovad, iova);
1656 }
1657
1658 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1659         size_t size, int dir)
1660 {
1661         struct pci_dev *pdev = to_pci_dev(dev);
1662         struct dmar_domain *domain;
1663         u64 flush_addr;
1664         unsigned int flush_size;
1665
1666         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1667                 return;
1668
1669         domain = find_domain(pdev);
1670         __intel_unmap_single(dev, dev_addr, size,
1671                 dir, &flush_addr, &flush_size);
1672         if (flush_size == 0)
1673                 return;
1674         if (iommu_flush_iotlb_psi(domain->iommu, domain->id, flush_addr,
1675                         flush_size >> PAGE_SHIFT_4K, 0))
1676                 iommu_flush_write_buffer(domain->iommu);
1677 }
1678
1679 static void * intel_alloc_coherent(struct device *hwdev, size_t size,
1680                        dma_addr_t *dma_handle, gfp_t flags)
1681 {
1682         void *vaddr;
1683         int order;
1684
1685         size = PAGE_ALIGN_4K(size);
1686         order = get_order(size);
1687         flags &= ~(GFP_DMA | GFP_DMA32);
1688
1689         vaddr = (void *)__get_free_pages(flags, order);
1690         if (!vaddr)
1691                 return NULL;
1692         memset(vaddr, 0, size);
1693
1694         *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
1695         if (*dma_handle)
1696                 return vaddr;
1697         free_pages((unsigned long)vaddr, order);
1698         return NULL;
1699 }
1700
1701 static void intel_free_coherent(struct device *hwdev, size_t size,
1702         void *vaddr, dma_addr_t dma_handle)
1703 {
1704         int order;
1705
1706         size = PAGE_ALIGN_4K(size);
1707         order = get_order(size);
1708
1709         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
1710         free_pages((unsigned long)vaddr, order);
1711 }
1712
1713 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sg,
1714         int nelems, int dir)
1715 {
1716         int i;
1717         struct pci_dev *pdev = to_pci_dev(hwdev);
1718         struct dmar_domain *domain;
1719         u64 flush_addr;
1720         unsigned int flush_size;
1721
1722         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1723                 return;
1724
1725         domain = find_domain(pdev);
1726         for (i = 0; i < nelems; i++, sg++)
1727                 __intel_unmap_single(hwdev, sg->dma_address,
1728                         sg->dma_length, dir, &flush_addr, &flush_size);
1729
1730         if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 0))
1731                 iommu_flush_write_buffer(domain->iommu);
1732 }
1733
1734 #define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
1735 static int intel_nontranslate_map_sg(struct device *hddev,
1736         struct scatterlist *sg, int nelems, int dir)
1737 {
1738         int i;
1739
1740         for (i = 0; i < nelems; i++) {
1741                 struct scatterlist *s = &sg[i];
1742                 BUG_ON(!s->page);
1743                 s->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(s));
1744                 s->dma_length = s->length;
1745         }
1746         return nelems;
1747 }
1748
1749 static int intel_map_sg(struct device *hwdev, struct scatterlist *sg,
1750         int nelems, int dir)
1751 {
1752         void *addr;
1753         int i;
1754         dma_addr_t dma_handle;
1755         struct pci_dev *pdev = to_pci_dev(hwdev);
1756         struct dmar_domain *domain;
1757         u64 flush_addr;
1758         unsigned int flush_size;
1759
1760         BUG_ON(dir == DMA_NONE);
1761         if (pdev->sysdata == DUMMY_DEVICE_DOMAIN_INFO)
1762                 return intel_nontranslate_map_sg(hwdev, sg, nelems, dir);
1763
1764         for (i = 0; i < nelems; i++, sg++) {
1765                 addr = SG_ENT_VIRT_ADDRESS(sg);
1766                 dma_handle = __intel_map_single(hwdev, addr,
1767                                 sg->length, dir, &flush_addr, &flush_size);
1768                 if (!dma_handle) {
1769                         intel_unmap_sg(hwdev, sg - i, i, dir);
1770                         sg[0].dma_length = 0;
1771                         return 0;
1772                 }
1773                 sg->dma_address = dma_handle;
1774                 sg->dma_length = sg->length;
1775         }
1776
1777         domain = find_domain(pdev);
1778
1779         /* it's a non-present to present mapping */
1780         if (iommu_flush_iotlb_dsi(domain->iommu, domain->id, 1))
1781                 iommu_flush_write_buffer(domain->iommu);
1782         return nelems;
1783 }
1784
1785 static struct dma_mapping_ops intel_dma_ops = {
1786         .alloc_coherent = intel_alloc_coherent,
1787         .free_coherent = intel_free_coherent,
1788         .map_single = intel_map_single,
1789         .unmap_single = intel_unmap_single,
1790         .map_sg = intel_map_sg,
1791         .unmap_sg = intel_unmap_sg,
1792 };
1793
1794 static inline int iommu_domain_cache_init(void)
1795 {
1796         int ret = 0;
1797
1798         iommu_domain_cache = kmem_cache_create("iommu_domain",
1799                                          sizeof(struct dmar_domain),
1800                                          0,
1801                                          SLAB_HWCACHE_ALIGN,
1802
1803                                          NULL);
1804         if (!iommu_domain_cache) {
1805                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
1806                 ret = -ENOMEM;
1807         }
1808
1809         return ret;
1810 }
1811
1812 static inline int iommu_devinfo_cache_init(void)
1813 {
1814         int ret = 0;
1815
1816         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
1817                                          sizeof(struct device_domain_info),
1818                                          0,
1819                                          SLAB_HWCACHE_ALIGN,
1820
1821                                          NULL);
1822         if (!iommu_devinfo_cache) {
1823                 printk(KERN_ERR "Couldn't create devinfo cache\n");
1824                 ret = -ENOMEM;
1825         }
1826
1827         return ret;
1828 }
1829
1830 static inline int iommu_iova_cache_init(void)
1831 {
1832         int ret = 0;
1833
1834         iommu_iova_cache = kmem_cache_create("iommu_iova",
1835                                          sizeof(struct iova),
1836                                          0,
1837                                          SLAB_HWCACHE_ALIGN,
1838
1839                                          NULL);
1840         if (!iommu_iova_cache) {
1841                 printk(KERN_ERR "Couldn't create iova cache\n");
1842                 ret = -ENOMEM;
1843         }
1844
1845         return ret;
1846 }
1847
1848 static int __init iommu_init_mempool(void)
1849 {
1850         int ret;
1851         ret = iommu_iova_cache_init();
1852         if (ret)
1853                 return ret;
1854
1855         ret = iommu_domain_cache_init();
1856         if (ret)
1857                 goto domain_error;
1858
1859         ret = iommu_devinfo_cache_init();
1860         if (!ret)
1861                 return ret;
1862
1863         kmem_cache_destroy(iommu_domain_cache);
1864 domain_error:
1865         kmem_cache_destroy(iommu_iova_cache);
1866
1867         return -ENOMEM;
1868 }
1869
1870 static void __init iommu_exit_mempool(void)
1871 {
1872         kmem_cache_destroy(iommu_devinfo_cache);
1873         kmem_cache_destroy(iommu_domain_cache);
1874         kmem_cache_destroy(iommu_iova_cache);
1875
1876 }
1877
1878 void __init detect_intel_iommu(void)
1879 {
1880         if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
1881                 return;
1882         if (early_dmar_detect()) {
1883                 iommu_detected = 1;
1884         }
1885 }
1886
1887 static void __init init_no_remapping_devices(void)
1888 {
1889         struct dmar_drhd_unit *drhd;
1890
1891         for_each_drhd_unit(drhd) {
1892                 if (!drhd->include_all) {
1893                         int i;
1894                         for (i = 0; i < drhd->devices_cnt; i++)
1895                                 if (drhd->devices[i] != NULL)
1896                                         break;
1897                         /* ignore DMAR unit if no pci devices exist */
1898                         if (i == drhd->devices_cnt)
1899                                 drhd->ignored = 1;
1900                 }
1901         }
1902
1903         if (dmar_map_gfx)
1904                 return;
1905
1906         for_each_drhd_unit(drhd) {
1907                 int i;
1908                 if (drhd->ignored || drhd->include_all)
1909                         continue;
1910
1911                 for (i = 0; i < drhd->devices_cnt; i++)
1912                         if (drhd->devices[i] &&
1913                                 !IS_GFX_DEVICE(drhd->devices[i]))
1914                                 break;
1915
1916                 if (i < drhd->devices_cnt)
1917                         continue;
1918
1919                 /* bypass IOMMU if it is just for gfx devices */
1920                 drhd->ignored = 1;
1921                 for (i = 0; i < drhd->devices_cnt; i++) {
1922                         if (!drhd->devices[i])
1923                                 continue;
1924                         drhd->devices[i]->sysdata = DUMMY_DEVICE_DOMAIN_INFO;
1925                 }
1926         }
1927 }
1928
1929 int __init intel_iommu_init(void)
1930 {
1931         int ret = 0;
1932
1933         if (no_iommu || swiotlb || dmar_disabled)
1934                 return -ENODEV;
1935
1936         if (dmar_table_init())
1937                 return  -ENODEV;
1938
1939         iommu_init_mempool();
1940         dmar_init_reserved_ranges();
1941
1942         init_no_remapping_devices();
1943
1944         ret = init_dmars();
1945         if (ret) {
1946                 printk(KERN_ERR "IOMMU: dmar init failed\n");
1947                 put_iova_domain(&reserved_iova_list);
1948                 iommu_exit_mempool();
1949                 return ret;
1950         }
1951         printk(KERN_INFO
1952         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
1953
1954         force_iommu = 1;
1955         dma_ops = &intel_dma_ops;
1956         return 0;
1957 }