drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186
 187 /*
 188  * 0: Present
 189  * 1-11: Reserved
 190  * 12-63: Context Ptr (12 - (haw-1))
 191  * 64-127: Reserved
 192  */
 193 struct root_entry {
 194         u64     lo;
 195         u64     hi;
 196 };
 197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 198
 199 /*
 200  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 201  * if marked present.
 202  */
 203 static phys_addr_t root_entry_lctp(struct root_entry *re)
 204 {
 205         if (!(re->lo & 1))
 206                 return 0;
 207
 208         return re->lo & VTD_PAGE_MASK;
 209 }
 210
 211 /*
 212  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 213  * if marked present.
 214  */
 215 static phys_addr_t root_entry_uctp(struct root_entry *re)
 216 {
 217         if (!(re->hi & 1))
 218                 return 0;
 219
 220         return re->hi & VTD_PAGE_MASK;
 221 }
 222 /*
 223  * low 64 bits:
 224  * 0: present
 225  * 1: fault processing disable
 226  * 2-3: translation type
 227  * 12-63: address space root
 228  * high 64 bits:
 229  * 0-2: address width
 230  * 3-6: aval
 231  * 8-23: domain id
 232  */
 233 struct context_entry {
 234         u64 lo;
 235         u64 hi;
 236 };
 237
 238 static inline void context_clear_pasid_enable(struct context_entry *context)
 239 {
 240         context->lo &= ~(1ULL << 11);
 241 }
 242
 243 static inline bool context_pasid_enabled(struct context_entry *context)
 244 {
 245         return !!(context->lo & (1ULL << 11));
 246 }
 247
 248 static inline void context_set_copied(struct context_entry *context)
 249 {
 250         context->hi |= (1ull << 3);
 251 }
 252
 253 static inline bool context_copied(struct context_entry *context)
 254 {
 255         return !!(context->hi & (1ULL << 3));
 256 }
 257
 258 static inline bool __context_present(struct context_entry *context)
 259 {
 260         return (context->lo & 1);
 261 }
 262
 263 static inline bool context_present(struct context_entry *context)
 264 {
 265         return context_pasid_enabled(context) ?
 266              __context_present(context) :
 267              __context_present(context) && !context_copied(context);
 268 }
 269
 270 static inline void context_set_present(struct context_entry *context)
 271 {
 272         context->lo |= 1;
 273 }
 274
 275 static inline void context_set_fault_enable(struct context_entry *context)
 276 {
 277         context->lo &= (((u64)-1) << 2) | 1;
 278 }
 279
 280 static inline void context_set_translation_type(struct context_entry *context,
 281                                                 unsigned long value)
 282 {
 283         context->lo &= (((u64)-1) << 4) | 3;
 284         context->lo |= (value & 3) << 2;
 285 }
 286
 287 static inline void context_set_address_root(struct context_entry *context,
 288                                             unsigned long value)
 289 {
 290         context->lo &= ~VTD_PAGE_MASK;
 291         context->lo |= value & VTD_PAGE_MASK;
 292 }
 293
 294 static inline void context_set_address_width(struct context_entry *context,
 295                                              unsigned long value)
 296 {
 297         context->hi |= value & 7;
 298 }
 299
 300 static inline void context_set_domain_id(struct context_entry *context,
 301                                          unsigned long value)
 302 {
 303         context->hi |= (value & ((1 << 16) - 1)) << 8;
 304 }
 305
 306 static inline int context_domain_id(struct context_entry *c)
 307 {
 308         return((c->hi >> 8) & 0xffff);
 309 }
 310
 311 static inline void context_clear_entry(struct context_entry *context)
 312 {
 313         context->lo = 0;
 314         context->hi = 0;
 315 }
 316
 317 /*
 318  * 0: readable
 319  * 1: writable
 320  * 2-6: reserved
 321  * 7: super page
 322  * 8-10: available
 323  * 11: snoop behavior
 324  * 12-63: Host physcial address
 325  */
 326 struct dma_pte {
 327         u64 val;
 328 };
 329
 330 static inline void dma_clear_pte(struct dma_pte *pte)
 331 {
 332         pte->val = 0;
 333 }
 334
 335 static inline u64 dma_pte_addr(struct dma_pte *pte)
 336 {
 337 #ifdef CONFIG_64BIT
 338         return pte->val & VTD_PAGE_MASK;
 339 #else
 340         /* Must have a full atomic 64-bit read */
 341         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 342 #endif
 343 }
 344
 345 static inline bool dma_pte_present(struct dma_pte *pte)
 346 {
 347         return (pte->val & 3) != 0;
 348 }
 349
 350 static inline bool dma_pte_superpage(struct dma_pte *pte)
 351 {
 352         return (pte->val & DMA_PTE_LARGE_PAGE);
 353 }
 354
 355 static inline int first_pte_in_page(struct dma_pte *pte)
 356 {
 357         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 358 }
 359
 360 /*
 361  * This domain is a statically identity mapping domain.
 362  *      1. This domain creats a static 1:1 mapping to all usable memory.
 363  *      2. It maps to each iommu if successful.
 364  *      3. Each iommu mapps to this domain if successful.
 365  */
 366 static struct dmar_domain *si_domain;
 367 static int hw_pass_through = 1;
 368
 369 /*
 370  * Domain represents a virtual machine, more than one devices
 371  * across iommus may be owned in one domain, e.g. kvm guest.
 372  */
 373 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 374
 375 /* si_domain contains mulitple devices */
 376 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 377
 378 #define for_each_domain_iommu(idx, domain)                      \
 379         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 380                 if (domain->iommu_refcnt[idx])
 381
 382 struct dmar_domain {
 383         int     nid;                    /* node id */
 384
 385         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 386                                         /* Refcount of devices per iommu */
 387
 388
 389         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 390                                         /* Domain ids per IOMMU. Use u16 since
 391                                          * domain ids are 16 bit wide according
 392                                          * to VT-d spec, section 9.3 */
 393
 394         bool has_iotlb_device;
 395         struct list_head devices;       /* all devices' list */
 396         struct iova_domain iovad;       /* iova's that belong to this domain */
 397
 398         struct dma_pte  *pgd;           /* virtual address */
 399         int             gaw;            /* max guest address width */
 400
 401         /* adjusted guest address width, 0 is level 2 30-bit */
 402         int             agaw;
 403
 404         int             flags;          /* flags to find out type of domain */
 405
 406         int             iommu_coherency;/* indicate coherency of iommu access */
 407         int             iommu_snooping; /* indicate snooping control feature*/
 408         int             iommu_count;    /* reference count of iommu */
 409         int             iommu_superpage;/* Level of superpages supported:
 410                                            0 == 4KiB (no superpages), 1 == 2MiB,
 411                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 412         u64             max_addr;       /* maximum mapped address */
 413
 414         struct iommu_domain domain;     /* generic domain data structure for
 415                                            iommu core */
 416 };
 417
 418 /* PCI domain-device relationship */
 419 struct device_domain_info {
 420         struct list_head link;  /* link to domain siblings */
 421         struct list_head global; /* link to global list */
 422         u8 bus;                 /* PCI bus number */
 423         u8 devfn;               /* PCI devfn number */
 424         u8 pasid_supported:3;
 425         u8 pasid_enabled:1;
 426         u8 pri_supported:1;
 427         u8 pri_enabled:1;
 428         u8 ats_supported:1;
 429         u8 ats_enabled:1;
 430         u8 ats_qdep;
 431         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 432         struct intel_iommu *iommu; /* IOMMU used by this device */
 433         struct dmar_domain *domain; /* pointer to domain */
 434 };
 435
 436 struct dmar_rmrr_unit {
 437         struct list_head list;          /* list of rmrr units   */
 438         struct acpi_dmar_header *hdr;   /* ACPI header          */
 439         u64     base_address;           /* reserved base address*/
 440         u64     end_address;            /* reserved end address */
 441         struct dmar_dev_scope *devices; /* target devices */
 442         int     devices_cnt;            /* target device count */
 443 };
 444
 445 struct dmar_atsr_unit {
 446         struct list_head list;          /* list of ATSR units */
 447         struct acpi_dmar_header *hdr;   /* ACPI header */
 448         struct dmar_dev_scope *devices; /* target devices */
 449         int devices_cnt;                /* target device count */
 450         u8 include_all:1;               /* include all ports */
 451 };
 452
 453 static LIST_HEAD(dmar_atsr_units);
 454 static LIST_HEAD(dmar_rmrr_units);
 455
 456 #define for_each_rmrr_units(rmrr) \
 457         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 458
 459 static void flush_unmaps_timeout(unsigned long data);
 460
 461 struct deferred_flush_entry {
 462         unsigned long iova_pfn;
 463         unsigned long nrpages;
 464         struct dmar_domain *domain;
 465         struct page *freelist;
 466 };
 467
 468 #define HIGH_WATER_MARK 250
 469 struct deferred_flush_table {
 470         int next;
 471         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 472 };
 473
 474 struct deferred_flush_data {
 475         spinlock_t lock;
 476         int timer_on;
 477         struct timer_list timer;
 478         long size;
 479         struct deferred_flush_table *tables;
 480 };
 481
 482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 483
 484 /* bitmap for indexing intel_iommus */
 485 static int g_num_of_iommus;
 486
 487 static void domain_exit(struct dmar_domain *domain);
 488 static void domain_remove_dev_info(struct dmar_domain *domain);
 489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 490                                      struct device *dev);
 491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 492 static void domain_context_clear(struct intel_iommu *iommu,
 493                                  struct device *dev);
 494 static int domain_detach_iommu(struct dmar_domain *domain,
 495                                struct intel_iommu *iommu);
 496
 497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 498 int dmar_disabled = 0;
 499 #else
 500 int dmar_disabled = 1;
 501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 502
 503 int intel_iommu_enabled = 0;
 504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 505
 506 static int dmar_map_gfx = 1;
 507 static int dmar_forcedac;
 508 static int intel_iommu_strict;
 509 static int intel_iommu_superpage = 1;
 510 static int intel_iommu_ecs = 1;
 511 static int intel_iommu_pasid28;
 512 static int iommu_identity_mapping;
 513
 514 #define IDENTMAP_ALL            1
 515 #define IDENTMAP_GFX            2
 516 #define IDENTMAP_AZALIA         4
 517
 518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 519  * level" translation of DMA requests-without-PASID doesn't actually happen
 520  * unless you also set the NESTE bit in an extended context-entry. Which of
 521  * course means that SVM doesn't work because it's trying to do nested
 522  * translation of the physical addresses it finds in the process page tables,
 523  * through the IOVA->phys mapping found in the "second level" page tables.
 524  *
 525  * The VT-d specification was retroactively changed to change the definition
 526  * of the capability bits and pretend that Broadwell/Skylake never happened...
 527  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 528  * for some reason it was the PASID capability bit which was redefined (from
 529  * bit 28 on BDW/SKL to bit 40 in future).
 530  *
 531  * So our test for ECS needs to eschew those implementations which set the old
 532  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 533  * Unless we are working around the 'pasid28' limitations, that is, by putting
 534  * the device into passthrough mode for normal DMA and thus masking the bug.
 535  */
 536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 537                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 539  * or new capability bits are set. */
 540 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 541                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 542
 543 int intel_iommu_gfx_mapped;
 544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 545
 546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 547 static DEFINE_SPINLOCK(device_domain_lock);
 548 static LIST_HEAD(device_domain_list);
 549
 550 static const struct iommu_ops intel_iommu_ops;
 551
 552 static bool translation_pre_enabled(struct intel_iommu *iommu)
 553 {
 554         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 555 }
 556
 557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 558 {
 559         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 560 }
 561
 562 static void init_translation_status(struct intel_iommu *iommu)
 563 {
 564         u32 gsts;
 565
 566         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 567         if (gsts & DMA_GSTS_TES)
 568                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 569 }
 570
 571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 573 {
 574         return container_of(dom, struct dmar_domain, domain);
 575 }
 576
 577 static int __init intel_iommu_setup(char *str)
 578 {
 579         if (!str)
 580                 return -EINVAL;
 581         while (*str) {
 582                 if (!strncmp(str, "on", 2)) {
 583                         dmar_disabled = 0;
 584                         pr_info("IOMMU enabled\n");
 585                 } else if (!strncmp(str, "off", 3)) {
 586                         dmar_disabled = 1;
 587                         pr_info("IOMMU disabled\n");
 588                 } else if (!strncmp(str, "igfx_off", 8)) {
 589                         dmar_map_gfx = 0;
 590                         pr_info("Disable GFX device mapping\n");
 591                 } else if (!strncmp(str, "forcedac", 8)) {
 592                         pr_info("Forcing DAC for PCI devices\n");
 593                         dmar_forcedac = 1;
 594                 } else if (!strncmp(str, "strict", 6)) {
 595                         pr_info("Disable batched IOTLB flush\n");
 596                         intel_iommu_strict = 1;
 597                 } else if (!strncmp(str, "sp_off", 6)) {
 598                         pr_info("Disable supported super page\n");
 599                         intel_iommu_superpage = 0;
 600                 } else if (!strncmp(str, "ecs_off", 7)) {
 601                         printk(KERN_INFO
 602                                 "Intel-IOMMU: disable extended context table support\n");
 603                         intel_iommu_ecs = 0;
 604                 } else if (!strncmp(str, "pasid28", 7)) {
 605                         printk(KERN_INFO
 606                                 "Intel-IOMMU: enable pre-production PASID support\n");
 607                         intel_iommu_pasid28 = 1;
 608                         iommu_identity_mapping |= IDENTMAP_GFX;
 609                 }
 610
 611                 str += strcspn(str, ",");
 612                 while (*str == ',')
 613                         str++;
 614         }
 615         return 0;
 616 }
 617 __setup("intel_iommu=", intel_iommu_setup);
 618
 619 static struct kmem_cache *iommu_domain_cache;
 620 static struct kmem_cache *iommu_devinfo_cache;
 621
 622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 623 {
 624         struct dmar_domain **domains;
 625         int idx = did >> 8;
 626
 627         domains = iommu->domains[idx];
 628         if (!domains)
 629                 return NULL;
 630
 631         return domains[did & 0xff];
 632 }
 633
 634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 635                              struct dmar_domain *domain)
 636 {
 637         struct dmar_domain **domains;
 638         int idx = did >> 8;
 639
 640         if (!iommu->domains[idx]) {
 641                 size_t size = 256 * sizeof(struct dmar_domain *);
 642                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 643         }
 644
 645         domains = iommu->domains[idx];
 646         if (WARN_ON(!domains))
 647                 return;
 648         else
 649                 domains[did & 0xff] = domain;
 650 }
 651
 652 static inline void *alloc_pgtable_page(int node)
 653 {
 654         struct page *page;
 655         void *vaddr = NULL;
 656
 657         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 658         if (page)
 659                 vaddr = page_address(page);
 660         return vaddr;
 661 }
 662
 663 static inline void free_pgtable_page(void *vaddr)
 664 {
 665         free_page((unsigned long)vaddr);
 666 }
 667
 668 static inline void *alloc_domain_mem(void)
 669 {
 670         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 671 }
 672
 673 static void free_domain_mem(void *vaddr)
 674 {
 675         kmem_cache_free(iommu_domain_cache, vaddr);
 676 }
 677
 678 static inline void * alloc_devinfo_mem(void)
 679 {
 680         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 681 }
 682
 683 static inline void free_devinfo_mem(void *vaddr)
 684 {
 685         kmem_cache_free(iommu_devinfo_cache, vaddr);
 686 }
 687
 688 static inline int domain_type_is_vm(struct dmar_domain *domain)
 689 {
 690         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 691 }
 692
 693 static inline int domain_type_is_si(struct dmar_domain *domain)
 694 {
 695         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 696 }
 697
 698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 699 {
 700         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 701                                 DOMAIN_FLAG_STATIC_IDENTITY);
 702 }
 703
 704 static inline int domain_pfn_supported(struct dmar_domain *domain,
 705                                        unsigned long pfn)
 706 {
 707         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 708
 709         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 710 }
 711
 712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 713 {
 714         unsigned long sagaw;
 715         int agaw = -1;
 716
 717         sagaw = cap_sagaw(iommu->cap);
 718         for (agaw = width_to_agaw(max_gaw);
 719              agaw >= 0; agaw--) {
 720                 if (test_bit(agaw, &sagaw))
 721                         break;
 722         }
 723
 724         return agaw;
 725 }
 726
 727 /*
 728  * Calculate max SAGAW for each iommu.
 729  */
 730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 731 {
 732         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 733 }
 734
 735 /*
 736  * calculate agaw for each iommu.
 737  * "SAGAW" may be different across iommus, use a default agaw, and
 738  * get a supported less agaw for iommus that don't support the default agaw.
 739  */
 740 int iommu_calculate_agaw(struct intel_iommu *iommu)
 741 {
 742         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 743 }
 744
 745 /* This functionin only returns single iommu in a domain */
 746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 747 {
 748         int iommu_id;
 749
 750         /* si_domain and vm domain should not get here. */
 751         BUG_ON(domain_type_is_vm_or_si(domain));
 752         for_each_domain_iommu(iommu_id, domain)
 753                 break;
 754
 755         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 756                 return NULL;
 757
 758         return g_iommus[iommu_id];
 759 }
 760
 761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 762 {
 763         struct dmar_drhd_unit *drhd;
 764         struct intel_iommu *iommu;
 765         bool found = false;
 766         int i;
 767
 768         domain->iommu_coherency = 1;
 769
 770         for_each_domain_iommu(i, domain) {
 771                 found = true;
 772                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 773                         domain->iommu_coherency = 0;
 774                         break;
 775                 }
 776         }
 777         if (found)
 778                 return;
 779
 780         /* No hardware attached; use lowest common denominator */
 781         rcu_read_lock();
 782         for_each_active_iommu(iommu, drhd) {
 783                 if (!ecap_coherent(iommu->ecap)) {
 784                         domain->iommu_coherency = 0;
 785                         break;
 786                 }
 787         }
 788         rcu_read_unlock();
 789 }
 790
 791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 792 {
 793         struct dmar_drhd_unit *drhd;
 794         struct intel_iommu *iommu;
 795         int ret = 1;
 796
 797         rcu_read_lock();
 798         for_each_active_iommu(iommu, drhd) {
 799                 if (iommu != skip) {
 800                         if (!ecap_sc_support(iommu->ecap)) {
 801                                 ret = 0;
 802                                 break;
 803                         }
 804                 }
 805         }
 806         rcu_read_unlock();
 807
 808         return ret;
 809 }
 810
 811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 812 {
 813         struct dmar_drhd_unit *drhd;
 814         struct intel_iommu *iommu;
 815         int mask = 0xf;
 816
 817         if (!intel_iommu_superpage) {
 818                 return 0;
 819         }
 820
 821         /* set iommu_superpage to the smallest common denominator */
 822         rcu_read_lock();
 823         for_each_active_iommu(iommu, drhd) {
 824                 if (iommu != skip) {
 825                         mask &= cap_super_page_val(iommu->cap);
 826                         if (!mask)
 827                                 break;
 828                 }
 829         }
 830         rcu_read_unlock();
 831
 832         return fls(mask);
 833 }
 834
 835 /* Some capabilities may be different across iommus */
 836 static void domain_update_iommu_cap(struct dmar_domain *domain)
 837 {
 838         domain_update_iommu_coherency(domain);
 839         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 840         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 841 }
 842
 843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 844                                                        u8 bus, u8 devfn, int alloc)
 845 {
 846         struct root_entry *root = &iommu->root_entry[bus];
 847         struct context_entry *context;
 848         u64 *entry;
 849
 850         entry = &root->lo;
 851         if (ecs_enabled(iommu)) {
 852                 if (devfn >= 0x80) {
 853                         devfn -= 0x80;
 854                         entry = &root->hi;
 855                 }
 856                 devfn *= 2;
 857         }
 858         if (*entry & 1)
 859                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 860         else {
 861                 unsigned long phy_addr;
 862                 if (!alloc)
 863                         return NULL;
 864
 865                 context = alloc_pgtable_page(iommu->node);
 866                 if (!context)
 867                         return NULL;
 868
 869                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 870                 phy_addr = virt_to_phys((void *)context);
 871                 *entry = phy_addr | 1;
 872                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 873         }
 874         return &context[devfn];
 875 }
 876
 877 static int iommu_dummy(struct device *dev)
 878 {
 879         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 880 }
 881
 882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 883 {
 884         struct dmar_drhd_unit *drhd = NULL;
 885         struct intel_iommu *iommu;
 886         struct device *tmp;
 887         struct pci_dev *ptmp, *pdev = NULL;
 888         u16 segment = 0;
 889         int i;
 890
 891         if (iommu_dummy(dev))
 892                 return NULL;
 893
 894         if (dev_is_pci(dev)) {
 895                 struct pci_dev *pf_pdev;
 896
 897                 pdev = to_pci_dev(dev);
 898                 /* VFs aren't listed in scope tables; we need to look up
 899                  * the PF instead to find the IOMMU. */
 900                 pf_pdev = pci_physfn(pdev);
 901                 dev = &pf_pdev->dev;
 902                 segment = pci_domain_nr(pdev->bus);
 903         } else if (has_acpi_companion(dev))
 904                 dev = &ACPI_COMPANION(dev)->dev;
 905
 906         rcu_read_lock();
 907         for_each_active_iommu(iommu, drhd) {
 908                 if (pdev && segment != drhd->segment)
 909                         continue;
 910
 911                 for_each_active_dev_scope(drhd->devices,
 912                                           drhd->devices_cnt, i, tmp) {
 913                         if (tmp == dev) {
 914                                 /* For a VF use its original BDF# not that of the PF
 915                                  * which we used for the IOMMU lookup. Strictly speaking
 916                                  * we could do this for all PCI devices; we only need to
 917                                  * get the BDF# from the scope table for ACPI matches. */
 918                                 if (pdev->is_virtfn)
 919                                         goto got_pdev;
 920
 921                                 *bus = drhd->devices[i].bus;
 922                                 *devfn = drhd->devices[i].devfn;
 923                                 goto out;
 924                         }
 925
 926                         if (!pdev || !dev_is_pci(tmp))
 927                                 continue;
 928
 929                         ptmp = to_pci_dev(tmp);
 930                         if (ptmp->subordinate &&
 931                             ptmp->subordinate->number <= pdev->bus->number &&
 932                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 933                                 goto got_pdev;
 934                 }
 935
 936                 if (pdev && drhd->include_all) {
 937                 got_pdev:
 938                         *bus = pdev->bus->number;
 939                         *devfn = pdev->devfn;
 940                         goto out;
 941                 }
 942         }
 943         iommu = NULL;
 944  out:
 945         rcu_read_unlock();
 946
 947         return iommu;
 948 }
 949
 950 static void domain_flush_cache(struct dmar_domain *domain,
 951                                void *addr, int size)
 952 {
 953         if (!domain->iommu_coherency)
 954                 clflush_cache_range(addr, size);
 955 }
 956
 957 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 958 {
 959         struct context_entry *context;
 960         int ret = 0;
 961         unsigned long flags;
 962
 963         spin_lock_irqsave(&iommu->lock, flags);
 964         context = iommu_context_addr(iommu, bus, devfn, 0);
 965         if (context)
 966                 ret = context_present(context);
 967         spin_unlock_irqrestore(&iommu->lock, flags);
 968         return ret;
 969 }
 970
 971 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 972 {
 973         struct context_entry *context;
 974         unsigned long flags;
 975
 976         spin_lock_irqsave(&iommu->lock, flags);
 977         context = iommu_context_addr(iommu, bus, devfn, 0);
 978         if (context) {
 979                 context_clear_entry(context);
 980                 __iommu_flush_cache(iommu, context, sizeof(*context));
 981         }
 982         spin_unlock_irqrestore(&iommu->lock, flags);
 983 }
 984
 985 static void free_context_table(struct intel_iommu *iommu)
 986 {
 987         int i;
 988         unsigned long flags;
 989         struct context_entry *context;
 990
 991         spin_lock_irqsave(&iommu->lock, flags);
 992         if (!iommu->root_entry) {
 993                 goto out;
 994         }
 995         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 996                 context = iommu_context_addr(iommu, i, 0, 0);
 997                 if (context)
 998                         free_pgtable_page(context);
 999
1000                 if (!ecs_enabled(iommu))
1001                         continue;
1002
1003                 context = iommu_context_addr(iommu, i, 0x80, 0);
1004                 if (context)
1005                         free_pgtable_page(context);
1006
1007         }
1008         free_pgtable_page(iommu->root_entry);
1009         iommu->root_entry = NULL;
1010 out:
1011         spin_unlock_irqrestore(&iommu->lock, flags);
1012 }
1013
1014 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1015                                       unsigned long pfn, int *target_level)
1016 {
1017         struct dma_pte *parent, *pte = NULL;
1018         int level = agaw_to_level(domain->agaw);
1019         int offset;
1020
1021         BUG_ON(!domain->pgd);
1022
1023         if (!domain_pfn_supported(domain, pfn))
1024                 /* Address beyond IOMMU's addressing capabilities. */
1025                 return NULL;
1026
1027         parent = domain->pgd;
1028
1029         while (1) {
1030                 void *tmp_page;
1031
1032                 offset = pfn_level_offset(pfn, level);
1033                 pte = &parent[offset];
1034                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1035                         break;
1036                 if (level == *target_level)
1037                         break;
1038
1039                 if (!dma_pte_present(pte)) {
1040                         uint64_t pteval;
1041
1042                         tmp_page = alloc_pgtable_page(domain->nid);
1043
1044                         if (!tmp_page)
1045                                 return NULL;
1046
1047                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1048                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1049                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1050                                 /* Someone else set it while we were thinking; use theirs. */
1051                                 free_pgtable_page(tmp_page);
1052                         else
1053                                 domain_flush_cache(domain, pte, sizeof(*pte));
1054                 }
1055                 if (level == 1)
1056                         break;
1057
1058                 parent = phys_to_virt(dma_pte_addr(pte));
1059                 level--;
1060         }
1061
1062         if (!*target_level)
1063                 *target_level = level;
1064
1065         return pte;
1066 }
1067
1068
1069 /* return address's pte at specific level */
1070 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1071                                          unsigned long pfn,
1072                                          int level, int *large_page)
1073 {
1074         struct dma_pte *parent, *pte = NULL;
1075         int total = agaw_to_level(domain->agaw);
1076         int offset;
1077
1078         parent = domain->pgd;
1079         while (level <= total) {
1080                 offset = pfn_level_offset(pfn, total);
1081                 pte = &parent[offset];
1082                 if (level == total)
1083                         return pte;
1084
1085                 if (!dma_pte_present(pte)) {
1086                         *large_page = total;
1087                         break;
1088                 }
1089
1090                 if (dma_pte_superpage(pte)) {
1091                         *large_page = total;
1092                         return pte;
1093                 }
1094
1095                 parent = phys_to_virt(dma_pte_addr(pte));
1096                 total--;
1097         }
1098         return NULL;
1099 }
1100
1101 /* clear last level pte, a tlb flush should be followed */
1102 static void dma_pte_clear_range(struct dmar_domain *domain,
1103                                 unsigned long start_pfn,
1104                                 unsigned long last_pfn)
1105 {
1106         unsigned int large_page = 1;
1107         struct dma_pte *first_pte, *pte;
1108
1109         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1110         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1111         BUG_ON(start_pfn > last_pfn);
1112
1113         /* we don't need lock here; nobody else touches the iova range */
1114         do {
1115                 large_page = 1;
1116                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1117                 if (!pte) {
1118                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1119                         continue;
1120                 }
1121                 do {
1122                         dma_clear_pte(pte);
1123                         start_pfn += lvl_to_nr_pages(large_page);
1124                         pte++;
1125                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1126
1127                 domain_flush_cache(domain, first_pte,
1128                                    (void *)pte - (void *)first_pte);
1129
1130         } while (start_pfn && start_pfn <= last_pfn);
1131 }
1132
1133 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1134                                struct dma_pte *pte, unsigned long pfn,
1135                                unsigned long start_pfn, unsigned long last_pfn)
1136 {
1137         pfn = max(start_pfn, pfn);
1138         pte = &pte[pfn_level_offset(pfn, level)];
1139
1140         do {
1141                 unsigned long level_pfn;
1142                 struct dma_pte *level_pte;
1143
1144                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1145                         goto next;
1146
1147                 level_pfn = pfn & level_mask(level);
1148                 level_pte = phys_to_virt(dma_pte_addr(pte));
1149
1150                 if (level > 2)
1151                         dma_pte_free_level(domain, level - 1, level_pte,
1152                                            level_pfn, start_pfn, last_pfn);
1153
1154                 /* If range covers entire pagetable, free it */
1155                 if (!(start_pfn > level_pfn ||
1156                       last_pfn < level_pfn + level_size(level) - 1)) {
1157                         dma_clear_pte(pte);
1158                         domain_flush_cache(domain, pte, sizeof(*pte));
1159                         free_pgtable_page(level_pte);
1160                 }
1161 next:
1162                 pfn += level_size(level);
1163         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1164 }
1165
1166 /* clear last level (leaf) ptes and free page table pages. */
1167 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1168                                    unsigned long start_pfn,
1169                                    unsigned long last_pfn)
1170 {
1171         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173         BUG_ON(start_pfn > last_pfn);
1174
1175         dma_pte_clear_range(domain, start_pfn, last_pfn);
1176
1177         /* We don't need lock here; nobody else touches the iova range */
1178         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1179                            domain->pgd, 0, start_pfn, last_pfn);
1180
1181         /* free pgd */
1182         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1183                 free_pgtable_page(domain->pgd);
1184                 domain->pgd = NULL;
1185         }
1186 }
1187
1188 /* When a page at a given level is being unlinked from its parent, we don't
1189    need to *modify* it at all. All we need to do is make a list of all the
1190    pages which can be freed just as soon as we've flushed the IOTLB and we
1191    know the hardware page-walk will no longer touch them.
1192    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1193    be freed. */
1194 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1195                                             int level, struct dma_pte *pte,
1196                                             struct page *freelist)
1197 {
1198         struct page *pg;
1199
1200         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1201         pg->freelist = freelist;
1202         freelist = pg;
1203
1204         if (level == 1)
1205                 return freelist;
1206
1207         pte = page_address(pg);
1208         do {
1209                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1210                         freelist = dma_pte_list_pagetables(domain, level - 1,
1211                                                            pte, freelist);
1212                 pte++;
1213         } while (!first_pte_in_page(pte));
1214
1215         return freelist;
1216 }
1217
1218 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1219                                         struct dma_pte *pte, unsigned long pfn,
1220                                         unsigned long start_pfn,
1221                                         unsigned long last_pfn,
1222                                         struct page *freelist)
1223 {
1224         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1225
1226         pfn = max(start_pfn, pfn);
1227         pte = &pte[pfn_level_offset(pfn, level)];
1228
1229         do {
1230                 unsigned long level_pfn;
1231
1232                 if (!dma_pte_present(pte))
1233                         goto next;
1234
1235                 level_pfn = pfn & level_mask(level);
1236
1237                 /* If range covers entire pagetable, free it */
1238                 if (start_pfn <= level_pfn &&
1239                     last_pfn >= level_pfn + level_size(level) - 1) {
1240                         /* These suborbinate page tables are going away entirely. Don't
1241                            bother to clear them; we're just going to *free* them. */
1242                         if (level > 1 && !dma_pte_superpage(pte))
1243                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1244
1245                         dma_clear_pte(pte);
1246                         if (!first_pte)
1247                                 first_pte = pte;
1248                         last_pte = pte;
1249                 } else if (level > 1) {
1250                         /* Recurse down into a level that isn't *entirely* obsolete */
1251                         freelist = dma_pte_clear_level(domain, level - 1,
1252                                                        phys_to_virt(dma_pte_addr(pte)),
1253                                                        level_pfn, start_pfn, last_pfn,
1254                                                        freelist);
1255                 }
1256 next:
1257                 pfn += level_size(level);
1258         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1259
1260         if (first_pte)
1261                 domain_flush_cache(domain, first_pte,
1262                                    (void *)++last_pte - (void *)first_pte);
1263
1264         return freelist;
1265 }
1266
1267 /* We can't just free the pages because the IOMMU may still be walking
1268    the page tables, and may have cached the intermediate levels. The
1269    pages can only be freed after the IOTLB flush has been done. */
1270 static struct page *domain_unmap(struct dmar_domain *domain,
1271                                  unsigned long start_pfn,
1272                                  unsigned long last_pfn)
1273 {
1274         struct page *freelist = NULL;
1275
1276         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1277         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1278         BUG_ON(start_pfn > last_pfn);
1279
1280         /* we don't need lock here; nobody else touches the iova range */
1281         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1282                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1283
1284         /* free pgd */
1285         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1286                 struct page *pgd_page = virt_to_page(domain->pgd);
1287                 pgd_page->freelist = freelist;
1288                 freelist = pgd_page;
1289
1290                 domain->pgd = NULL;
1291         }
1292
1293         return freelist;
1294 }
1295
1296 static void dma_free_pagelist(struct page *freelist)
1297 {
1298         struct page *pg;
1299
1300         while ((pg = freelist)) {
1301                 freelist = pg->freelist;
1302                 free_pgtable_page(page_address(pg));
1303         }
1304 }
1305
1306 /* iommu handling */
1307 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1308 {
1309         struct root_entry *root;
1310         unsigned long flags;
1311
1312         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1313         if (!root) {
1314                 pr_err("Allocating root entry for %s failed\n",
1315                         iommu->name);
1316                 return -ENOMEM;
1317         }
1318
1319         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1320
1321         spin_lock_irqsave(&iommu->lock, flags);
1322         iommu->root_entry = root;
1323         spin_unlock_irqrestore(&iommu->lock, flags);
1324
1325         return 0;
1326 }
1327
1328 static void iommu_set_root_entry(struct intel_iommu *iommu)
1329 {
1330         u64 addr;
1331         u32 sts;
1332         unsigned long flag;
1333
1334         addr = virt_to_phys(iommu->root_entry);
1335         if (ecs_enabled(iommu))
1336                 addr |= DMA_RTADDR_RTT;
1337
1338         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1339         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1340
1341         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1342
1343         /* Make sure hardware complete it */
1344         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1345                       readl, (sts & DMA_GSTS_RTPS), sts);
1346
1347         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1348 }
1349
1350 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1351 {
1352         u32 val;
1353         unsigned long flag;
1354
1355         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1356                 return;
1357
1358         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1359         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1360
1361         /* Make sure hardware complete it */
1362         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1363                       readl, (!(val & DMA_GSTS_WBFS)), val);
1364
1365         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1366 }
1367
1368 /* return value determine if we need a write buffer flush */
1369 static void __iommu_flush_context(struct intel_iommu *iommu,
1370                                   u16 did, u16 source_id, u8 function_mask,
1371                                   u64 type)
1372 {
1373         u64 val = 0;
1374         unsigned long flag;
1375
1376         switch (type) {
1377         case DMA_CCMD_GLOBAL_INVL:
1378                 val = DMA_CCMD_GLOBAL_INVL;
1379                 break;
1380         case DMA_CCMD_DOMAIN_INVL:
1381                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1382                 break;
1383         case DMA_CCMD_DEVICE_INVL:
1384                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1385                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1386                 break;
1387         default:
1388                 BUG();
1389         }
1390         val |= DMA_CCMD_ICC;
1391
1392         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1393         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1394
1395         /* Make sure hardware complete it */
1396         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1397                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1398
1399         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1400 }
1401
1402 /* return value determine if we need a write buffer flush */
1403 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1404                                 u64 addr, unsigned int size_order, u64 type)
1405 {
1406         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1407         u64 val = 0, val_iva = 0;
1408         unsigned long flag;
1409
1410         switch (type) {
1411         case DMA_TLB_GLOBAL_FLUSH:
1412                 /* global flush doesn't need set IVA_REG */
1413                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1414                 break;
1415         case DMA_TLB_DSI_FLUSH:
1416                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1417                 break;
1418         case DMA_TLB_PSI_FLUSH:
1419                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1420                 /* IH bit is passed in as part of address */
1421                 val_iva = size_order | addr;
1422                 break;
1423         default:
1424                 BUG();
1425         }
1426         /* Note: set drain read/write */
1427 #if 0
1428         /*
1429          * This is probably to be super secure.. Looks like we can
1430          * ignore it without any impact.
1431          */
1432         if (cap_read_drain(iommu->cap))
1433                 val |= DMA_TLB_READ_DRAIN;
1434 #endif
1435         if (cap_write_drain(iommu->cap))
1436                 val |= DMA_TLB_WRITE_DRAIN;
1437
1438         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1439         /* Note: Only uses first TLB reg currently */
1440         if (val_iva)
1441                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1442         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1443
1444         /* Make sure hardware complete it */
1445         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1446                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1447
1448         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1449
1450         /* check IOTLB invalidation granularity */
1451         if (DMA_TLB_IAIG(val) == 0)
1452                 pr_err("Flush IOTLB failed\n");
1453         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1454                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1455                         (unsigned long long)DMA_TLB_IIRG(type),
1456                         (unsigned long long)DMA_TLB_IAIG(val));
1457 }
1458
1459 static struct device_domain_info *
1460 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1461                          u8 bus, u8 devfn)
1462 {
1463         struct device_domain_info *info;
1464
1465         assert_spin_locked(&device_domain_lock);
1466
1467         if (!iommu->qi)
1468                 return NULL;
1469
1470         list_for_each_entry(info, &domain->devices, link)
1471                 if (info->iommu == iommu && info->bus == bus &&
1472                     info->devfn == devfn) {
1473                         if (info->ats_supported && info->dev)
1474                                 return info;
1475                         break;
1476                 }
1477
1478         return NULL;
1479 }
1480
1481 static void domain_update_iotlb(struct dmar_domain *domain)
1482 {
1483         struct device_domain_info *info;
1484         bool has_iotlb_device = false;
1485
1486         assert_spin_locked(&device_domain_lock);
1487
1488         list_for_each_entry(info, &domain->devices, link) {
1489                 struct pci_dev *pdev;
1490
1491                 if (!info->dev || !dev_is_pci(info->dev))
1492                         continue;
1493
1494                 pdev = to_pci_dev(info->dev);
1495                 if (pdev->ats_enabled) {
1496                         has_iotlb_device = true;
1497                         break;
1498                 }
1499         }
1500
1501         domain->has_iotlb_device = has_iotlb_device;
1502 }
1503
1504 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1505 {
1506         struct pci_dev *pdev;
1507
1508         assert_spin_locked(&device_domain_lock);
1509
1510         if (!info || !dev_is_pci(info->dev))
1511                 return;
1512
1513         pdev = to_pci_dev(info->dev);
1514
1515 #ifdef CONFIG_INTEL_IOMMU_SVM
1516         /* The PCIe spec, in its wisdom, declares that the behaviour of
1517            the device if you enable PASID support after ATS support is
1518            undefined. So always enable PASID support on devices which
1519            have it, even if we can't yet know if we're ever going to
1520            use it. */
1521         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1522                 info->pasid_enabled = 1;
1523
1524         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1525                 info->pri_enabled = 1;
1526 #endif
1527         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1528                 info->ats_enabled = 1;
1529                 domain_update_iotlb(info->domain);
1530                 info->ats_qdep = pci_ats_queue_depth(pdev);
1531         }
1532 }
1533
1534 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1535 {
1536         struct pci_dev *pdev;
1537
1538         assert_spin_locked(&device_domain_lock);
1539
1540         if (!dev_is_pci(info->dev))
1541                 return;
1542
1543         pdev = to_pci_dev(info->dev);
1544
1545         if (info->ats_enabled) {
1546                 pci_disable_ats(pdev);
1547                 info->ats_enabled = 0;
1548                 domain_update_iotlb(info->domain);
1549         }
1550 #ifdef CONFIG_INTEL_IOMMU_SVM
1551         if (info->pri_enabled) {
1552                 pci_disable_pri(pdev);
1553                 info->pri_enabled = 0;
1554         }
1555         if (info->pasid_enabled) {
1556                 pci_disable_pasid(pdev);
1557                 info->pasid_enabled = 0;
1558         }
1559 #endif
1560 }
1561
1562 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1563                                   u64 addr, unsigned mask)
1564 {
1565         u16 sid, qdep;
1566         unsigned long flags;
1567         struct device_domain_info *info;
1568
1569         if (!domain->has_iotlb_device)
1570                 return;
1571
1572         spin_lock_irqsave(&device_domain_lock, flags);
1573         list_for_each_entry(info, &domain->devices, link) {
1574                 if (!info->ats_enabled)
1575                         continue;
1576
1577                 sid = info->bus << 8 | info->devfn;
1578                 qdep = info->ats_qdep;
1579                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1580         }
1581         spin_unlock_irqrestore(&device_domain_lock, flags);
1582 }
1583
1584 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1585                                   struct dmar_domain *domain,
1586                                   unsigned long pfn, unsigned int pages,
1587                                   int ih, int map)
1588 {
1589         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1590         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1591         u16 did = domain->iommu_did[iommu->seq_id];
1592
1593         BUG_ON(pages == 0);
1594
1595         if (ih)
1596                 ih = 1 << 6;
1597         /*
1598          * Fallback to domain selective flush if no PSI support or the size is
1599          * too big.
1600          * PSI requires page size to be 2 ^ x, and the base address is naturally
1601          * aligned to the size
1602          */
1603         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1604                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1605                                                 DMA_TLB_DSI_FLUSH);
1606         else
1607                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1608                                                 DMA_TLB_PSI_FLUSH);
1609
1610         /*
1611          * In caching mode, changes of pages from non-present to present require
1612          * flush. However, device IOTLB doesn't need to be flushed in this case.
1613          */
1614         if (!cap_caching_mode(iommu->cap) || !map)
1615                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1616                                       addr, mask);
1617 }
1618
1619 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1620 {
1621         u32 pmen;
1622         unsigned long flags;
1623
1624         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1625         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1626         pmen &= ~DMA_PMEN_EPM;
1627         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1628
1629         /* wait for the protected region status bit to clear */
1630         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1631                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1632
1633         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1634 }
1635
1636 static void iommu_enable_translation(struct intel_iommu *iommu)
1637 {
1638         u32 sts;
1639         unsigned long flags;
1640
1641         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1642         iommu->gcmd |= DMA_GCMD_TE;
1643         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1644
1645         /* Make sure hardware complete it */
1646         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1647                       readl, (sts & DMA_GSTS_TES), sts);
1648
1649         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1650 }
1651
1652 static void iommu_disable_translation(struct intel_iommu *iommu)
1653 {
1654         u32 sts;
1655         unsigned long flag;
1656
1657         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1658         iommu->gcmd &= ~DMA_GCMD_TE;
1659         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1660
1661         /* Make sure hardware complete it */
1662         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1663                       readl, (!(sts & DMA_GSTS_TES)), sts);
1664
1665         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1666 }
1667
1668
1669 static int iommu_init_domains(struct intel_iommu *iommu)
1670 {
1671         u32 ndomains, nlongs;
1672         size_t size;
1673
1674         ndomains = cap_ndoms(iommu->cap);
1675         pr_debug("%s: Number of Domains supported <%d>\n",
1676                  iommu->name, ndomains);
1677         nlongs = BITS_TO_LONGS(ndomains);
1678
1679         spin_lock_init(&iommu->lock);
1680
1681         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1682         if (!iommu->domain_ids) {
1683                 pr_err("%s: Allocating domain id array failed\n",
1684                        iommu->name);
1685                 return -ENOMEM;
1686         }
1687
1688         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1689         iommu->domains = kzalloc(size, GFP_KERNEL);
1690
1691         if (iommu->domains) {
1692                 size = 256 * sizeof(struct dmar_domain *);
1693                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1694         }
1695
1696         if (!iommu->domains || !iommu->domains[0]) {
1697                 pr_err("%s: Allocating domain array failed\n",
1698                        iommu->name);
1699                 kfree(iommu->domain_ids);
1700                 kfree(iommu->domains);
1701                 iommu->domain_ids = NULL;
1702                 iommu->domains    = NULL;
1703                 return -ENOMEM;
1704         }
1705
1706
1707
1708         /*
1709          * If Caching mode is set, then invalid translations are tagged
1710          * with domain-id 0, hence we need to pre-allocate it. We also
1711          * use domain-id 0 as a marker for non-allocated domain-id, so
1712          * make sure it is not used for a real domain.
1713          */
1714         set_bit(0, iommu->domain_ids);
1715
1716         return 0;
1717 }
1718
1719 static void disable_dmar_iommu(struct intel_iommu *iommu)
1720 {
1721         struct device_domain_info *info, *tmp;
1722         unsigned long flags;
1723
1724         if (!iommu->domains || !iommu->domain_ids)
1725                 return;
1726
1727 again:
1728         spin_lock_irqsave(&device_domain_lock, flags);
1729         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1730                 struct dmar_domain *domain;
1731
1732                 if (info->iommu != iommu)
1733                         continue;
1734
1735                 if (!info->dev || !info->domain)
1736                         continue;
1737
1738                 domain = info->domain;
1739
1740                 __dmar_remove_one_dev_info(info);
1741
1742                 if (!domain_type_is_vm_or_si(domain)) {
1743                         /*
1744                          * The domain_exit() function  can't be called under
1745                          * device_domain_lock, as it takes this lock itself.
1746                          * So release the lock here and re-run the loop
1747                          * afterwards.
1748                          */
1749                         spin_unlock_irqrestore(&device_domain_lock, flags);
1750                         domain_exit(domain);
1751                         goto again;
1752                 }
1753         }
1754         spin_unlock_irqrestore(&device_domain_lock, flags);
1755
1756         if (iommu->gcmd & DMA_GCMD_TE)
1757                 iommu_disable_translation(iommu);
1758 }
1759
1760 static void free_dmar_iommu(struct intel_iommu *iommu)
1761 {
1762         if ((iommu->domains) && (iommu->domain_ids)) {
1763                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1764                 int i;
1765
1766                 for (i = 0; i < elems; i++)
1767                         kfree(iommu->domains[i]);
1768                 kfree(iommu->domains);
1769                 kfree(iommu->domain_ids);
1770                 iommu->domains = NULL;
1771                 iommu->domain_ids = NULL;
1772         }
1773
1774         g_iommus[iommu->seq_id] = NULL;
1775
1776         /* free context mapping */
1777         free_context_table(iommu);
1778
1779 #ifdef CONFIG_INTEL_IOMMU_SVM
1780         if (pasid_enabled(iommu)) {
1781                 if (ecap_prs(iommu->ecap))
1782                         intel_svm_finish_prq(iommu);
1783                 intel_svm_free_pasid_tables(iommu);
1784         }
1785 #endif
1786 }
1787
1788 static struct dmar_domain *alloc_domain(int flags)
1789 {
1790         struct dmar_domain *domain;
1791
1792         domain = alloc_domain_mem();
1793         if (!domain)
1794                 return NULL;
1795
1796         memset(domain, 0, sizeof(*domain));
1797         domain->nid = -1;
1798         domain->flags = flags;
1799         domain->has_iotlb_device = false;
1800         INIT_LIST_HEAD(&domain->devices);
1801
1802         return domain;
1803 }
1804
1805 /* Must be called with iommu->lock */
1806 static int domain_attach_iommu(struct dmar_domain *domain,
1807                                struct intel_iommu *iommu)
1808 {
1809         unsigned long ndomains;
1810         int num;
1811
1812         assert_spin_locked(&device_domain_lock);
1813         assert_spin_locked(&iommu->lock);
1814
1815         domain->iommu_refcnt[iommu->seq_id] += 1;
1816         domain->iommu_count += 1;
1817         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1818                 ndomains = cap_ndoms(iommu->cap);
1819                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1820
1821                 if (num >= ndomains) {
1822                         pr_err("%s: No free domain ids\n", iommu->name);
1823                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1824                         domain->iommu_count -= 1;
1825                         return -ENOSPC;
1826                 }
1827
1828                 set_bit(num, iommu->domain_ids);
1829                 set_iommu_domain(iommu, num, domain);
1830
1831                 domain->iommu_did[iommu->seq_id] = num;
1832                 domain->nid                      = iommu->node;
1833
1834                 domain_update_iommu_cap(domain);
1835         }
1836
1837         return 0;
1838 }
1839
1840 static int domain_detach_iommu(struct dmar_domain *domain,
1841                                struct intel_iommu *iommu)
1842 {
1843         int num, count = INT_MAX;
1844
1845         assert_spin_locked(&device_domain_lock);
1846         assert_spin_locked(&iommu->lock);
1847
1848         domain->iommu_refcnt[iommu->seq_id] -= 1;
1849         count = --domain->iommu_count;
1850         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1851                 num = domain->iommu_did[iommu->seq_id];
1852                 clear_bit(num, iommu->domain_ids);
1853                 set_iommu_domain(iommu, num, NULL);
1854
1855                 domain_update_iommu_cap(domain);
1856                 domain->iommu_did[iommu->seq_id] = 0;
1857         }
1858
1859         return count;
1860 }
1861
1862 static struct iova_domain reserved_iova_list;
1863 static struct lock_class_key reserved_rbtree_key;
1864
1865 static int dmar_init_reserved_ranges(void)
1866 {
1867         struct pci_dev *pdev = NULL;
1868         struct iova *iova;
1869         int i;
1870
1871         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1872                         DMA_32BIT_PFN);
1873
1874         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1875                 &reserved_rbtree_key);
1876
1877         /* IOAPIC ranges shouldn't be accessed by DMA */
1878         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1879                 IOVA_PFN(IOAPIC_RANGE_END));
1880         if (!iova) {
1881                 pr_err("Reserve IOAPIC range failed\n");
1882                 return -ENODEV;
1883         }
1884
1885         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1886         for_each_pci_dev(pdev) {
1887                 struct resource *r;
1888
1889                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1890                         r = &pdev->resource[i];
1891                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1892                                 continue;
1893                         iova = reserve_iova(&reserved_iova_list,
1894                                             IOVA_PFN(r->start),
1895                                             IOVA_PFN(r->end));
1896                         if (!iova) {
1897                                 pr_err("Reserve iova failed\n");
1898                                 return -ENODEV;
1899                         }
1900                 }
1901         }
1902         return 0;
1903 }
1904
1905 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1906 {
1907         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1908 }
1909
1910 static inline int guestwidth_to_adjustwidth(int gaw)
1911 {
1912         int agaw;
1913         int r = (gaw - 12) % 9;
1914
1915         if (r == 0)
1916                 agaw = gaw;
1917         else
1918                 agaw = gaw + 9 - r;
1919         if (agaw > 64)
1920                 agaw = 64;
1921         return agaw;
1922 }
1923
1924 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1925                        int guest_width)
1926 {
1927         int adjust_width, agaw;
1928         unsigned long sagaw;
1929
1930         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1931                         DMA_32BIT_PFN);
1932         domain_reserve_special_ranges(domain);
1933
1934         /* calculate AGAW */
1935         if (guest_width > cap_mgaw(iommu->cap))
1936                 guest_width = cap_mgaw(iommu->cap);
1937         domain->gaw = guest_width;
1938         adjust_width = guestwidth_to_adjustwidth(guest_width);
1939         agaw = width_to_agaw(adjust_width);
1940         sagaw = cap_sagaw(iommu->cap);
1941         if (!test_bit(agaw, &sagaw)) {
1942                 /* hardware doesn't support it, choose a bigger one */
1943                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1944                 agaw = find_next_bit(&sagaw, 5, agaw);
1945                 if (agaw >= 5)
1946                         return -ENODEV;
1947         }
1948         domain->agaw = agaw;
1949
1950         if (ecap_coherent(iommu->ecap))
1951                 domain->iommu_coherency = 1;
1952         else
1953                 domain->iommu_coherency = 0;
1954
1955         if (ecap_sc_support(iommu->ecap))
1956                 domain->iommu_snooping = 1;
1957         else
1958                 domain->iommu_snooping = 0;
1959
1960         if (intel_iommu_superpage)
1961                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1962         else
1963                 domain->iommu_superpage = 0;
1964
1965         domain->nid = iommu->node;
1966
1967         /* always allocate the top pgd */
1968         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1969         if (!domain->pgd)
1970                 return -ENOMEM;
1971         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1972         return 0;
1973 }
1974
1975 static void domain_exit(struct dmar_domain *domain)
1976 {
1977         struct page *freelist = NULL;
1978
1979         /* Domain 0 is reserved, so dont process it */
1980         if (!domain)
1981                 return;
1982
1983         /* Flush any lazy unmaps that may reference this domain */
1984         if (!intel_iommu_strict) {
1985                 int cpu;
1986
1987                 for_each_possible_cpu(cpu)
1988                         flush_unmaps_timeout(cpu);
1989         }
1990
1991         /* Remove associated devices and clear attached or cached domains */
1992         rcu_read_lock();
1993         domain_remove_dev_info(domain);
1994         rcu_read_unlock();
1995
1996         /* destroy iovas */
1997         put_iova_domain(&domain->iovad);
1998
1999         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2000
2001         dma_free_pagelist(freelist);
2002
2003         free_domain_mem(domain);
2004 }
2005
2006 static int domain_context_mapping_one(struct dmar_domain *domain,
2007                                       struct intel_iommu *iommu,
2008                                       u8 bus, u8 devfn)
2009 {
2010         u16 did = domain->iommu_did[iommu->seq_id];
2011         int translation = CONTEXT_TT_MULTI_LEVEL;
2012         struct device_domain_info *info = NULL;
2013         struct context_entry *context;
2014         unsigned long flags;
2015         struct dma_pte *pgd;
2016         int ret, agaw;
2017
2018         WARN_ON(did == 0);
2019
2020         if (hw_pass_through && domain_type_is_si(domain))
2021                 translation = CONTEXT_TT_PASS_THROUGH;
2022
2023         pr_debug("Set context mapping for %02x:%02x.%d\n",
2024                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2025
2026         BUG_ON(!domain->pgd);
2027
2028         spin_lock_irqsave(&device_domain_lock, flags);
2029         spin_lock(&iommu->lock);
2030
2031         ret = -ENOMEM;
2032         context = iommu_context_addr(iommu, bus, devfn, 1);
2033         if (!context)
2034                 goto out_unlock;
2035
2036         ret = 0;
2037         if (context_present(context))
2038                 goto out_unlock;
2039
2040         /*
2041          * For kdump cases, old valid entries may be cached due to the
2042          * in-flight DMA and copied pgtable, but there is no unmapping
2043          * behaviour for them, thus we need an explicit cache flush for
2044          * the newly-mapped device. For kdump, at this point, the device
2045          * is supposed to finish reset at its driver probe stage, so no
2046          * in-flight DMA will exist, and we don't need to worry anymore
2047          * hereafter.
2048          */
2049         if (context_copied(context)) {
2050                 u16 did_old = context_domain_id(context);
2051
2052                 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap))
2053                         iommu->flush.flush_context(iommu, did_old,
2054                                                    (((u16)bus) << 8) | devfn,
2055                                                    DMA_CCMD_MASK_NOBIT,
2056                                                    DMA_CCMD_DEVICE_INVL);
2057         }
2058
2059         pgd = domain->pgd;
2060
2061         context_clear_entry(context);
2062         context_set_domain_id(context, did);
2063
2064         /*
2065          * Skip top levels of page tables for iommu which has less agaw
2066          * than default.  Unnecessary for PT mode.
2067          */
2068         if (translation != CONTEXT_TT_PASS_THROUGH) {
2069                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2070                         ret = -ENOMEM;
2071                         pgd = phys_to_virt(dma_pte_addr(pgd));
2072                         if (!dma_pte_present(pgd))
2073                                 goto out_unlock;
2074                 }
2075
2076                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2077                 if (info && info->ats_supported)
2078                         translation = CONTEXT_TT_DEV_IOTLB;
2079                 else
2080                         translation = CONTEXT_TT_MULTI_LEVEL;
2081
2082                 context_set_address_root(context, virt_to_phys(pgd));
2083                 context_set_address_width(context, iommu->agaw);
2084         } else {
2085                 /*
2086                  * In pass through mode, AW must be programmed to
2087                  * indicate the largest AGAW value supported by
2088                  * hardware. And ASR is ignored by hardware.
2089                  */
2090                 context_set_address_width(context, iommu->msagaw);
2091         }
2092
2093         context_set_translation_type(context, translation);
2094         context_set_fault_enable(context);
2095         context_set_present(context);
2096         domain_flush_cache(domain, context, sizeof(*context));
2097
2098         /*
2099          * It's a non-present to present mapping. If hardware doesn't cache
2100          * non-present entry we only need to flush the write-buffer. If the
2101          * _does_ cache non-present entries, then it does so in the special
2102          * domain #0, which we have to flush:
2103          */
2104         if (cap_caching_mode(iommu->cap)) {
2105                 iommu->flush.flush_context(iommu, 0,
2106                                            (((u16)bus) << 8) | devfn,
2107                                            DMA_CCMD_MASK_NOBIT,
2108                                            DMA_CCMD_DEVICE_INVL);
2109                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2110         } else {
2111                 iommu_flush_write_buffer(iommu);
2112         }
2113         iommu_enable_dev_iotlb(info);
2114
2115         ret = 0;
2116
2117 out_unlock:
2118         spin_unlock(&iommu->lock);
2119         spin_unlock_irqrestore(&device_domain_lock, flags);
2120
2121         return ret;
2122 }
2123
2124 struct domain_context_mapping_data {
2125         struct dmar_domain *domain;
2126         struct intel_iommu *iommu;
2127 };
2128
2129 static int domain_context_mapping_cb(struct pci_dev *pdev,
2130                                      u16 alias, void *opaque)
2131 {
2132         struct domain_context_mapping_data *data = opaque;
2133
2134         return domain_context_mapping_one(data->domain, data->iommu,
2135                                           PCI_BUS_NUM(alias), alias & 0xff);
2136 }
2137
2138 static int
2139 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2140 {
2141         struct intel_iommu *iommu;
2142         u8 bus, devfn;
2143         struct domain_context_mapping_data data;
2144
2145         iommu = device_to_iommu(dev, &bus, &devfn);
2146         if (!iommu)
2147                 return -ENODEV;
2148
2149         if (!dev_is_pci(dev))
2150                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2151
2152         data.domain = domain;
2153         data.iommu = iommu;
2154
2155         return pci_for_each_dma_alias(to_pci_dev(dev),
2156                                       &domain_context_mapping_cb, &data);
2157 }
2158
2159 static int domain_context_mapped_cb(struct pci_dev *pdev,
2160                                     u16 alias, void *opaque)
2161 {
2162         struct intel_iommu *iommu = opaque;
2163
2164         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2165 }
2166
2167 static int domain_context_mapped(struct device *dev)
2168 {
2169         struct intel_iommu *iommu;
2170         u8 bus, devfn;
2171
2172         iommu = device_to_iommu(dev, &bus, &devfn);
2173         if (!iommu)
2174                 return -ENODEV;
2175
2176         if (!dev_is_pci(dev))
2177                 return device_context_mapped(iommu, bus, devfn);
2178
2179         return !pci_for_each_dma_alias(to_pci_dev(dev),
2180                                        domain_context_mapped_cb, iommu);
2181 }
2182
2183 /* Returns a number of VTD pages, but aligned to MM page size */
2184 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2185                                             size_t size)
2186 {
2187         host_addr &= ~PAGE_MASK;
2188         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2189 }
2190
2191 /* Return largest possible superpage level for a given mapping */
2192 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2193                                           unsigned long iov_pfn,
2194                                           unsigned long phy_pfn,
2195                                           unsigned long pages)
2196 {
2197         int support, level = 1;
2198         unsigned long pfnmerge;
2199
2200         support = domain->iommu_superpage;
2201
2202         /* To use a large page, the virtual *and* physical addresses
2203            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2204            of them will mean we have to use smaller pages. So just
2205            merge them and check both at once. */
2206         pfnmerge = iov_pfn | phy_pfn;
2207
2208         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2209                 pages >>= VTD_STRIDE_SHIFT;
2210                 if (!pages)
2211                         break;
2212                 pfnmerge >>= VTD_STRIDE_SHIFT;
2213                 level++;
2214                 support--;
2215         }
2216         return level;
2217 }
2218
2219 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2220                             struct scatterlist *sg, unsigned long phys_pfn,
2221                             unsigned long nr_pages, int prot)
2222 {
2223         struct dma_pte *first_pte = NULL, *pte = NULL;
2224         phys_addr_t uninitialized_var(pteval);
2225         unsigned long sg_res = 0;
2226         unsigned int largepage_lvl = 0;
2227         unsigned long lvl_pages = 0;
2228
2229         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2230
2231         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2232                 return -EINVAL;
2233
2234         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2235
2236         if (!sg) {
2237                 sg_res = nr_pages;
2238                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2239         }
2240
2241         while (nr_pages > 0) {
2242                 uint64_t tmp;
2243
2244                 if (!sg_res) {
2245                         sg_res = aligned_nrpages(sg->offset, sg->length);
2246                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2247                         sg->dma_length = sg->length;
2248                         pteval = page_to_phys(sg_page(sg)) | prot;
2249                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250                 }
2251
2252                 if (!pte) {
2253                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2254
2255                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2256                         if (!pte)
2257                                 return -ENOMEM;
2258                         /* It is large page*/
2259                         if (largepage_lvl > 1) {
2260                                 unsigned long nr_superpages, end_pfn;
2261
2262                                 pteval |= DMA_PTE_LARGE_PAGE;
2263                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2264
2265                                 nr_superpages = sg_res / lvl_pages;
2266                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2267
2268                                 /*
2269                                  * Ensure that old small page tables are
2270                                  * removed to make room for superpage(s).
2271                                  */
2272                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2273                         } else {
2274                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2275                         }
2276
2277                 }
2278                 /* We don't need lock here, nobody else
2279                  * touches the iova range
2280                  */
2281                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2282                 if (tmp) {
2283                         static int dumps = 5;
2284                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2285                                 iov_pfn, tmp, (unsigned long long)pteval);
2286                         if (dumps) {
2287                                 dumps--;
2288                                 debug_dma_dump_mappings(NULL);
2289                         }
2290                         WARN_ON(1);
2291                 }
2292
2293                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2294
2295                 BUG_ON(nr_pages < lvl_pages);
2296                 BUG_ON(sg_res < lvl_pages);
2297
2298                 nr_pages -= lvl_pages;
2299                 iov_pfn += lvl_pages;
2300                 phys_pfn += lvl_pages;
2301                 pteval += lvl_pages * VTD_PAGE_SIZE;
2302                 sg_res -= lvl_pages;
2303
2304                 /* If the next PTE would be the first in a new page, then we
2305                    need to flush the cache on the entries we've just written.
2306                    And then we'll need to recalculate 'pte', so clear it and
2307                    let it get set again in the if (!pte) block above.
2308
2309                    If we're done (!nr_pages) we need to flush the cache too.
2310
2311                    Also if we've been setting superpages, we may need to
2312                    recalculate 'pte' and switch back to smaller pages for the
2313                    end of the mapping, if the trailing size is not enough to
2314                    use another superpage (i.e. sg_res < lvl_pages). */
2315                 pte++;
2316                 if (!nr_pages || first_pte_in_page(pte) ||
2317                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2318                         domain_flush_cache(domain, first_pte,
2319                                            (void *)pte - (void *)first_pte);
2320                         pte = NULL;
2321                 }
2322
2323                 if (!sg_res && nr_pages)
2324                         sg = sg_next(sg);
2325         }
2326         return 0;
2327 }
2328
2329 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2330                                     struct scatterlist *sg, unsigned long nr_pages,
2331                                     int prot)
2332 {
2333         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2334 }
2335
2336 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2337                                      unsigned long phys_pfn, unsigned long nr_pages,
2338                                      int prot)
2339 {
2340         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2341 }
2342
2343 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2344 {
2345         if (!iommu)
2346                 return;
2347
2348         clear_context_table(iommu, bus, devfn);
2349         iommu->flush.flush_context(iommu, 0, 0, 0,
2350                                            DMA_CCMD_GLOBAL_INVL);
2351         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2352 }
2353
2354 static inline void unlink_domain_info(struct device_domain_info *info)
2355 {
2356         assert_spin_locked(&device_domain_lock);
2357         list_del(&info->link);
2358         list_del(&info->global);
2359         if (info->dev)
2360                 info->dev->archdata.iommu = NULL;
2361 }
2362
2363 static void domain_remove_dev_info(struct dmar_domain *domain)
2364 {
2365         struct device_domain_info *info, *tmp;
2366         unsigned long flags;
2367
2368         spin_lock_irqsave(&device_domain_lock, flags);
2369         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2370                 __dmar_remove_one_dev_info(info);
2371         spin_unlock_irqrestore(&device_domain_lock, flags);
2372 }
2373
2374 /*
2375  * find_domain
2376  * Note: we use struct device->archdata.iommu stores the info
2377  */
2378 static struct dmar_domain *find_domain(struct device *dev)
2379 {
2380         struct device_domain_info *info;
2381
2382         /* No lock here, assumes no domain exit in normal case */
2383         info = dev->archdata.iommu;
2384         if (info)
2385                 return info->domain;
2386         return NULL;
2387 }
2388
2389 static inline struct device_domain_info *
2390 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2391 {
2392         struct device_domain_info *info;
2393
2394         list_for_each_entry(info, &device_domain_list, global)
2395                 if (info->iommu->segment == segment && info->bus == bus &&
2396                     info->devfn == devfn)
2397                         return info;
2398
2399         return NULL;
2400 }
2401
2402 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2403                                                     int bus, int devfn,
2404                                                     struct device *dev,
2405                                                     struct dmar_domain *domain)
2406 {
2407         struct dmar_domain *found = NULL;
2408         struct device_domain_info *info;
2409         unsigned long flags;
2410         int ret;
2411
2412         info = alloc_devinfo_mem();
2413         if (!info)
2414                 return NULL;
2415
2416         info->bus = bus;
2417         info->devfn = devfn;
2418         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2419         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2420         info->ats_qdep = 0;
2421         info->dev = dev;
2422         info->domain = domain;
2423         info->iommu = iommu;
2424
2425         if (dev && dev_is_pci(dev)) {
2426                 struct pci_dev *pdev = to_pci_dev(info->dev);
2427
2428                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2429                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2430                     dmar_find_matched_atsr_unit(pdev))
2431                         info->ats_supported = 1;
2432
2433                 if (ecs_enabled(iommu)) {
2434                         if (pasid_enabled(iommu)) {
2435                                 int features = pci_pasid_features(pdev);
2436                                 if (features >= 0)
2437                                         info->pasid_supported = features | 1;
2438                         }
2439
2440                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2441                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2442                                 info->pri_supported = 1;
2443                 }
2444         }
2445
2446         spin_lock_irqsave(&device_domain_lock, flags);
2447         if (dev)
2448                 found = find_domain(dev);
2449
2450         if (!found) {
2451                 struct device_domain_info *info2;
2452                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2453                 if (info2) {
2454                         found      = info2->domain;
2455                         info2->dev = dev;
2456                 }
2457         }
2458
2459         if (found) {
2460                 spin_unlock_irqrestore(&device_domain_lock, flags);
2461                 free_devinfo_mem(info);
2462                 /* Caller must free the original domain */
2463                 return found;
2464         }
2465
2466         spin_lock(&iommu->lock);
2467         ret = domain_attach_iommu(domain, iommu);
2468         spin_unlock(&iommu->lock);
2469
2470         if (ret) {
2471                 spin_unlock_irqrestore(&device_domain_lock, flags);
2472                 free_devinfo_mem(info);
2473                 return NULL;
2474         }
2475
2476         list_add(&info->link, &domain->devices);
2477         list_add(&info->global, &device_domain_list);
2478         if (dev)
2479                 dev->archdata.iommu = info;
2480         spin_unlock_irqrestore(&device_domain_lock, flags);
2481
2482         if (dev && domain_context_mapping(domain, dev)) {
2483                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2484                 dmar_remove_one_dev_info(domain, dev);
2485                 return NULL;
2486         }
2487
2488         return domain;
2489 }
2490
2491 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2492 {
2493         *(u16 *)opaque = alias;
2494         return 0;
2495 }
2496
2497 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2498 {
2499         struct device_domain_info *info = NULL;
2500         struct dmar_domain *domain = NULL;
2501         struct intel_iommu *iommu;
2502         u16 req_id, dma_alias;
2503         unsigned long flags;
2504         u8 bus, devfn;
2505
2506         iommu = device_to_iommu(dev, &bus, &devfn);
2507         if (!iommu)
2508                 return NULL;
2509
2510         req_id = ((u16)bus << 8) | devfn;
2511
2512         if (dev_is_pci(dev)) {
2513                 struct pci_dev *pdev = to_pci_dev(dev);
2514
2515                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2516
2517                 spin_lock_irqsave(&device_domain_lock, flags);
2518                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2519                                                       PCI_BUS_NUM(dma_alias),
2520                                                       dma_alias & 0xff);
2521                 if (info) {
2522                         iommu = info->iommu;
2523                         domain = info->domain;
2524                 }
2525                 spin_unlock_irqrestore(&device_domain_lock, flags);
2526
2527                 /* DMA alias already has a domain, use it */
2528                 if (info)
2529                         goto out;
2530         }
2531
2532         /* Allocate and initialize new domain for the device */
2533         domain = alloc_domain(0);
2534         if (!domain)
2535                 return NULL;
2536         if (domain_init(domain, iommu, gaw)) {
2537                 domain_exit(domain);
2538                 return NULL;
2539         }
2540
2541 out:
2542
2543         return domain;
2544 }
2545
2546 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2547                                               struct dmar_domain *domain)
2548 {
2549         struct intel_iommu *iommu;
2550         struct dmar_domain *tmp;
2551         u16 req_id, dma_alias;
2552         u8 bus, devfn;
2553
2554         iommu = device_to_iommu(dev, &bus, &devfn);
2555         if (!iommu)
2556                 return NULL;
2557
2558         req_id = ((u16)bus << 8) | devfn;
2559
2560         if (dev_is_pci(dev)) {
2561                 struct pci_dev *pdev = to_pci_dev(dev);
2562
2563                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2564
2565                 /* register PCI DMA alias device */
2566                 if (req_id != dma_alias) {
2567                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2568                                         dma_alias & 0xff, NULL, domain);
2569
2570                         if (!tmp || tmp != domain)
2571                                 return tmp;
2572                 }
2573         }
2574
2575         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2576         if (!tmp || tmp != domain)
2577                 return tmp;
2578
2579         return domain;
2580 }
2581
2582 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2583 {
2584         struct dmar_domain *domain, *tmp;
2585
2586         domain = find_domain(dev);
2587         if (domain)
2588                 goto out;
2589
2590         domain = find_or_alloc_domain(dev, gaw);
2591         if (!domain)
2592                 goto out;
2593
2594         tmp = set_domain_for_dev(dev, domain);
2595         if (!tmp || domain != tmp) {
2596                 domain_exit(domain);
2597                 domain = tmp;
2598         }
2599
2600 out:
2601
2602         return domain;
2603 }
2604
2605 static int iommu_domain_identity_map(struct dmar_domain *domain,
2606                                      unsigned long long start,
2607                                      unsigned long long end)
2608 {
2609         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2610         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2611
2612         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2613                           dma_to_mm_pfn(last_vpfn))) {
2614                 pr_err("Reserving iova failed\n");
2615                 return -ENOMEM;
2616         }
2617
2618         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2619         /*
2620          * RMRR range might have overlap with physical memory range,
2621          * clear it first
2622          */
2623         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2624
2625         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2626                                   last_vpfn - first_vpfn + 1,
2627                                   DMA_PTE_READ|DMA_PTE_WRITE);
2628 }
2629
2630 static int domain_prepare_identity_map(struct device *dev,
2631                                        struct dmar_domain *domain,
2632                                        unsigned long long start,
2633                                        unsigned long long end)
2634 {
2635         /* For _hardware_ passthrough, don't bother. But for software
2636            passthrough, we do it anyway -- it may indicate a memory
2637            range which is reserved in E820, so which didn't get set
2638            up to start with in si_domain */
2639         if (domain == si_domain && hw_pass_through) {
2640                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2641                         dev_name(dev), start, end);
2642                 return 0;
2643         }
2644
2645         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2646                 dev_name(dev), start, end);
2647
2648         if (end < start) {
2649                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2650                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2651                         dmi_get_system_info(DMI_BIOS_VENDOR),
2652                         dmi_get_system_info(DMI_BIOS_VERSION),
2653                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2654                 return -EIO;
2655         }
2656
2657         if (end >> agaw_to_width(domain->agaw)) {
2658                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2659                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2660                      agaw_to_width(domain->agaw),
2661                      dmi_get_system_info(DMI_BIOS_VENDOR),
2662                      dmi_get_system_info(DMI_BIOS_VERSION),
2663                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2664                 return -EIO;
2665         }
2666
2667         return iommu_domain_identity_map(domain, start, end);
2668 }
2669
2670 static int iommu_prepare_identity_map(struct device *dev,
2671                                       unsigned long long start,
2672                                       unsigned long long end)
2673 {
2674         struct dmar_domain *domain;
2675         int ret;
2676
2677         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2678         if (!domain)
2679                 return -ENOMEM;
2680
2681         ret = domain_prepare_identity_map(dev, domain, start, end);
2682         if (ret)
2683                 domain_exit(domain);
2684
2685         return ret;
2686 }
2687
2688 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2689                                          struct device *dev)
2690 {
2691         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2692                 return 0;
2693         return iommu_prepare_identity_map(dev, rmrr->base_address,
2694                                           rmrr->end_address);
2695 }
2696
2697 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2698 static inline void iommu_prepare_isa(void)
2699 {
2700         struct pci_dev *pdev;
2701         int ret;
2702
2703         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2704         if (!pdev)
2705                 return;
2706
2707         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2708         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2709
2710         if (ret)
2711                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2712
2713         pci_dev_put(pdev);
2714 }
2715 #else
2716 static inline void iommu_prepare_isa(void)
2717 {
2718         return;
2719 }
2720 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2721
2722 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2723
2724 static int __init si_domain_init(int hw)
2725 {
2726         int nid, ret = 0;
2727
2728         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2729         if (!si_domain)
2730                 return -EFAULT;
2731
2732         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2733                 domain_exit(si_domain);
2734                 return -EFAULT;
2735         }
2736
2737         pr_debug("Identity mapping domain allocated\n");
2738
2739         if (hw)
2740                 return 0;
2741
2742         for_each_online_node(nid) {
2743                 unsigned long start_pfn, end_pfn;
2744                 int i;
2745
2746                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2747                         ret = iommu_domain_identity_map(si_domain,
2748                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2749                         if (ret)
2750                                 return ret;
2751                 }
2752         }
2753
2754         return 0;
2755 }
2756
2757 static int identity_mapping(struct device *dev)
2758 {
2759         struct device_domain_info *info;
2760
2761         if (likely(!iommu_identity_mapping))
2762                 return 0;
2763
2764         info = dev->archdata.iommu;
2765         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2766                 return (info->domain == si_domain);
2767
2768         return 0;
2769 }
2770
2771 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2772 {
2773         struct dmar_domain *ndomain;
2774         struct intel_iommu *iommu;
2775         u8 bus, devfn;
2776
2777         iommu = device_to_iommu(dev, &bus, &devfn);
2778         if (!iommu)
2779                 return -ENODEV;
2780
2781         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2782         if (ndomain != domain)
2783                 return -EBUSY;
2784
2785         return 0;
2786 }
2787
2788 static bool device_has_rmrr(struct device *dev)
2789 {
2790         struct dmar_rmrr_unit *rmrr;
2791         struct device *tmp;
2792         int i;
2793
2794         rcu_read_lock();
2795         for_each_rmrr_units(rmrr) {
2796                 /*
2797                  * Return TRUE if this RMRR contains the device that
2798                  * is passed in.
2799                  */
2800                 for_each_active_dev_scope(rmrr->devices,
2801                                           rmrr->devices_cnt, i, tmp)
2802                         if (tmp == dev) {
2803                                 rcu_read_unlock();
2804                                 return true;
2805                         }
2806         }
2807         rcu_read_unlock();
2808         return false;
2809 }
2810
2811 /*
2812  * There are a couple cases where we need to restrict the functionality of
2813  * devices associated with RMRRs.  The first is when evaluating a device for
2814  * identity mapping because problems exist when devices are moved in and out
2815  * of domains and their respective RMRR information is lost.  This means that
2816  * a device with associated RMRRs will never be in a "passthrough" domain.
2817  * The second is use of the device through the IOMMU API.  This interface
2818  * expects to have full control of the IOVA space for the device.  We cannot
2819  * satisfy both the requirement that RMRR access is maintained and have an
2820  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2821  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2822  * We therefore prevent devices associated with an RMRR from participating in
2823  * the IOMMU API, which eliminates them from device assignment.
2824  *
2825  * In both cases we assume that PCI USB devices with RMRRs have them largely
2826  * for historical reasons and that the RMRR space is not actively used post
2827  * boot.  This exclusion may change if vendors begin to abuse it.
2828  *
2829  * The same exception is made for graphics devices, with the requirement that
2830  * any use of the RMRR regions will be torn down before assigning the device
2831  * to a guest.
2832  */
2833 static bool device_is_rmrr_locked(struct device *dev)
2834 {
2835         if (!device_has_rmrr(dev))
2836                 return false;
2837
2838         if (dev_is_pci(dev)) {
2839                 struct pci_dev *pdev = to_pci_dev(dev);
2840
2841                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2842                         return false;
2843         }
2844
2845         return true;
2846 }
2847
2848 static int iommu_should_identity_map(struct device *dev, int startup)
2849 {
2850
2851         if (dev_is_pci(dev)) {
2852                 struct pci_dev *pdev = to_pci_dev(dev);
2853
2854                 if (device_is_rmrr_locked(dev))
2855                         return 0;
2856
2857                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2858                         return 1;
2859
2860                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2861                         return 1;
2862
2863                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2864                         return 0;
2865
2866                 /*
2867                  * We want to start off with all devices in the 1:1 domain, and
2868                  * take them out later if we find they can't access all of memory.
2869                  *
2870                  * However, we can't do this for PCI devices behind bridges,
2871                  * because all PCI devices behind the same bridge will end up
2872                  * with the same source-id on their transactions.
2873                  *
2874                  * Practically speaking, we can't change things around for these
2875                  * devices at run-time, because we can't be sure there'll be no
2876                  * DMA transactions in flight for any of their siblings.
2877                  *
2878                  * So PCI devices (unless they're on the root bus) as well as
2879                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2880                  * the 1:1 domain, just in _case_ one of their siblings turns out
2881                  * not to be able to map all of memory.
2882                  */
2883                 if (!pci_is_pcie(pdev)) {
2884                         if (!pci_is_root_bus(pdev->bus))
2885                                 return 0;
2886                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2887                                 return 0;
2888                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2889                         return 0;
2890         } else {
2891                 if (device_has_rmrr(dev))
2892                         return 0;
2893         }
2894
2895         /*
2896          * At boot time, we don't yet know if devices will be 64-bit capable.
2897          * Assume that they will — if they turn out not to be, then we can
2898          * take them out of the 1:1 domain later.
2899          */
2900         if (!startup) {
2901                 /*
2902                  * If the device's dma_mask is less than the system's memory
2903                  * size then this is not a candidate for identity mapping.
2904                  */
2905                 u64 dma_mask = *dev->dma_mask;
2906
2907                 if (dev->coherent_dma_mask &&
2908                     dev->coherent_dma_mask < dma_mask)
2909                         dma_mask = dev->coherent_dma_mask;
2910
2911                 return dma_mask >= dma_get_required_mask(dev);
2912         }
2913
2914         return 1;
2915 }
2916
2917 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2918 {
2919         int ret;
2920
2921         if (!iommu_should_identity_map(dev, 1))
2922                 return 0;
2923
2924         ret = domain_add_dev_info(si_domain, dev);
2925         if (!ret)
2926                 pr_info("%s identity mapping for device %s\n",
2927                         hw ? "Hardware" : "Software", dev_name(dev));
2928         else if (ret == -ENODEV)
2929                 /* device not associated with an iommu */
2930                 ret = 0;
2931
2932         return ret;
2933 }
2934
2935
2936 static int __init iommu_prepare_static_identity_mapping(int hw)
2937 {
2938         struct pci_dev *pdev = NULL;
2939         struct dmar_drhd_unit *drhd;
2940         struct intel_iommu *iommu;
2941         struct device *dev;
2942         int i;
2943         int ret = 0;
2944
2945         for_each_pci_dev(pdev) {
2946                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2947                 if (ret)
2948                         return ret;
2949         }
2950
2951         for_each_active_iommu(iommu, drhd)
2952                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2953                         struct acpi_device_physical_node *pn;
2954                         struct acpi_device *adev;
2955
2956                         if (dev->bus != &acpi_bus_type)
2957                                 continue;
2958
2959                         adev= to_acpi_device(dev);
2960                         mutex_lock(&adev->physical_node_lock);
2961                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2962                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2963                                 if (ret)
2964                                         break;
2965                         }
2966                         mutex_unlock(&adev->physical_node_lock);
2967                         if (ret)
2968                                 return ret;
2969                 }
2970
2971         return 0;
2972 }
2973
2974 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2975 {
2976         /*
2977          * Start from the sane iommu hardware state.
2978          * If the queued invalidation is already initialized by us
2979          * (for example, while enabling interrupt-remapping) then
2980          * we got the things already rolling from a sane state.
2981          */
2982         if (!iommu->qi) {
2983                 /*
2984                  * Clear any previous faults.
2985                  */
2986                 dmar_fault(-1, iommu);
2987                 /*
2988                  * Disable queued invalidation if supported and already enabled
2989                  * before OS handover.
2990                  */
2991                 dmar_disable_qi(iommu);
2992         }
2993
2994         if (dmar_enable_qi(iommu)) {
2995                 /*
2996                  * Queued Invalidate not enabled, use Register Based Invalidate
2997                  */
2998                 iommu->flush.flush_context = __iommu_flush_context;
2999                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3000                 pr_info("%s: Using Register based invalidation\n",
3001                         iommu->name);
3002         } else {
3003                 iommu->flush.flush_context = qi_flush_context;
3004                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3005                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3006         }
3007 }
3008
3009 static int copy_context_table(struct intel_iommu *iommu,
3010                               struct root_entry *old_re,
3011                               struct context_entry **tbl,
3012                               int bus, bool ext)
3013 {
3014         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3015         struct context_entry *new_ce = NULL, ce;
3016         struct context_entry *old_ce = NULL;
3017         struct root_entry re;
3018         phys_addr_t old_ce_phys;
3019
3020         tbl_idx = ext ? bus * 2 : bus;
3021         memcpy(&re, old_re, sizeof(re));
3022
3023         for (devfn = 0; devfn < 256; devfn++) {
3024                 /* First calculate the correct index */
3025                 idx = (ext ? devfn * 2 : devfn) % 256;
3026
3027                 if (idx == 0) {
3028                         /* First save what we may have and clean up */
3029                         if (new_ce) {
3030                                 tbl[tbl_idx] = new_ce;
3031                                 __iommu_flush_cache(iommu, new_ce,
3032                                                     VTD_PAGE_SIZE);
3033                                 pos = 1;
3034                         }
3035
3036                         if (old_ce)
3037                                 iounmap(old_ce);
3038
3039                         ret = 0;
3040                         if (devfn < 0x80)
3041                                 old_ce_phys = root_entry_lctp(&re);
3042                         else
3043                                 old_ce_phys = root_entry_uctp(&re);
3044
3045                         if (!old_ce_phys) {
3046                                 if (ext && devfn == 0) {
3047                                         /* No LCTP, try UCTP */
3048                                         devfn = 0x7f;
3049                                         continue;
3050                                 } else {
3051                                         goto out;
3052                                 }
3053                         }
3054
3055                         ret = -ENOMEM;
3056                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3057                                         MEMREMAP_WB);
3058                         if (!old_ce)
3059                                 goto out;
3060
3061                         new_ce = alloc_pgtable_page(iommu->node);
3062                         if (!new_ce)
3063                                 goto out_unmap;
3064
3065                         ret = 0;
3066                 }
3067
3068                 /* Now copy the context entry */
3069                 memcpy(&ce, old_ce + idx, sizeof(ce));
3070
3071                 if (!__context_present(&ce))
3072                         continue;
3073
3074                 did = context_domain_id(&ce);
3075                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3076                         set_bit(did, iommu->domain_ids);
3077
3078                 /*
3079                  * We need a marker for copied context entries. This
3080                  * marker needs to work for the old format as well as
3081                  * for extended context entries.
3082                  *
3083                  * Bit 67 of the context entry is used. In the old
3084                  * format this bit is available to software, in the
3085                  * extended format it is the PGE bit, but PGE is ignored
3086                  * by HW if PASIDs are disabled (and thus still
3087                  * available).
3088                  *
3089                  * So disable PASIDs first and then mark the entry
3090                  * copied. This means that we don't copy PASID
3091                  * translations from the old kernel, but this is fine as
3092                  * faults there are not fatal.
3093                  */
3094                 context_clear_pasid_enable(&ce);
3095                 context_set_copied(&ce);
3096
3097                 new_ce[idx] = ce;
3098         }
3099
3100         tbl[tbl_idx + pos] = new_ce;
3101
3102         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3103
3104 out_unmap:
3105         memunmap(old_ce);
3106
3107 out:
3108         return ret;
3109 }
3110
3111 static int copy_translation_tables(struct intel_iommu *iommu)
3112 {
3113         struct context_entry **ctxt_tbls;
3114         struct root_entry *old_rt;
3115         phys_addr_t old_rt_phys;
3116         int ctxt_table_entries;
3117         unsigned long flags;
3118         u64 rtaddr_reg;
3119         int bus, ret;
3120         bool new_ext, ext;
3121
3122         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3123         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3124         new_ext    = !!ecap_ecs(iommu->ecap);
3125
3126         /*
3127          * The RTT bit can only be changed when translation is disabled,
3128          * but disabling translation means to open a window for data
3129          * corruption. So bail out and don't copy anything if we would
3130          * have to change the bit.
3131          */
3132         if (new_ext != ext)
3133                 return -EINVAL;
3134
3135         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3136         if (!old_rt_phys)
3137                 return -EINVAL;
3138
3139         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3140         if (!old_rt)
3141                 return -ENOMEM;
3142
3143         /* This is too big for the stack - allocate it from slab */
3144         ctxt_table_entries = ext ? 512 : 256;
3145         ret = -ENOMEM;
3146         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3147         if (!ctxt_tbls)
3148                 goto out_unmap;
3149
3150         for (bus = 0; bus < 256; bus++) {
3151                 ret = copy_context_table(iommu, &old_rt[bus],
3152                                          ctxt_tbls, bus, ext);
3153                 if (ret) {
3154                         pr_err("%s: Failed to copy context table for bus %d\n",
3155                                 iommu->name, bus);
3156                         continue;
3157                 }
3158         }
3159
3160         spin_lock_irqsave(&iommu->lock, flags);
3161
3162         /* Context tables are copied, now write them to the root_entry table */
3163         for (bus = 0; bus < 256; bus++) {
3164                 int idx = ext ? bus * 2 : bus;
3165                 u64 val;
3166
3167                 if (ctxt_tbls[idx]) {
3168                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3169                         iommu->root_entry[bus].lo = val;
3170                 }
3171
3172                 if (!ext || !ctxt_tbls[idx + 1])
3173                         continue;
3174
3175                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3176                 iommu->root_entry[bus].hi = val;
3177         }
3178
3179         spin_unlock_irqrestore(&iommu->lock, flags);
3180
3181         kfree(ctxt_tbls);
3182
3183         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3184
3185         ret = 0;
3186
3187 out_unmap:
3188         memunmap(old_rt);
3189
3190         return ret;
3191 }
3192
3193 static int __init init_dmars(void)
3194 {
3195         struct dmar_drhd_unit *drhd;
3196         struct dmar_rmrr_unit *rmrr;
3197         bool copied_tables = false;
3198         struct device *dev;
3199         struct intel_iommu *iommu;
3200         int i, ret, cpu;
3201
3202         /*
3203          * for each drhd
3204          *    allocate root
3205          *    initialize and program root entry to not present
3206          * endfor
3207          */
3208         for_each_drhd_unit(drhd) {
3209                 /*
3210                  * lock not needed as this is only incremented in the single
3211                  * threaded kernel __init code path all other access are read
3212                  * only
3213                  */
3214                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3215                         g_num_of_iommus++;
3216                         continue;
3217                 }
3218                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3219         }
3220
3221         /* Preallocate enough resources for IOMMU hot-addition */
3222         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3223                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3224
3225         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3226                         GFP_KERNEL);
3227         if (!g_iommus) {
3228                 pr_err("Allocating global iommu array failed\n");
3229                 ret = -ENOMEM;
3230                 goto error;
3231         }
3232
3233         for_each_possible_cpu(cpu) {
3234                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3235                                                               cpu);
3236
3237                 dfd->tables = kzalloc(g_num_of_iommus *
3238                                       sizeof(struct deferred_flush_table),
3239                                       GFP_KERNEL);
3240                 if (!dfd->tables) {
3241                         ret = -ENOMEM;
3242                         goto free_g_iommus;
3243                 }
3244
3245                 spin_lock_init(&dfd->lock);
3246                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3247         }
3248
3249         for_each_active_iommu(iommu, drhd) {
3250                 g_iommus[iommu->seq_id] = iommu;
3251
3252                 intel_iommu_init_qi(iommu);
3253
3254                 ret = iommu_init_domains(iommu);
3255                 if (ret)
3256                         goto free_iommu;
3257
3258                 init_translation_status(iommu);
3259
3260                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3261                         iommu_disable_translation(iommu);
3262                         clear_translation_pre_enabled(iommu);
3263                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3264                                 iommu->name);
3265                 }
3266
3267                 /*
3268                  * TBD:
3269                  * we could share the same root & context tables
3270                  * among all IOMMU's. Need to Split it later.
3271                  */
3272                 ret = iommu_alloc_root_entry(iommu);
3273                 if (ret)
3274                         goto free_iommu;
3275
3276                 if (translation_pre_enabled(iommu)) {
3277                         pr_info("Translation already enabled - trying to copy translation structures\n");
3278
3279                         ret = copy_translation_tables(iommu);
3280                         if (ret) {
3281                                 /*
3282                                  * We found the IOMMU with translation
3283                                  * enabled - but failed to copy over the
3284                                  * old root-entry table. Try to proceed
3285                                  * by disabling translation now and
3286                                  * allocating a clean root-entry table.
3287                                  * This might cause DMAR faults, but
3288                                  * probably the dump will still succeed.
3289                                  */
3290                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3291                                        iommu->name);
3292                                 iommu_disable_translation(iommu);
3293                                 clear_translation_pre_enabled(iommu);
3294                         } else {
3295                                 pr_info("Copied translation tables from previous kernel for %s\n",
3296                                         iommu->name);
3297                                 copied_tables = true;
3298                         }
3299                 }
3300
3301                 if (!ecap_pass_through(iommu->ecap))
3302                         hw_pass_through = 0;
3303 #ifdef CONFIG_INTEL_IOMMU_SVM
3304                 if (pasid_enabled(iommu))
3305                         intel_svm_alloc_pasid_tables(iommu);
3306 #endif
3307         }
3308
3309         /*
3310          * Now that qi is enabled on all iommus, set the root entry and flush
3311          * caches. This is required on some Intel X58 chipsets, otherwise the
3312          * flush_context function will loop forever and the boot hangs.
3313          */
3314         for_each_active_iommu(iommu, drhd) {
3315                 iommu_flush_write_buffer(iommu);
3316                 iommu_set_root_entry(iommu);
3317                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3318                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3319         }
3320
3321         if (iommu_pass_through)
3322                 iommu_identity_mapping |= IDENTMAP_ALL;
3323
3324 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3325         iommu_identity_mapping |= IDENTMAP_GFX;
3326 #endif
3327
3328         check_tylersburg_isoch();
3329
3330         if (iommu_identity_mapping) {
3331                 ret = si_domain_init(hw_pass_through);
3332                 if (ret)
3333                         goto free_iommu;
3334         }
3335
3336
3337         /*
3338          * If we copied translations from a previous kernel in the kdump
3339          * case, we can not assign the devices to domains now, as that
3340          * would eliminate the old mappings. So skip this part and defer
3341          * the assignment to device driver initialization time.
3342          */
3343         if (copied_tables)
3344                 goto domains_done;
3345
3346         /*
3347          * If pass through is not set or not enabled, setup context entries for
3348          * identity mappings for rmrr, gfx, and isa and may fall back to static
3349          * identity mapping if iommu_identity_mapping is set.
3350          */
3351         if (iommu_identity_mapping) {
3352                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3353                 if (ret) {
3354                         pr_crit("Failed to setup IOMMU pass-through\n");
3355                         goto free_iommu;
3356                 }
3357         }
3358         /*
3359          * For each rmrr
3360          *   for each dev attached to rmrr
3361          *   do
3362          *     locate drhd for dev, alloc domain for dev
3363          *     allocate free domain
3364          *     allocate page table entries for rmrr
3365          *     if context not allocated for bus
3366          *           allocate and init context
3367          *           set present in root table for this bus
3368          *     init context with domain, translation etc
3369          *    endfor
3370          * endfor
3371          */
3372         pr_info("Setting RMRR:\n");
3373         for_each_rmrr_units(rmrr) {
3374                 /* some BIOS lists non-exist devices in DMAR table. */
3375                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3376                                           i, dev) {
3377                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3378                         if (ret)
3379                                 pr_err("Mapping reserved region failed\n");
3380                 }
3381         }
3382
3383         iommu_prepare_isa();
3384
3385 domains_done:
3386
3387         /*
3388          * for each drhd
3389          *   enable fault log
3390          *   global invalidate context cache
3391          *   global invalidate iotlb
3392          *   enable translation
3393          */
3394         for_each_iommu(iommu, drhd) {
3395                 if (drhd->ignored) {
3396                         /*
3397                          * we always have to disable PMRs or DMA may fail on
3398                          * this device
3399                          */
3400                         if (force_on)
3401                                 iommu_disable_protect_mem_regions(iommu);
3402                         continue;
3403                 }
3404
3405                 iommu_flush_write_buffer(iommu);
3406
3407 #ifdef CONFIG_INTEL_IOMMU_SVM
3408                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3409                         ret = intel_svm_enable_prq(iommu);
3410                         if (ret)
3411                                 goto free_iommu;
3412                 }
3413 #endif
3414                 ret = dmar_set_interrupt(iommu);
3415                 if (ret)
3416                         goto free_iommu;
3417
3418                 if (!translation_pre_enabled(iommu))
3419                         iommu_enable_translation(iommu);
3420
3421                 iommu_disable_protect_mem_regions(iommu);
3422         }
3423
3424         return 0;
3425
3426 free_iommu:
3427         for_each_active_iommu(iommu, drhd) {
3428                 disable_dmar_iommu(iommu);
3429                 free_dmar_iommu(iommu);
3430         }
3431 free_g_iommus:
3432         for_each_possible_cpu(cpu)
3433                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3434         kfree(g_iommus);
3435 error:
3436         return ret;
3437 }
3438
3439 /* This takes a number of _MM_ pages, not VTD pages */
3440 static unsigned long intel_alloc_iova(struct device *dev,
3441                                      struct dmar_domain *domain,
3442                                      unsigned long nrpages, uint64_t dma_mask)
3443 {
3444         unsigned long iova_pfn = 0;
3445
3446         /* Restrict dma_mask to the width that the iommu can handle */
3447         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3448         /* Ensure we reserve the whole size-aligned region */
3449         nrpages = __roundup_pow_of_two(nrpages);
3450
3451         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3452                 /*
3453                  * First try to allocate an io virtual address in
3454                  * DMA_BIT_MASK(32) and if that fails then try allocating
3455                  * from higher range
3456                  */
3457                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3458                                            IOVA_PFN(DMA_BIT_MASK(32)));
3459                 if (iova_pfn)
3460                         return iova_pfn;
3461         }
3462         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3463         if (unlikely(!iova_pfn)) {
3464                 pr_err("Allocating %ld-page iova for %s failed",
3465                        nrpages, dev_name(dev));
3466                 return 0;
3467         }
3468
3469         return iova_pfn;
3470 }
3471
3472 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3473 {
3474         struct dmar_domain *domain, *tmp;
3475         struct dmar_rmrr_unit *rmrr;
3476         struct device *i_dev;
3477         int i, ret;
3478
3479         domain = find_domain(dev);
3480         if (domain)
3481                 goto out;
3482
3483         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3484         if (!domain)
3485                 goto out;
3486
3487         /* We have a new domain - setup possible RMRRs for the device */
3488         rcu_read_lock();
3489         for_each_rmrr_units(rmrr) {
3490                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3491                                           i, i_dev) {
3492                         if (i_dev != dev)
3493                                 continue;
3494
3495                         ret = domain_prepare_identity_map(dev, domain,
3496                                                           rmrr->base_address,
3497                                                           rmrr->end_address);
3498                         if (ret)
3499                                 dev_err(dev, "Mapping reserved region failed\n");
3500                 }
3501         }
3502         rcu_read_unlock();
3503
3504         tmp = set_domain_for_dev(dev, domain);
3505         if (!tmp || domain != tmp) {
3506                 domain_exit(domain);
3507                 domain = tmp;
3508         }
3509
3510 out:
3511
3512         if (!domain)
3513                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3514
3515
3516         return domain;
3517 }
3518
3519 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3520 {
3521         struct device_domain_info *info;
3522
3523         /* No lock here, assumes no domain exit in normal case */
3524         info = dev->archdata.iommu;
3525         if (likely(info))
3526                 return info->domain;
3527
3528         return __get_valid_domain_for_dev(dev);
3529 }
3530
3531 /* Check if the dev needs to go through non-identity map and unmap process.*/
3532 static int iommu_no_mapping(struct device *dev)
3533 {
3534         int found;
3535
3536         if (iommu_dummy(dev))
3537                 return 1;
3538
3539         if (!iommu_identity_mapping)
3540                 return 0;
3541
3542         found = identity_mapping(dev);
3543         if (found) {
3544                 if (iommu_should_identity_map(dev, 0))
3545                         return 1;
3546                 else {
3547                         /*
3548                          * 32 bit DMA is removed from si_domain and fall back
3549                          * to non-identity mapping.
3550                          */
3551                         dmar_remove_one_dev_info(si_domain, dev);
3552                         pr_info("32bit %s uses non-identity mapping\n",
3553                                 dev_name(dev));
3554                         return 0;
3555                 }
3556         } else {
3557                 /*
3558                  * In case of a detached 64 bit DMA device from vm, the device
3559                  * is put into si_domain for identity mapping.
3560                  */
3561                 if (iommu_should_identity_map(dev, 0)) {
3562                         int ret;
3563                         ret = domain_add_dev_info(si_domain, dev);
3564                         if (!ret) {
3565                                 pr_info("64bit %s uses identity mapping\n",
3566                                         dev_name(dev));
3567                                 return 1;
3568                         }
3569                 }
3570         }
3571
3572         return 0;
3573 }
3574
3575 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3576                                      size_t size, int dir, u64 dma_mask)
3577 {
3578         struct dmar_domain *domain;
3579         phys_addr_t start_paddr;
3580         unsigned long iova_pfn;
3581         int prot = 0;
3582         int ret;
3583         struct intel_iommu *iommu;
3584         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3585
3586         BUG_ON(dir == DMA_NONE);
3587
3588         if (iommu_no_mapping(dev))
3589                 return paddr;
3590
3591         domain = get_valid_domain_for_dev(dev);
3592         if (!domain)
3593                 return 0;
3594
3595         iommu = domain_get_iommu(domain);
3596         size = aligned_nrpages(paddr, size);
3597
3598         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3599         if (!iova_pfn)
3600                 goto error;
3601
3602         /*
3603          * Check if DMAR supports zero-length reads on write only
3604          * mappings..
3605          */
3606         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3607                         !cap_zlr(iommu->cap))
3608                 prot |= DMA_PTE_READ;
3609         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3610                 prot |= DMA_PTE_WRITE;
3611         /*
3612          * paddr - (paddr + size) might be partial page, we should map the whole
3613          * page.  Note: if two part of one page are separately mapped, we
3614          * might have two guest_addr mapping to the same host paddr, but this
3615          * is not a big problem
3616          */
3617         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3618                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3619         if (ret)
3620                 goto error;
3621
3622         /* it's a non-present to present mapping. Only flush if caching mode */
3623         if (cap_caching_mode(iommu->cap))
3624                 iommu_flush_iotlb_psi(iommu, domain,
3625                                       mm_to_dma_pfn(iova_pfn),
3626                                       size, 0, 1);
3627         else
3628                 iommu_flush_write_buffer(iommu);
3629
3630         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3631         start_paddr += paddr & ~PAGE_MASK;
3632         return start_paddr;
3633
3634 error:
3635         if (iova_pfn)
3636                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3637         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3638                 dev_name(dev), size, (unsigned long long)paddr, dir);
3639         return 0;
3640 }
3641
3642 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3643                                  unsigned long offset, size_t size,
3644                                  enum dma_data_direction dir,
3645                                  unsigned long attrs)
3646 {
3647         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3648                                   dir, *dev->dma_mask);
3649 }
3650
3651 static void flush_unmaps(struct deferred_flush_data *flush_data)
3652 {
3653         int i, j;
3654
3655         flush_data->timer_on = 0;
3656
3657         /* just flush them all */
3658         for (i = 0; i < g_num_of_iommus; i++) {
3659                 struct intel_iommu *iommu = g_iommus[i];
3660                 struct deferred_flush_table *flush_table =
3661                                 &flush_data->tables[i];
3662                 if (!iommu)
3663                         continue;
3664
3665                 if (!flush_table->next)
3666                         continue;
3667
3668                 /* In caching mode, global flushes turn emulation expensive */
3669                 if (!cap_caching_mode(iommu->cap))
3670                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3671                                          DMA_TLB_GLOBAL_FLUSH);
3672                 for (j = 0; j < flush_table->next; j++) {
3673                         unsigned long mask;
3674                         struct deferred_flush_entry *entry =
3675                                                 &flush_table->entries[j];
3676                         unsigned long iova_pfn = entry->iova_pfn;
3677                         unsigned long nrpages = entry->nrpages;
3678                         struct dmar_domain *domain = entry->domain;
3679                         struct page *freelist = entry->freelist;
3680
3681                         /* On real hardware multiple invalidations are expensive */
3682                         if (cap_caching_mode(iommu->cap))
3683                                 iommu_flush_iotlb_psi(iommu, domain,
3684                                         mm_to_dma_pfn(iova_pfn),
3685                                         nrpages, !freelist, 0);
3686                         else {
3687                                 mask = ilog2(nrpages);
3688                                 iommu_flush_dev_iotlb(domain,
3689                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3690                         }
3691                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3692                         if (freelist)
3693                                 dma_free_pagelist(freelist);
3694                 }
3695                 flush_table->next = 0;
3696         }
3697
3698         flush_data->size = 0;
3699 }
3700
3701 static void flush_unmaps_timeout(unsigned long cpuid)
3702 {
3703         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3704         unsigned long flags;
3705
3706         spin_lock_irqsave(&flush_data->lock, flags);
3707         flush_unmaps(flush_data);
3708         spin_unlock_irqrestore(&flush_data->lock, flags);
3709 }
3710
3711 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3712                       unsigned long nrpages, struct page *freelist)
3713 {
3714         unsigned long flags;
3715         int entry_id, iommu_id;
3716         struct intel_iommu *iommu;
3717         struct deferred_flush_entry *entry;
3718         struct deferred_flush_data *flush_data;
3719         unsigned int cpuid;
3720
3721         cpuid = get_cpu();
3722         flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3723
3724         /* Flush all CPUs' entries to avoid deferring too much.  If
3725          * this becomes a bottleneck, can just flush us, and rely on
3726          * flush timer for the rest.
3727          */
3728         if (flush_data->size == HIGH_WATER_MARK) {
3729                 int cpu;
3730
3731                 for_each_online_cpu(cpu)
3732                         flush_unmaps_timeout(cpu);
3733         }
3734
3735         spin_lock_irqsave(&flush_data->lock, flags);
3736
3737         iommu = domain_get_iommu(dom);
3738         iommu_id = iommu->seq_id;
3739
3740         entry_id = flush_data->tables[iommu_id].next;
3741         ++(flush_data->tables[iommu_id].next);
3742
3743         entry = &flush_data->tables[iommu_id].entries[entry_id];
3744         entry->domain = dom;
3745         entry->iova_pfn = iova_pfn;
3746         entry->nrpages = nrpages;
3747         entry->freelist = freelist;
3748
3749         if (!flush_data->timer_on) {
3750                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3751                 flush_data->timer_on = 1;
3752         }
3753         flush_data->size++;
3754         spin_unlock_irqrestore(&flush_data->lock, flags);
3755
3756         put_cpu();
3757 }
3758
3759 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3760 {
3761         struct dmar_domain *domain;
3762         unsigned long start_pfn, last_pfn;
3763         unsigned long nrpages;
3764         unsigned long iova_pfn;
3765         struct intel_iommu *iommu;
3766         struct page *freelist;
3767
3768         if (iommu_no_mapping(dev))
3769                 return;
3770
3771         domain = find_domain(dev);
3772         BUG_ON(!domain);
3773
3774         iommu = domain_get_iommu(domain);
3775
3776         iova_pfn = IOVA_PFN(dev_addr);
3777
3778         nrpages = aligned_nrpages(dev_addr, size);
3779         start_pfn = mm_to_dma_pfn(iova_pfn);
3780         last_pfn = start_pfn + nrpages - 1;
3781
3782         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3783                  dev_name(dev), start_pfn, last_pfn);
3784
3785         freelist = domain_unmap(domain, start_pfn, last_pfn);
3786
3787         if (intel_iommu_strict) {
3788                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3789                                       nrpages, !freelist, 0);
3790                 /* free iova */
3791                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3792                 dma_free_pagelist(freelist);
3793         } else {
3794                 add_unmap(domain, iova_pfn, nrpages, freelist);
3795                 /*
3796                  * queue up the release of the unmap to save the 1/6th of the
3797                  * cpu used up by the iotlb flush operation...
3798                  */
3799         }
3800 }
3801
3802 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3803                              size_t size, enum dma_data_direction dir,
3804                              unsigned long attrs)
3805 {
3806         intel_unmap(dev, dev_addr, size);
3807 }
3808
3809 static void *intel_alloc_coherent(struct device *dev, size_t size,
3810                                   dma_addr_t *dma_handle, gfp_t flags,
3811                                   unsigned long attrs)
3812 {
3813         struct page *page = NULL;
3814         int order;
3815
3816         size = PAGE_ALIGN(size);
3817         order = get_order(size);
3818
3819         if (!iommu_no_mapping(dev))
3820                 flags &= ~(GFP_DMA | GFP_DMA32);
3821         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3822                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3823                         flags |= GFP_DMA;
3824                 else
3825                         flags |= GFP_DMA32;
3826         }
3827
3828         if (gfpflags_allow_blocking(flags)) {
3829                 unsigned int count = size >> PAGE_SHIFT;
3830
3831                 page = dma_alloc_from_contiguous(dev, count, order);
3832                 if (page && iommu_no_mapping(dev) &&
3833                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3834                         dma_release_from_contiguous(dev, page, count);
3835                         page = NULL;
3836                 }
3837         }
3838
3839         if (!page)
3840                 page = alloc_pages(flags, order);
3841         if (!page)
3842                 return NULL;
3843         memset(page_address(page), 0, size);
3844
3845         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3846                                          DMA_BIDIRECTIONAL,
3847                                          dev->coherent_dma_mask);
3848         if (*dma_handle)
3849                 return page_address(page);
3850         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3851                 __free_pages(page, order);
3852
3853         return NULL;
3854 }
3855
3856 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3857                                 dma_addr_t dma_handle, unsigned long attrs)
3858 {
3859         int order;
3860         struct page *page = virt_to_page(vaddr);
3861
3862         size = PAGE_ALIGN(size);
3863         order = get_order(size);
3864
3865         intel_unmap(dev, dma_handle, size);
3866         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3867                 __free_pages(page, order);
3868 }
3869
3870 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3871                            int nelems, enum dma_data_direction dir,
3872                            unsigned long attrs)
3873 {
3874         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3875         unsigned long nrpages = 0;
3876         struct scatterlist *sg;
3877         int i;
3878
3879         for_each_sg(sglist, sg, nelems, i) {
3880                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3881         }
3882
3883         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3884 }
3885
3886 static int intel_nontranslate_map_sg(struct device *hddev,
3887         struct scatterlist *sglist, int nelems, int dir)
3888 {
3889         int i;
3890         struct scatterlist *sg;
3891
3892         for_each_sg(sglist, sg, nelems, i) {
3893                 BUG_ON(!sg_page(sg));
3894                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3895                 sg->dma_length = sg->length;
3896         }
3897         return nelems;
3898 }
3899
3900 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3901                         enum dma_data_direction dir, unsigned long attrs)
3902 {
3903         int i;
3904         struct dmar_domain *domain;
3905         size_t size = 0;
3906         int prot = 0;
3907         unsigned long iova_pfn;
3908         int ret;
3909         struct scatterlist *sg;
3910         unsigned long start_vpfn;
3911         struct intel_iommu *iommu;
3912
3913         BUG_ON(dir == DMA_NONE);
3914         if (iommu_no_mapping(dev))
3915                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3916
3917         domain = get_valid_domain_for_dev(dev);
3918         if (!domain)
3919                 return 0;
3920
3921         iommu = domain_get_iommu(domain);
3922
3923         for_each_sg(sglist, sg, nelems, i)
3924                 size += aligned_nrpages(sg->offset, sg->length);
3925
3926         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3927                                 *dev->dma_mask);
3928         if (!iova_pfn) {
3929                 sglist->dma_length = 0;
3930                 return 0;
3931         }
3932
3933         /*
3934          * Check if DMAR supports zero-length reads on write only
3935          * mappings..
3936          */
3937         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3938                         !cap_zlr(iommu->cap))
3939                 prot |= DMA_PTE_READ;
3940         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3941                 prot |= DMA_PTE_WRITE;
3942
3943         start_vpfn = mm_to_dma_pfn(iova_pfn);
3944
3945         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3946         if (unlikely(ret)) {
3947                 dma_pte_free_pagetable(domain, start_vpfn,
3948                                        start_vpfn + size - 1);
3949                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3950                 return 0;
3951         }
3952
3953         /* it's a non-present to present mapping. Only flush if caching mode */
3954         if (cap_caching_mode(iommu->cap))
3955                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3956         else
3957                 iommu_flush_write_buffer(iommu);
3958
3959         return nelems;
3960 }
3961
3962 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3963 {
3964         return !dma_addr;
3965 }
3966
3967 struct dma_map_ops intel_dma_ops = {
3968         .alloc = intel_alloc_coherent,
3969         .free = intel_free_coherent,
3970         .map_sg = intel_map_sg,
3971         .unmap_sg = intel_unmap_sg,
3972         .map_page = intel_map_page,
3973         .unmap_page = intel_unmap_page,
3974         .mapping_error = intel_mapping_error,
3975 };
3976
3977 static inline int iommu_domain_cache_init(void)
3978 {
3979         int ret = 0;
3980
3981         iommu_domain_cache = kmem_cache_create("iommu_domain",
3982                                          sizeof(struct dmar_domain),
3983                                          0,
3984                                          SLAB_HWCACHE_ALIGN,
3985
3986                                          NULL);
3987         if (!iommu_domain_cache) {
3988                 pr_err("Couldn't create iommu_domain cache\n");
3989                 ret = -ENOMEM;
3990         }
3991
3992         return ret;
3993 }
3994
3995 static inline int iommu_devinfo_cache_init(void)
3996 {
3997         int ret = 0;
3998
3999         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
4000                                          sizeof(struct device_domain_info),
4001                                          0,
4002                                          SLAB_HWCACHE_ALIGN,
4003                                          NULL);
4004         if (!iommu_devinfo_cache) {
4005                 pr_err("Couldn't create devinfo cache\n");
4006                 ret = -ENOMEM;
4007         }
4008
4009         return ret;
4010 }
4011
4012 static int __init iommu_init_mempool(void)
4013 {
4014         int ret;
4015         ret = iova_cache_get();
4016         if (ret)
4017                 return ret;
4018
4019         ret = iommu_domain_cache_init();
4020         if (ret)
4021                 goto domain_error;
4022
4023         ret = iommu_devinfo_cache_init();
4024         if (!ret)
4025                 return ret;
4026
4027         kmem_cache_destroy(iommu_domain_cache);
4028 domain_error:
4029         iova_cache_put();
4030
4031         return -ENOMEM;
4032 }
4033
4034 static void __init iommu_exit_mempool(void)
4035 {
4036         kmem_cache_destroy(iommu_devinfo_cache);
4037         kmem_cache_destroy(iommu_domain_cache);
4038         iova_cache_put();
4039 }
4040
4041 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4042 {
4043         struct dmar_drhd_unit *drhd;
4044         u32 vtbar;
4045         int rc;
4046
4047         /* We know that this device on this chipset has its own IOMMU.
4048          * If we find it under a different IOMMU, then the BIOS is lying
4049          * to us. Hope that the IOMMU for this device is actually
4050          * disabled, and it needs no translation...
4051          */
4052         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4053         if (rc) {
4054                 /* "can't" happen */
4055                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4056                 return;
4057         }
4058         vtbar &= 0xffff0000;
4059
4060         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4061         drhd = dmar_find_matched_drhd_unit(pdev);
4062         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4063                             TAINT_FIRMWARE_WORKAROUND,
4064                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4065                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4066 }
4067 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4068
4069 static void __init init_no_remapping_devices(void)
4070 {
4071         struct dmar_drhd_unit *drhd;
4072         struct device *dev;
4073         int i;
4074
4075         for_each_drhd_unit(drhd) {
4076                 if (!drhd->include_all) {
4077                         for_each_active_dev_scope(drhd->devices,
4078                                                   drhd->devices_cnt, i, dev)
4079                                 break;
4080                         /* ignore DMAR unit if no devices exist */
4081                         if (i == drhd->devices_cnt)
4082                                 drhd->ignored = 1;
4083                 }
4084         }
4085
4086         for_each_active_drhd_unit(drhd) {
4087                 if (drhd->include_all)
4088                         continue;
4089
4090                 for_each_active_dev_scope(drhd->devices,
4091                                           drhd->devices_cnt, i, dev)
4092                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4093                                 break;
4094                 if (i < drhd->devices_cnt)
4095                         continue;
4096
4097                 /* This IOMMU has *only* gfx devices. Either bypass it or
4098                    set the gfx_mapped flag, as appropriate */
4099                 if (dmar_map_gfx) {
4100                         intel_iommu_gfx_mapped = 1;
4101                 } else {
4102                         drhd->ignored = 1;
4103                         for_each_active_dev_scope(drhd->devices,
4104                                                   drhd->devices_cnt, i, dev)
4105                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4106                 }
4107         }
4108 }
4109
4110 #ifdef CONFIG_SUSPEND
4111 static int init_iommu_hw(void)
4112 {
4113         struct dmar_drhd_unit *drhd;
4114         struct intel_iommu *iommu = NULL;
4115
4116         for_each_active_iommu(iommu, drhd)
4117                 if (iommu->qi)
4118                         dmar_reenable_qi(iommu);
4119
4120         for_each_iommu(iommu, drhd) {
4121                 if (drhd->ignored) {
4122                         /*
4123                          * we always have to disable PMRs or DMA may fail on
4124                          * this device
4125                          */
4126                         if (force_on)
4127                                 iommu_disable_protect_mem_regions(iommu);
4128                         continue;
4129                 }
4130
4131                 iommu_flush_write_buffer(iommu);
4132
4133                 iommu_set_root_entry(iommu);
4134
4135                 iommu->flush.flush_context(iommu, 0, 0, 0,
4136                                            DMA_CCMD_GLOBAL_INVL);
4137                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4138                 iommu_enable_translation(iommu);
4139                 iommu_disable_protect_mem_regions(iommu);
4140         }
4141
4142         return 0;
4143 }
4144
4145 static void iommu_flush_all(void)
4146 {
4147         struct dmar_drhd_unit *drhd;
4148         struct intel_iommu *iommu;
4149
4150         for_each_active_iommu(iommu, drhd) {
4151                 iommu->flush.flush_context(iommu, 0, 0, 0,
4152                                            DMA_CCMD_GLOBAL_INVL);
4153                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4154                                          DMA_TLB_GLOBAL_FLUSH);
4155         }
4156 }
4157
4158 static int iommu_suspend(void)
4159 {
4160         struct dmar_drhd_unit *drhd;
4161         struct intel_iommu *iommu = NULL;
4162         unsigned long flag;
4163
4164         for_each_active_iommu(iommu, drhd) {
4165                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4166                                                  GFP_ATOMIC);
4167                 if (!iommu->iommu_state)
4168                         goto nomem;
4169         }
4170
4171         iommu_flush_all();
4172
4173         for_each_active_iommu(iommu, drhd) {
4174                 iommu_disable_translation(iommu);
4175
4176                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4177
4178                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4179                         readl(iommu->reg + DMAR_FECTL_REG);
4180                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4181                         readl(iommu->reg + DMAR_FEDATA_REG);
4182                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4183                         readl(iommu->reg + DMAR_FEADDR_REG);
4184                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4185                         readl(iommu->reg + DMAR_FEUADDR_REG);
4186
4187                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4188         }
4189         return 0;
4190
4191 nomem:
4192         for_each_active_iommu(iommu, drhd)
4193                 kfree(iommu->iommu_state);
4194
4195         return -ENOMEM;
4196 }
4197
4198 static void iommu_resume(void)
4199 {
4200         struct dmar_drhd_unit *drhd;
4201         struct intel_iommu *iommu = NULL;
4202         unsigned long flag;
4203
4204         if (init_iommu_hw()) {
4205                 if (force_on)
4206                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4207                 else
4208                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4209                 return;
4210         }
4211
4212         for_each_active_iommu(iommu, drhd) {
4213
4214                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4215
4216                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4217                         iommu->reg + DMAR_FECTL_REG);
4218                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4219                         iommu->reg + DMAR_FEDATA_REG);
4220                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4221                         iommu->reg + DMAR_FEADDR_REG);
4222                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4223                         iommu->reg + DMAR_FEUADDR_REG);
4224
4225                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4226         }
4227
4228         for_each_active_iommu(iommu, drhd)
4229                 kfree(iommu->iommu_state);
4230 }
4231
4232 static struct syscore_ops iommu_syscore_ops = {
4233         .resume         = iommu_resume,
4234         .suspend        = iommu_suspend,
4235 };
4236
4237 static void __init init_iommu_pm_ops(void)
4238 {
4239         register_syscore_ops(&iommu_syscore_ops);
4240 }
4241
4242 #else
4243 static inline void init_iommu_pm_ops(void) {}
4244 #endif  /* CONFIG_PM */
4245
4246
4247 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4248 {
4249         struct acpi_dmar_reserved_memory *rmrr;
4250         struct dmar_rmrr_unit *rmrru;
4251
4252         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4253         if (!rmrru)
4254                 return -ENOMEM;
4255
4256         rmrru->hdr = header;
4257         rmrr = (struct acpi_dmar_reserved_memory *)header;
4258         rmrru->base_address = rmrr->base_address;
4259         rmrru->end_address = rmrr->end_address;
4260         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4261                                 ((void *)rmrr) + rmrr->header.length,
4262                                 &rmrru->devices_cnt);
4263         if (rmrru->devices_cnt && rmrru->devices == NULL) {
4264                 kfree(rmrru);
4265                 return -ENOMEM;
4266         }
4267
4268         list_add(&rmrru->list, &dmar_rmrr_units);
4269
4270         return 0;
4271 }
4272
4273 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4274 {
4275         struct dmar_atsr_unit *atsru;
4276         struct acpi_dmar_atsr *tmp;
4277
4278         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4279                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4280                 if (atsr->segment != tmp->segment)
4281                         continue;
4282                 if (atsr->header.length != tmp->header.length)
4283                         continue;
4284                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4285                         return atsru;
4286         }
4287
4288         return NULL;
4289 }
4290
4291 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4292 {
4293         struct acpi_dmar_atsr *atsr;
4294         struct dmar_atsr_unit *atsru;
4295
4296         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4297                 return 0;
4298
4299         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4300         atsru = dmar_find_atsr(atsr);
4301         if (atsru)
4302                 return 0;
4303
4304         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4305         if (!atsru)
4306                 return -ENOMEM;
4307
4308         /*
4309          * If memory is allocated from slab by ACPI _DSM method, we need to
4310          * copy the memory content because the memory buffer will be freed
4311          * on return.
4312          */
4313         atsru->hdr = (void *)(atsru + 1);
4314         memcpy(atsru->hdr, hdr, hdr->length);
4315         atsru->include_all = atsr->flags & 0x1;
4316         if (!atsru->include_all) {
4317                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4318                                 (void *)atsr + atsr->header.length,
4319                                 &atsru->devices_cnt);
4320                 if (atsru->devices_cnt && atsru->devices == NULL) {
4321                         kfree(atsru);
4322                         return -ENOMEM;
4323                 }
4324         }
4325
4326         list_add_rcu(&atsru->list, &dmar_atsr_units);
4327
4328         return 0;
4329 }
4330
4331 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4332 {
4333         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4334         kfree(atsru);
4335 }
4336
4337 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4338 {
4339         struct acpi_dmar_atsr *atsr;
4340         struct dmar_atsr_unit *atsru;
4341
4342         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4343         atsru = dmar_find_atsr(atsr);
4344         if (atsru) {
4345                 list_del_rcu(&atsru->list);
4346                 synchronize_rcu();
4347                 intel_iommu_free_atsr(atsru);
4348         }
4349
4350         return 0;
4351 }
4352
4353 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4354 {
4355         int i;
4356         struct device *dev;
4357         struct acpi_dmar_atsr *atsr;
4358         struct dmar_atsr_unit *atsru;
4359
4360         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4361         atsru = dmar_find_atsr(atsr);
4362         if (!atsru)
4363                 return 0;
4364
4365         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4366                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4367                                           i, dev)
4368                         return -EBUSY;
4369         }
4370
4371         return 0;
4372 }
4373
4374 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4375 {
4376         int sp, ret = 0;
4377         struct intel_iommu *iommu = dmaru->iommu;
4378
4379         if (g_iommus[iommu->seq_id])
4380                 return 0;
4381
4382         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4383                 pr_warn("%s: Doesn't support hardware pass through.\n",
4384                         iommu->name);
4385                 return -ENXIO;
4386         }
4387         if (!ecap_sc_support(iommu->ecap) &&
4388             domain_update_iommu_snooping(iommu)) {
4389                 pr_warn("%s: Doesn't support snooping.\n",
4390                         iommu->name);
4391                 return -ENXIO;
4392         }
4393         sp = domain_update_iommu_superpage(iommu) - 1;
4394         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4395                 pr_warn("%s: Doesn't support large page.\n",
4396                         iommu->name);
4397                 return -ENXIO;
4398         }
4399
4400         /*
4401          * Disable translation if already enabled prior to OS handover.
4402          */
4403         if (iommu->gcmd & DMA_GCMD_TE)
4404                 iommu_disable_translation(iommu);
4405
4406         g_iommus[iommu->seq_id] = iommu;
4407         ret = iommu_init_domains(iommu);
4408         if (ret == 0)
4409                 ret = iommu_alloc_root_entry(iommu);
4410         if (ret)
4411                 goto out;
4412
4413 #ifdef CONFIG_INTEL_IOMMU_SVM
4414         if (pasid_enabled(iommu))
4415                 intel_svm_alloc_pasid_tables(iommu);
4416 #endif
4417
4418         if (dmaru->ignored) {
4419                 /*
4420                  * we always have to disable PMRs or DMA may fail on this device
4421                  */
4422                 if (force_on)
4423                         iommu_disable_protect_mem_regions(iommu);
4424                 return 0;
4425         }
4426
4427         intel_iommu_init_qi(iommu);
4428         iommu_flush_write_buffer(iommu);
4429
4430 #ifdef CONFIG_INTEL_IOMMU_SVM
4431         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4432                 ret = intel_svm_enable_prq(iommu);
4433                 if (ret)
4434                         goto disable_iommu;
4435         }
4436 #endif
4437         ret = dmar_set_interrupt(iommu);
4438         if (ret)
4439                 goto disable_iommu;
4440
4441         iommu_set_root_entry(iommu);
4442         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4443         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4444         iommu_enable_translation(iommu);
4445
4446         iommu_disable_protect_mem_regions(iommu);
4447         return 0;
4448
4449 disable_iommu:
4450         disable_dmar_iommu(iommu);
4451 out:
4452         free_dmar_iommu(iommu);
4453         return ret;
4454 }
4455
4456 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4457 {
4458         int ret = 0;
4459         struct intel_iommu *iommu = dmaru->iommu;
4460
4461         if (!intel_iommu_enabled)
4462                 return 0;
4463         if (iommu == NULL)
4464                 return -EINVAL;
4465
4466         if (insert) {
4467                 ret = intel_iommu_add(dmaru);
4468         } else {
4469                 disable_dmar_iommu(iommu);
4470                 free_dmar_iommu(iommu);
4471         }
4472
4473         return ret;
4474 }
4475
4476 static void intel_iommu_free_dmars(void)
4477 {
4478         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4479         struct dmar_atsr_unit *atsru, *atsr_n;
4480
4481         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4482                 list_del(&rmrru->list);
4483                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4484                 kfree(rmrru);
4485         }
4486
4487         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4488                 list_del(&atsru->list);
4489                 intel_iommu_free_atsr(atsru);
4490         }
4491 }
4492
4493 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4494 {
4495         int i, ret = 1;
4496         struct pci_bus *bus;
4497         struct pci_dev *bridge = NULL;
4498         struct device *tmp;
4499         struct acpi_dmar_atsr *atsr;
4500         struct dmar_atsr_unit *atsru;
4501
4502         dev = pci_physfn(dev);
4503         for (bus = dev->bus; bus; bus = bus->parent) {
4504                 bridge = bus->self;
4505                 /* If it's an integrated device, allow ATS */
4506                 if (!bridge)
4507                         return 1;
4508                 /* Connected via non-PCIe: no ATS */
4509                 if (!pci_is_pcie(bridge) ||
4510                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4511                         return 0;
4512                 /* If we found the root port, look it up in the ATSR */
4513                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4514                         break;
4515         }
4516
4517         rcu_read_lock();
4518         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4519                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4520                 if (atsr->segment != pci_domain_nr(dev->bus))
4521                         continue;
4522
4523                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4524                         if (tmp == &bridge->dev)
4525                                 goto out;
4526
4527                 if (atsru->include_all)
4528                         goto out;
4529         }
4530         ret = 0;
4531 out:
4532         rcu_read_unlock();
4533
4534         return ret;
4535 }
4536
4537 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4538 {
4539         int ret = 0;
4540         struct dmar_rmrr_unit *rmrru;
4541         struct dmar_atsr_unit *atsru;
4542         struct acpi_dmar_atsr *atsr;
4543         struct acpi_dmar_reserved_memory *rmrr;
4544
4545         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4546                 return 0;
4547
4548         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4549                 rmrr = container_of(rmrru->hdr,
4550                                     struct acpi_dmar_reserved_memory, header);
4551                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4552                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4553                                 ((void *)rmrr) + rmrr->header.length,
4554                                 rmrr->segment, rmrru->devices,
4555                                 rmrru->devices_cnt);
4556                         if(ret < 0)
4557                                 return ret;
4558                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4559                         dmar_remove_dev_scope(info, rmrr->segment,
4560                                 rmrru->devices, rmrru->devices_cnt);
4561                 }
4562         }
4563
4564         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4565                 if (atsru->include_all)
4566                         continue;
4567
4568                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4569                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4570                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4571                                         (void *)atsr + atsr->header.length,
4572                                         atsr->segment, atsru->devices,
4573                                         atsru->devices_cnt);
4574                         if (ret > 0)
4575                                 break;
4576                         else if(ret < 0)
4577                                 return ret;
4578                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4579                         if (dmar_remove_dev_scope(info, atsr->segment,
4580                                         atsru->devices, atsru->devices_cnt))
4581                                 break;
4582                 }
4583         }
4584
4585         return 0;
4586 }
4587
4588 /*
4589  * Here we only respond to action of unbound device from driver.
4590  *
4591  * Added device is not attached to its DMAR domain here yet. That will happen
4592  * when mapping the device to iova.
4593  */
4594 static int device_notifier(struct notifier_block *nb,
4595                                   unsigned long action, void *data)
4596 {
4597         struct device *dev = data;
4598         struct dmar_domain *domain;
4599
4600         if (iommu_dummy(dev))
4601                 return 0;
4602
4603         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4604                 return 0;
4605
4606         domain = find_domain(dev);
4607         if (!domain)
4608                 return 0;
4609
4610         dmar_remove_one_dev_info(domain, dev);
4611         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4612                 domain_exit(domain);
4613
4614         return 0;
4615 }
4616
4617 static struct notifier_block device_nb = {
4618         .notifier_call = device_notifier,
4619 };
4620
4621 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4622                                        unsigned long val, void *v)
4623 {
4624         struct memory_notify *mhp = v;
4625         unsigned long long start, end;
4626         unsigned long start_vpfn, last_vpfn;
4627
4628         switch (val) {
4629         case MEM_GOING_ONLINE:
4630                 start = mhp->start_pfn << PAGE_SHIFT;
4631                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4632                 if (iommu_domain_identity_map(si_domain, start, end)) {
4633                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4634                                 start, end);
4635                         return NOTIFY_BAD;
4636                 }
4637                 break;
4638
4639         case MEM_OFFLINE:
4640         case MEM_CANCEL_ONLINE:
4641                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4642                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4643                 while (start_vpfn <= last_vpfn) {
4644                         struct iova *iova;
4645                         struct dmar_drhd_unit *drhd;
4646                         struct intel_iommu *iommu;
4647                         struct page *freelist;
4648
4649                         iova = find_iova(&si_domain->iovad, start_vpfn);
4650                         if (iova == NULL) {
4651                                 pr_debug("Failed get IOVA for PFN %lx\n",
4652                                          start_vpfn);
4653                                 break;
4654                         }
4655
4656                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4657                                                      start_vpfn, last_vpfn);
4658                         if (iova == NULL) {
4659                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4660                                         start_vpfn, last_vpfn);
4661                                 return NOTIFY_BAD;
4662                         }
4663
4664                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4665                                                iova->pfn_hi);
4666
4667                         rcu_read_lock();
4668                         for_each_active_iommu(iommu, drhd)
4669                                 iommu_flush_iotlb_psi(iommu, si_domain,
4670                                         iova->pfn_lo, iova_size(iova),
4671                                         !freelist, 0);
4672                         rcu_read_unlock();
4673                         dma_free_pagelist(freelist);
4674
4675                         start_vpfn = iova->pfn_hi + 1;
4676                         free_iova_mem(iova);
4677                 }
4678                 break;
4679         }
4680
4681         return NOTIFY_OK;
4682 }
4683
4684 static struct notifier_block intel_iommu_memory_nb = {
4685         .notifier_call = intel_iommu_memory_notifier,
4686         .priority = 0
4687 };
4688
4689 static void free_all_cpu_cached_iovas(unsigned int cpu)
4690 {
4691         int i;
4692
4693         for (i = 0; i < g_num_of_iommus; i++) {
4694                 struct intel_iommu *iommu = g_iommus[i];
4695                 struct dmar_domain *domain;
4696                 int did;
4697
4698                 if (!iommu)
4699                         continue;
4700
4701                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4702                         domain = get_iommu_domain(iommu, (u16)did);
4703
4704                         if (!domain)
4705                                 continue;
4706                         free_cpu_cached_iovas(cpu, &domain->iovad);
4707                 }
4708         }
4709 }
4710
4711 static int intel_iommu_cpu_dead(unsigned int cpu)
4712 {
4713         free_all_cpu_cached_iovas(cpu);
4714         flush_unmaps_timeout(cpu);
4715         return 0;
4716 }
4717
4718 static ssize_t intel_iommu_show_version(struct device *dev,
4719                                         struct device_attribute *attr,
4720                                         char *buf)
4721 {
4722         struct intel_iommu *iommu = dev_get_drvdata(dev);
4723         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4724         return sprintf(buf, "%d:%d\n",
4725                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4726 }
4727 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4728
4729 static ssize_t intel_iommu_show_address(struct device *dev,
4730                                         struct device_attribute *attr,
4731                                         char *buf)
4732 {
4733         struct intel_iommu *iommu = dev_get_drvdata(dev);
4734         return sprintf(buf, "%llx\n", iommu->reg_phys);
4735 }
4736 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4737
4738 static ssize_t intel_iommu_show_cap(struct device *dev,
4739                                     struct device_attribute *attr,
4740                                     char *buf)
4741 {
4742         struct intel_iommu *iommu = dev_get_drvdata(dev);
4743         return sprintf(buf, "%llx\n", iommu->cap);
4744 }
4745 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4746
4747 static ssize_t intel_iommu_show_ecap(struct device *dev,
4748                                     struct device_attribute *attr,
4749                                     char *buf)
4750 {
4751         struct intel_iommu *iommu = dev_get_drvdata(dev);
4752         return sprintf(buf, "%llx\n", iommu->ecap);
4753 }
4754 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4755
4756 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4757                                       struct device_attribute *attr,
4758                                       char *buf)
4759 {
4760         struct intel_iommu *iommu = dev_get_drvdata(dev);
4761         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4762 }
4763 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4764
4765 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4766                                            struct device_attribute *attr,
4767                                            char *buf)
4768 {
4769         struct intel_iommu *iommu = dev_get_drvdata(dev);
4770         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4771                                                   cap_ndoms(iommu->cap)));
4772 }
4773 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4774
4775 static struct attribute *intel_iommu_attrs[] = {
4776         &dev_attr_version.attr,
4777         &dev_attr_address.attr,
4778         &dev_attr_cap.attr,
4779         &dev_attr_ecap.attr,
4780         &dev_attr_domains_supported.attr,
4781         &dev_attr_domains_used.attr,
4782         NULL,
4783 };
4784
4785 static struct attribute_group intel_iommu_group = {
4786         .name = "intel-iommu",
4787         .attrs = intel_iommu_attrs,
4788 };
4789
4790 const struct attribute_group *intel_iommu_groups[] = {
4791         &intel_iommu_group,
4792         NULL,
4793 };
4794
4795 int __init intel_iommu_init(void)
4796 {
4797         int ret = -ENODEV;
4798         struct dmar_drhd_unit *drhd;
4799         struct intel_iommu *iommu;
4800
4801         /* VT-d is required for a TXT/tboot launch, so enforce that */
4802         force_on = tboot_force_iommu();
4803
4804         if (iommu_init_mempool()) {
4805                 if (force_on)
4806                         panic("tboot: Failed to initialize iommu memory\n");
4807                 return -ENOMEM;
4808         }
4809
4810         down_write(&dmar_global_lock);
4811         if (dmar_table_init()) {
4812                 if (force_on)
4813                         panic("tboot: Failed to initialize DMAR table\n");
4814                 goto out_free_dmar;
4815         }
4816
4817         if (dmar_dev_scope_init() < 0) {
4818                 if (force_on)
4819                         panic("tboot: Failed to initialize DMAR device scope\n");
4820                 goto out_free_dmar;
4821         }
4822
4823         if (no_iommu || dmar_disabled)
4824                 goto out_free_dmar;
4825
4826         if (list_empty(&dmar_rmrr_units))
4827                 pr_info("No RMRR found\n");
4828
4829         if (list_empty(&dmar_atsr_units))
4830                 pr_info("No ATSR found\n");
4831
4832         if (dmar_init_reserved_ranges()) {
4833                 if (force_on)
4834                         panic("tboot: Failed to reserve iommu ranges\n");
4835                 goto out_free_reserved_range;
4836         }
4837
4838         init_no_remapping_devices();
4839
4840         ret = init_dmars();
4841         if (ret) {
4842                 if (force_on)
4843                         panic("tboot: Failed to initialize DMARs\n");
4844                 pr_err("Initialization failed\n");
4845                 goto out_free_reserved_range;
4846         }
4847         up_write(&dmar_global_lock);
4848         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4849
4850 #ifdef CONFIG_SWIOTLB
4851         swiotlb = 0;
4852 #endif
4853         dma_ops = &intel_dma_ops;
4854
4855         init_iommu_pm_ops();
4856
4857         for_each_active_iommu(iommu, drhd)
4858                 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4859                                                        intel_iommu_groups,
4860                                                        "%s", iommu->name);
4861
4862         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4863         bus_register_notifier(&pci_bus_type, &device_nb);
4864         if (si_domain && !hw_pass_through)
4865                 register_memory_notifier(&intel_iommu_memory_nb);
4866         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4867                           intel_iommu_cpu_dead);
4868         intel_iommu_enabled = 1;
4869
4870         return 0;
4871
4872 out_free_reserved_range:
4873         put_iova_domain(&reserved_iova_list);
4874 out_free_dmar:
4875         intel_iommu_free_dmars();
4876         up_write(&dmar_global_lock);
4877         iommu_exit_mempool();
4878         return ret;
4879 }
4880
4881 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4882 {
4883         struct intel_iommu *iommu = opaque;
4884
4885         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4886         return 0;
4887 }
4888
4889 /*
4890  * NB - intel-iommu lacks any sort of reference counting for the users of
4891  * dependent devices.  If multiple endpoints have intersecting dependent
4892  * devices, unbinding the driver from any one of them will possibly leave
4893  * the others unable to operate.
4894  */
4895 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4896 {
4897         if (!iommu || !dev || !dev_is_pci(dev))
4898                 return;
4899
4900         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4901 }
4902
4903 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4904 {
4905         struct intel_iommu *iommu;
4906         unsigned long flags;
4907
4908         assert_spin_locked(&device_domain_lock);
4909
4910         if (WARN_ON(!info))
4911                 return;
4912
4913         iommu = info->iommu;
4914
4915         if (info->dev) {
4916                 iommu_disable_dev_iotlb(info);
4917                 domain_context_clear(iommu, info->dev);
4918         }
4919
4920         unlink_domain_info(info);
4921
4922         spin_lock_irqsave(&iommu->lock, flags);
4923         domain_detach_iommu(info->domain, iommu);
4924         spin_unlock_irqrestore(&iommu->lock, flags);
4925
4926         free_devinfo_mem(info);
4927 }
4928
4929 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4930                                      struct device *dev)
4931 {
4932         struct device_domain_info *info;
4933         unsigned long flags;
4934
4935         spin_lock_irqsave(&device_domain_lock, flags);
4936         info = dev->archdata.iommu;
4937         __dmar_remove_one_dev_info(info);
4938         spin_unlock_irqrestore(&device_domain_lock, flags);
4939 }
4940
4941 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4942 {
4943         int adjust_width;
4944
4945         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4946                         DMA_32BIT_PFN);
4947         domain_reserve_special_ranges(domain);
4948
4949         /* calculate AGAW */
4950         domain->gaw = guest_width;
4951         adjust_width = guestwidth_to_adjustwidth(guest_width);
4952         domain->agaw = width_to_agaw(adjust_width);
4953
4954         domain->iommu_coherency = 0;
4955         domain->iommu_snooping = 0;
4956         domain->iommu_superpage = 0;
4957         domain->max_addr = 0;
4958
4959         /* always allocate the top pgd */
4960         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4961         if (!domain->pgd)
4962                 return -ENOMEM;
4963         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4964         return 0;
4965 }
4966
4967 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4968 {
4969         struct dmar_domain *dmar_domain;
4970         struct iommu_domain *domain;
4971
4972         if (type != IOMMU_DOMAIN_UNMANAGED)
4973                 return NULL;
4974
4975         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4976         if (!dmar_domain) {
4977                 pr_err("Can't allocate dmar_domain\n");
4978                 return NULL;
4979         }
4980         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4981                 pr_err("Domain initialization failed\n");
4982                 domain_exit(dmar_domain);
4983                 return NULL;
4984         }
4985         domain_update_iommu_cap(dmar_domain);
4986
4987         domain = &dmar_domain->domain;
4988         domain->geometry.aperture_start = 0;
4989         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4990         domain->geometry.force_aperture = true;
4991
4992         return domain;
4993 }
4994
4995 static void intel_iommu_domain_free(struct iommu_domain *domain)
4996 {
4997         domain_exit(to_dmar_domain(domain));
4998 }
4999
5000 static int intel_iommu_attach_device(struct iommu_domain *domain,
5001                                      struct device *dev)
5002 {
5003         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5004         struct intel_iommu *iommu;
5005         int addr_width;
5006         u8 bus, devfn;
5007
5008         if (device_is_rmrr_locked(dev)) {
5009                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5010                 return -EPERM;
5011         }
5012
5013         /* normally dev is not mapped */
5014         if (unlikely(domain_context_mapped(dev))) {
5015                 struct dmar_domain *old_domain;
5016
5017                 old_domain = find_domain(dev);
5018                 if (old_domain) {
5019                         rcu_read_lock();
5020                         dmar_remove_one_dev_info(old_domain, dev);
5021                         rcu_read_unlock();
5022
5023                         if (!domain_type_is_vm_or_si(old_domain) &&
5024                              list_empty(&old_domain->devices))
5025                                 domain_exit(old_domain);
5026                 }
5027         }
5028
5029         iommu = device_to_iommu(dev, &bus, &devfn);
5030         if (!iommu)
5031                 return -ENODEV;
5032
5033         /* check if this iommu agaw is sufficient for max mapped address */
5034         addr_width = agaw_to_width(iommu->agaw);
5035         if (addr_width > cap_mgaw(iommu->cap))
5036                 addr_width = cap_mgaw(iommu->cap);
5037
5038         if (dmar_domain->max_addr > (1LL << addr_width)) {
5039                 pr_err("%s: iommu width (%d) is not "
5040                        "sufficient for the mapped address (%llx)\n",
5041                        __func__, addr_width, dmar_domain->max_addr);
5042                 return -EFAULT;
5043         }
5044         dmar_domain->gaw = addr_width;
5045
5046         /*
5047          * Knock out extra levels of page tables if necessary
5048          */
5049         while (iommu->agaw < dmar_domain->agaw) {
5050                 struct dma_pte *pte;
5051
5052                 pte = dmar_domain->pgd;
5053                 if (dma_pte_present(pte)) {
5054                         dmar_domain->pgd = (struct dma_pte *)
5055                                 phys_to_virt(dma_pte_addr(pte));
5056                         free_pgtable_page(pte);
5057                 }
5058                 dmar_domain->agaw--;
5059         }
5060
5061         return domain_add_dev_info(dmar_domain, dev);
5062 }
5063
5064 static void intel_iommu_detach_device(struct iommu_domain *domain,
5065                                       struct device *dev)
5066 {
5067         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5068 }
5069
5070 static int intel_iommu_map(struct iommu_domain *domain,
5071                            unsigned long iova, phys_addr_t hpa,
5072                            size_t size, int iommu_prot)
5073 {
5074         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5075         u64 max_addr;
5076         int prot = 0;
5077         int ret;
5078
5079         if (iommu_prot & IOMMU_READ)
5080                 prot |= DMA_PTE_READ;
5081         if (iommu_prot & IOMMU_WRITE)
5082                 prot |= DMA_PTE_WRITE;
5083         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5084                 prot |= DMA_PTE_SNP;
5085
5086         max_addr = iova + size;
5087         if (dmar_domain->max_addr < max_addr) {
5088                 u64 end;
5089
5090                 /* check if minimum agaw is sufficient for mapped address */
5091                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5092                 if (end < max_addr) {
5093                         pr_err("%s: iommu width (%d) is not "
5094                                "sufficient for the mapped address (%llx)\n",
5095                                __func__, dmar_domain->gaw, max_addr);
5096                         return -EFAULT;
5097                 }
5098                 dmar_domain->max_addr = max_addr;
5099         }
5100         /* Round up size to next multiple of PAGE_SIZE, if it and
5101            the low bits of hpa would take us onto the next page */
5102         size = aligned_nrpages(hpa, size);
5103         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5104                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5105         return ret;
5106 }
5107
5108 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5109                                 unsigned long iova, size_t size)
5110 {
5111         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5112         struct page *freelist = NULL;
5113         struct intel_iommu *iommu;
5114         unsigned long start_pfn, last_pfn;
5115         unsigned int npages;
5116         int iommu_id, level = 0;
5117
5118         /* Cope with horrid API which requires us to unmap more than the
5119            size argument if it happens to be a large-page mapping. */
5120         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5121
5122         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5123                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5124
5125         start_pfn = iova >> VTD_PAGE_SHIFT;
5126         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5127
5128         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5129
5130         npages = last_pfn - start_pfn + 1;
5131
5132         for_each_domain_iommu(iommu_id, dmar_domain) {
5133                 iommu = g_iommus[iommu_id];
5134
5135                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5136                                       start_pfn, npages, !freelist, 0);
5137         }
5138
5139         dma_free_pagelist(freelist);
5140
5141         if (dmar_domain->max_addr == iova + size)
5142                 dmar_domain->max_addr = iova;
5143
5144         return size;
5145 }
5146
5147 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5148                                             dma_addr_t iova)
5149 {
5150         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5151         struct dma_pte *pte;
5152         int level = 0;
5153         u64 phys = 0;
5154
5155         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5156         if (pte)
5157                 phys = dma_pte_addr(pte);
5158
5159         return phys;
5160 }
5161
5162 static bool intel_iommu_capable(enum iommu_cap cap)
5163 {
5164         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5165                 return domain_update_iommu_snooping(NULL) == 1;
5166         if (cap == IOMMU_CAP_INTR_REMAP)
5167                 return irq_remapping_enabled == 1;
5168
5169         return false;
5170 }
5171
5172 static int intel_iommu_add_device(struct device *dev)
5173 {
5174         struct intel_iommu *iommu;
5175         struct iommu_group *group;
5176         u8 bus, devfn;
5177
5178         iommu = device_to_iommu(dev, &bus, &devfn);
5179         if (!iommu)
5180                 return -ENODEV;
5181
5182         iommu_device_link(iommu->iommu_dev, dev);
5183
5184         group = iommu_group_get_for_dev(dev);
5185
5186         if (IS_ERR(group))
5187                 return PTR_ERR(group);
5188
5189         iommu_group_put(group);
5190         return 0;
5191 }
5192
5193 static void intel_iommu_remove_device(struct device *dev)
5194 {
5195         struct intel_iommu *iommu;
5196         u8 bus, devfn;
5197
5198         iommu = device_to_iommu(dev, &bus, &devfn);
5199         if (!iommu)
5200                 return;
5201
5202         iommu_group_remove_device(dev);
5203
5204         iommu_device_unlink(iommu->iommu_dev, dev);
5205 }
5206
5207 #ifdef CONFIG_INTEL_IOMMU_SVM
5208 #define MAX_NR_PASID_BITS (20)
5209 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5210 {
5211         /*
5212          * Convert ecap_pss to extend context entry pts encoding, also
5213          * respect the soft pasid_max value set by the iommu.
5214          * - number of PASID bits = ecap_pss + 1
5215          * - number of PASID table entries = 2^(pts + 5)
5216          * Therefore, pts = ecap_pss - 4
5217          * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5218          */
5219         if (ecap_pss(iommu->ecap) < 5)
5220                 return 0;
5221
5222         /* pasid_max is encoded as actual number of entries not the bits */
5223         return find_first_bit((unsigned long *)&iommu->pasid_max,
5224                         MAX_NR_PASID_BITS) - 5;
5225 }
5226
5227 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5228 {
5229         struct device_domain_info *info;
5230         struct context_entry *context;
5231         struct dmar_domain *domain;
5232         unsigned long flags;
5233         u64 ctx_lo;
5234         int ret;
5235
5236         domain = get_valid_domain_for_dev(sdev->dev);
5237         if (!domain)
5238                 return -EINVAL;
5239
5240         spin_lock_irqsave(&device_domain_lock, flags);
5241         spin_lock(&iommu->lock);
5242
5243         ret = -EINVAL;
5244         info = sdev->dev->archdata.iommu;
5245         if (!info || !info->pasid_supported)
5246                 goto out;
5247
5248         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5249         if (WARN_ON(!context))
5250                 goto out;
5251
5252         ctx_lo = context[0].lo;
5253
5254         sdev->did = domain->iommu_did[iommu->seq_id];
5255         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5256
5257         if (!(ctx_lo & CONTEXT_PASIDE)) {
5258                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5259                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5260                         intel_iommu_get_pts(iommu);
5261
5262                 wmb();
5263                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5264                  * extended to permit requests-with-PASID if the PASIDE bit
5265                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5266                  * however, the PASIDE bit is ignored and requests-with-PASID
5267                  * are unconditionally blocked. Which makes less sense.
5268                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5269                  * "guest mode" translation types depending on whether ATS
5270                  * is available or not. Annoyingly, we can't use the new
5271                  * modes *unless* PASIDE is set. */
5272                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5273                         ctx_lo &= ~CONTEXT_TT_MASK;
5274                         if (info->ats_supported)
5275                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5276                         else
5277                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5278                 }
5279                 ctx_lo |= CONTEXT_PASIDE;
5280                 if (iommu->pasid_state_table)
5281                         ctx_lo |= CONTEXT_DINVE;
5282                 if (info->pri_supported)
5283                         ctx_lo |= CONTEXT_PRS;
5284                 context[0].lo = ctx_lo;
5285                 wmb();
5286                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5287                                            DMA_CCMD_MASK_NOBIT,
5288                                            DMA_CCMD_DEVICE_INVL);
5289         }
5290
5291         /* Enable PASID support in the device, if it wasn't already */
5292         if (!info->pasid_enabled)
5293                 iommu_enable_dev_iotlb(info);
5294
5295         if (info->ats_enabled) {
5296                 sdev->dev_iotlb = 1;
5297                 sdev->qdep = info->ats_qdep;
5298                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5299                         sdev->qdep = 0;
5300         }
5301         ret = 0;
5302
5303  out:
5304         spin_unlock(&iommu->lock);
5305         spin_unlock_irqrestore(&device_domain_lock, flags);
5306
5307         return ret;
5308 }
5309
5310 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5311 {
5312         struct intel_iommu *iommu;
5313         u8 bus, devfn;
5314
5315         if (iommu_dummy(dev)) {
5316                 dev_warn(dev,
5317                          "No IOMMU translation for device; cannot enable SVM\n");
5318                 return NULL;
5319         }
5320
5321         iommu = device_to_iommu(dev, &bus, &devfn);
5322         if ((!iommu)) {
5323                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5324                 return NULL;
5325         }
5326
5327         if (!iommu->pasid_table) {
5328                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5329                 return NULL;
5330         }
5331
5332         return iommu;
5333 }
5334 #endif /* CONFIG_INTEL_IOMMU_SVM */
5335
5336 static const struct iommu_ops intel_iommu_ops = {
5337         .capable        = intel_iommu_capable,
5338         .domain_alloc   = intel_iommu_domain_alloc,
5339         .domain_free    = intel_iommu_domain_free,
5340         .attach_dev     = intel_iommu_attach_device,
5341         .detach_dev     = intel_iommu_detach_device,
5342         .map            = intel_iommu_map,
5343         .unmap          = intel_iommu_unmap,
5344         .map_sg         = default_iommu_map_sg,
5345         .iova_to_phys   = intel_iommu_iova_to_phys,
5346         .add_device     = intel_iommu_add_device,
5347         .remove_device  = intel_iommu_remove_device,
5348         .device_group   = pci_device_group,
5349         .pgsize_bitmap  = INTEL_IOMMU_PGSIZES,
5350 };
5351
5352 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5353 {
5354         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5355         pr_info("Disabling IOMMU for graphics on this chipset\n");
5356         dmar_map_gfx = 0;
5357 }
5358
5359 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5360 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5361 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5362 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5363 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5364 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5365 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5366
5367 static void quirk_iommu_rwbf(struct pci_dev *dev)
5368 {
5369         /*
5370          * Mobile 4 Series Chipset neglects to set RWBF capability,
5371          * but needs it. Same seems to hold for the desktop versions.
5372          */
5373         pr_info("Forcing write-buffer flush capability\n");
5374         rwbf_quirk = 1;
5375 }
5376
5377 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5378 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5379 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5380 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5381 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5383 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5384
5385 #define GGC 0x52
5386 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5387 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5388 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5389 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5390 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5391 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5392 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5393 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5394
5395 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5396 {
5397         unsigned short ggc;
5398
5399         if (pci_read_config_word(dev, GGC, &ggc))
5400                 return;
5401
5402         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5403                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5404                 dmar_map_gfx = 0;
5405         } else if (dmar_map_gfx) {
5406                 /* we have to ensure the gfx device is idle before we flush */
5407                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5408                 intel_iommu_strict = 1;
5409        }
5410 }
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5414 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5415
5416 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5417    ISOCH DMAR unit for the Azalia sound device, but not give it any
5418    TLB entries, which causes it to deadlock. Check for that.  We do
5419    this in a function called from init_dmars(), instead of in a PCI
5420    quirk, because we don't want to print the obnoxious "BIOS broken"
5421    message if VT-d is actually disabled.
5422 */
5423 static void __init check_tylersburg_isoch(void)
5424 {
5425         struct pci_dev *pdev;
5426         uint32_t vtisochctrl;
5427
5428         /* If there's no Azalia in the system anyway, forget it. */
5429         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5430         if (!pdev)
5431                 return;
5432         pci_dev_put(pdev);
5433
5434         /* System Management Registers. Might be hidden, in which case
5435            we can't do the sanity check. But that's OK, because the
5436            known-broken BIOSes _don't_ actually hide it, so far. */
5437         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5438         if (!pdev)
5439                 return;
5440
5441         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5442                 pci_dev_put(pdev);
5443                 return;
5444         }
5445
5446         pci_dev_put(pdev);
5447
5448         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5449         if (vtisochctrl & 1)
5450                 return;
5451
5452         /* Drop all bits other than the number of TLB entries */
5453         vtisochctrl &= 0x1c;
5454
5455         /* If we have the recommended number of TLB entries (16), fine. */
5456         if (vtisochctrl == 0x10)
5457                 return;
5458
5459         /* Zero TLB entries? You get to ride the short bus to school. */
5460         if (!vtisochctrl) {
5461                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5462                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5463                      dmi_get_system_info(DMI_BIOS_VENDOR),
5464                      dmi_get_system_info(DMI_BIOS_VERSION),
5465                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5466                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5467                 return;
5468         }
5469
5470         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5471                vtisochctrl);
5472 }