drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/crash_dump.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/cacheflush.h>
  51 #include <asm/iommu.h>
  52
  53 #include "irq_remapping.h"
  54
  55 #define ROOT_SIZE               VTD_PAGE_SIZE
  56 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  57
  58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  62
  63 #define IOAPIC_RANGE_START      (0xfee00000)
  64 #define IOAPIC_RANGE_END        (0xfeefffff)
  65 #define IOVA_START_ADDR         (0x1000)
  66
  67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  68
  69 #define MAX_AGAW_WIDTH 64
  70 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  71
  72 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  74
  75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  76    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  77 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  78                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  79 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  80
  81 /* IO virtual address start page frame number */
  82 #define IOVA_START_PFN          (1)
  83
  84 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  85 #define DMA_32BIT_PFN           IOVA_PFN(DMA_BIT_MASK(32))
  86 #define DMA_64BIT_PFN           IOVA_PFN(DMA_BIT_MASK(64))
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186 int intel_iommu_tboot_noforce;
 187
 188 /*
 189  * 0: Present
 190  * 1-11: Reserved
 191  * 12-63: Context Ptr (12 - (haw-1))
 192  * 64-127: Reserved
 193  */
 194 struct root_entry {
 195         u64     lo;
 196         u64     hi;
 197 };
 198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 199
 200 /*
 201  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 202  * if marked present.
 203  */
 204 static phys_addr_t root_entry_lctp(struct root_entry *re)
 205 {
 206         if (!(re->lo & 1))
 207                 return 0;
 208
 209         return re->lo & VTD_PAGE_MASK;
 210 }
 211
 212 /*
 213  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 214  * if marked present.
 215  */
 216 static phys_addr_t root_entry_uctp(struct root_entry *re)
 217 {
 218         if (!(re->hi & 1))
 219                 return 0;
 220
 221         return re->hi & VTD_PAGE_MASK;
 222 }
 223 /*
 224  * low 64 bits:
 225  * 0: present
 226  * 1: fault processing disable
 227  * 2-3: translation type
 228  * 12-63: address space root
 229  * high 64 bits:
 230  * 0-2: address width
 231  * 3-6: aval
 232  * 8-23: domain id
 233  */
 234 struct context_entry {
 235         u64 lo;
 236         u64 hi;
 237 };
 238
 239 static inline void context_clear_pasid_enable(struct context_entry *context)
 240 {
 241         context->lo &= ~(1ULL << 11);
 242 }
 243
 244 static inline bool context_pasid_enabled(struct context_entry *context)
 245 {
 246         return !!(context->lo & (1ULL << 11));
 247 }
 248
 249 static inline void context_set_copied(struct context_entry *context)
 250 {
 251         context->hi |= (1ull << 3);
 252 }
 253
 254 static inline bool context_copied(struct context_entry *context)
 255 {
 256         return !!(context->hi & (1ULL << 3));
 257 }
 258
 259 static inline bool __context_present(struct context_entry *context)
 260 {
 261         return (context->lo & 1);
 262 }
 263
 264 static inline bool context_present(struct context_entry *context)
 265 {
 266         return context_pasid_enabled(context) ?
 267              __context_present(context) :
 268              __context_present(context) && !context_copied(context);
 269 }
 270
 271 static inline void context_set_present(struct context_entry *context)
 272 {
 273         context->lo |= 1;
 274 }
 275
 276 static inline void context_set_fault_enable(struct context_entry *context)
 277 {
 278         context->lo &= (((u64)-1) << 2) | 1;
 279 }
 280
 281 static inline void context_set_translation_type(struct context_entry *context,
 282                                                 unsigned long value)
 283 {
 284         context->lo &= (((u64)-1) << 4) | 3;
 285         context->lo |= (value & 3) << 2;
 286 }
 287
 288 static inline void context_set_address_root(struct context_entry *context,
 289                                             unsigned long value)
 290 {
 291         context->lo &= ~VTD_PAGE_MASK;
 292         context->lo |= value & VTD_PAGE_MASK;
 293 }
 294
 295 static inline void context_set_address_width(struct context_entry *context,
 296                                              unsigned long value)
 297 {
 298         context->hi |= value & 7;
 299 }
 300
 301 static inline void context_set_domain_id(struct context_entry *context,
 302                                          unsigned long value)
 303 {
 304         context->hi |= (value & ((1 << 16) - 1)) << 8;
 305 }
 306
 307 static inline int context_domain_id(struct context_entry *c)
 308 {
 309         return((c->hi >> 8) & 0xffff);
 310 }
 311
 312 static inline void context_clear_entry(struct context_entry *context)
 313 {
 314         context->lo = 0;
 315         context->hi = 0;
 316 }
 317
 318 /*
 319  * 0: readable
 320  * 1: writable
 321  * 2-6: reserved
 322  * 7: super page
 323  * 8-10: available
 324  * 11: snoop behavior
 325  * 12-63: Host physcial address
 326  */
 327 struct dma_pte {
 328         u64 val;
 329 };
 330
 331 static inline void dma_clear_pte(struct dma_pte *pte)
 332 {
 333         pte->val = 0;
 334 }
 335
 336 static inline u64 dma_pte_addr(struct dma_pte *pte)
 337 {
 338 #ifdef CONFIG_64BIT
 339         return pte->val & VTD_PAGE_MASK;
 340 #else
 341         /* Must have a full atomic 64-bit read */
 342         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 343 #endif
 344 }
 345
 346 static inline bool dma_pte_present(struct dma_pte *pte)
 347 {
 348         return (pte->val & 3) != 0;
 349 }
 350
 351 static inline bool dma_pte_superpage(struct dma_pte *pte)
 352 {
 353         return (pte->val & DMA_PTE_LARGE_PAGE);
 354 }
 355
 356 static inline int first_pte_in_page(struct dma_pte *pte)
 357 {
 358         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 359 }
 360
 361 /*
 362  * This domain is a statically identity mapping domain.
 363  *      1. This domain creats a static 1:1 mapping to all usable memory.
 364  *      2. It maps to each iommu if successful.
 365  *      3. Each iommu mapps to this domain if successful.
 366  */
 367 static struct dmar_domain *si_domain;
 368 static int hw_pass_through = 1;
 369
 370 /*
 371  * Domain represents a virtual machine, more than one devices
 372  * across iommus may be owned in one domain, e.g. kvm guest.
 373  */
 374 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 375
 376 /* si_domain contains mulitple devices */
 377 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 378
 379 #define for_each_domain_iommu(idx, domain)                      \
 380         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 381                 if (domain->iommu_refcnt[idx])
 382
 383 struct dmar_domain {
 384         int     nid;                    /* node id */
 385
 386         unsigned        iommu_refcnt[DMAR_UNITS_SUPPORTED];
 387                                         /* Refcount of devices per iommu */
 388
 389
 390         u16             iommu_did[DMAR_UNITS_SUPPORTED];
 391                                         /* Domain ids per IOMMU. Use u16 since
 392                                          * domain ids are 16 bit wide according
 393                                          * to VT-d spec, section 9.3 */
 394
 395         bool has_iotlb_device;
 396         struct list_head devices;       /* all devices' list */
 397         struct iova_domain iovad;       /* iova's that belong to this domain */
 398
 399         struct dma_pte  *pgd;           /* virtual address */
 400         int             gaw;            /* max guest address width */
 401
 402         /* adjusted guest address width, 0 is level 2 30-bit */
 403         int             agaw;
 404
 405         int             flags;          /* flags to find out type of domain */
 406
 407         int             iommu_coherency;/* indicate coherency of iommu access */
 408         int             iommu_snooping; /* indicate snooping control feature*/
 409         int             iommu_count;    /* reference count of iommu */
 410         int             iommu_superpage;/* Level of superpages supported:
 411                                            0 == 4KiB (no superpages), 1 == 2MiB,
 412                                            2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 413         u64             max_addr;       /* maximum mapped address */
 414
 415         struct iommu_domain domain;     /* generic domain data structure for
 416                                            iommu core */
 417 };
 418
 419 /* PCI domain-device relationship */
 420 struct device_domain_info {
 421         struct list_head link;  /* link to domain siblings */
 422         struct list_head global; /* link to global list */
 423         u8 bus;                 /* PCI bus number */
 424         u8 devfn;               /* PCI devfn number */
 425         u8 pasid_supported:3;
 426         u8 pasid_enabled:1;
 427         u8 pri_supported:1;
 428         u8 pri_enabled:1;
 429         u8 ats_supported:1;
 430         u8 ats_enabled:1;
 431         u8 ats_qdep;
 432         struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
 433         struct intel_iommu *iommu; /* IOMMU used by this device */
 434         struct dmar_domain *domain; /* pointer to domain */
 435 };
 436
 437 struct dmar_rmrr_unit {
 438         struct list_head list;          /* list of rmrr units   */
 439         struct acpi_dmar_header *hdr;   /* ACPI header          */
 440         u64     base_address;           /* reserved base address*/
 441         u64     end_address;            /* reserved end address */
 442         struct dmar_dev_scope *devices; /* target devices */
 443         int     devices_cnt;            /* target device count */
 444         struct iommu_resv_region *resv; /* reserved region handle */
 445 };
 446
 447 struct dmar_atsr_unit {
 448         struct list_head list;          /* list of ATSR units */
 449         struct acpi_dmar_header *hdr;   /* ACPI header */
 450         struct dmar_dev_scope *devices; /* target devices */
 451         int devices_cnt;                /* target device count */
 452         u8 include_all:1;               /* include all ports */
 453 };
 454
 455 static LIST_HEAD(dmar_atsr_units);
 456 static LIST_HEAD(dmar_rmrr_units);
 457
 458 #define for_each_rmrr_units(rmrr) \
 459         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 460
 461 static void flush_unmaps_timeout(unsigned long data);
 462
 463 struct deferred_flush_entry {
 464         unsigned long iova_pfn;
 465         unsigned long nrpages;
 466         struct dmar_domain *domain;
 467         struct page *freelist;
 468 };
 469
 470 #define HIGH_WATER_MARK 250
 471 struct deferred_flush_table {
 472         int next;
 473         struct deferred_flush_entry entries[HIGH_WATER_MARK];
 474 };
 475
 476 struct deferred_flush_data {
 477         spinlock_t lock;
 478         int timer_on;
 479         struct timer_list timer;
 480         long size;
 481         struct deferred_flush_table *tables;
 482 };
 483
 484 static DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
 485
 486 /* bitmap for indexing intel_iommus */
 487 static int g_num_of_iommus;
 488
 489 static void domain_exit(struct dmar_domain *domain);
 490 static void domain_remove_dev_info(struct dmar_domain *domain);
 491 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 492                                      struct device *dev);
 493 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 494 static void domain_context_clear(struct intel_iommu *iommu,
 495                                  struct device *dev);
 496 static int domain_detach_iommu(struct dmar_domain *domain,
 497                                struct intel_iommu *iommu);
 498
 499 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 500 int dmar_disabled = 0;
 501 #else
 502 int dmar_disabled = 1;
 503 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 504
 505 int intel_iommu_enabled = 0;
 506 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 507
 508 static int dmar_map_gfx = 1;
 509 static int dmar_forcedac;
 510 static int intel_iommu_strict;
 511 static int intel_iommu_superpage = 1;
 512 static int intel_iommu_ecs = 1;
 513 static int intel_iommu_pasid28;
 514 static int iommu_identity_mapping;
 515
 516 #define IDENTMAP_ALL            1
 517 #define IDENTMAP_GFX            2
 518 #define IDENTMAP_AZALIA         4
 519
 520 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 521  * level" translation of DMA requests-without-PASID doesn't actually happen
 522  * unless you also set the NESTE bit in an extended context-entry. Which of
 523  * course means that SVM doesn't work because it's trying to do nested
 524  * translation of the physical addresses it finds in the process page tables,
 525  * through the IOVA->phys mapping found in the "second level" page tables.
 526  *
 527  * The VT-d specification was retroactively changed to change the definition
 528  * of the capability bits and pretend that Broadwell/Skylake never happened...
 529  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 530  * for some reason it was the PASID capability bit which was redefined (from
 531  * bit 28 on BDW/SKL to bit 40 in future).
 532  *
 533  * So our test for ECS needs to eschew those implementations which set the old
 534  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 535  * Unless we are working around the 'pasid28' limitations, that is, by putting
 536  * the device into passthrough mode for normal DMA and thus masking the bug.
 537  */
 538 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 539                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 540 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 541  * or new capability bits are set. */
 542 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 543                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 544
 545 int intel_iommu_gfx_mapped;
 546 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 547
 548 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 549 static DEFINE_SPINLOCK(device_domain_lock);
 550 static LIST_HEAD(device_domain_list);
 551
 552 const struct iommu_ops intel_iommu_ops;
 553
 554 static bool translation_pre_enabled(struct intel_iommu *iommu)
 555 {
 556         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 557 }
 558
 559 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 560 {
 561         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 562 }
 563
 564 static void init_translation_status(struct intel_iommu *iommu)
 565 {
 566         u32 gsts;
 567
 568         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 569         if (gsts & DMA_GSTS_TES)
 570                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 571 }
 572
 573 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 574 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 575 {
 576         return container_of(dom, struct dmar_domain, domain);
 577 }
 578
 579 static int __init intel_iommu_setup(char *str)
 580 {
 581         if (!str)
 582                 return -EINVAL;
 583         while (*str) {
 584                 if (!strncmp(str, "on", 2)) {
 585                         dmar_disabled = 0;
 586                         pr_info("IOMMU enabled\n");
 587                 } else if (!strncmp(str, "off", 3)) {
 588                         dmar_disabled = 1;
 589                         pr_info("IOMMU disabled\n");
 590                 } else if (!strncmp(str, "igfx_off", 8)) {
 591                         dmar_map_gfx = 0;
 592                         pr_info("Disable GFX device mapping\n");
 593                 } else if (!strncmp(str, "forcedac", 8)) {
 594                         pr_info("Forcing DAC for PCI devices\n");
 595                         dmar_forcedac = 1;
 596                 } else if (!strncmp(str, "strict", 6)) {
 597                         pr_info("Disable batched IOTLB flush\n");
 598                         intel_iommu_strict = 1;
 599                 } else if (!strncmp(str, "sp_off", 6)) {
 600                         pr_info("Disable supported super page\n");
 601                         intel_iommu_superpage = 0;
 602                 } else if (!strncmp(str, "ecs_off", 7)) {
 603                         printk(KERN_INFO
 604                                 "Intel-IOMMU: disable extended context table support\n");
 605                         intel_iommu_ecs = 0;
 606                 } else if (!strncmp(str, "pasid28", 7)) {
 607                         printk(KERN_INFO
 608                                 "Intel-IOMMU: enable pre-production PASID support\n");
 609                         intel_iommu_pasid28 = 1;
 610                         iommu_identity_mapping |= IDENTMAP_GFX;
 611                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 612                         printk(KERN_INFO
 613                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 614                         intel_iommu_tboot_noforce = 1;
 615                 }
 616
 617                 str += strcspn(str, ",");
 618                 while (*str == ',')
 619                         str++;
 620         }
 621         return 0;
 622 }
 623 __setup("intel_iommu=", intel_iommu_setup);
 624
 625 static struct kmem_cache *iommu_domain_cache;
 626 static struct kmem_cache *iommu_devinfo_cache;
 627
 628 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 629 {
 630         struct dmar_domain **domains;
 631         int idx = did >> 8;
 632
 633         domains = iommu->domains[idx];
 634         if (!domains)
 635                 return NULL;
 636
 637         return domains[did & 0xff];
 638 }
 639
 640 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 641                              struct dmar_domain *domain)
 642 {
 643         struct dmar_domain **domains;
 644         int idx = did >> 8;
 645
 646         if (!iommu->domains[idx]) {
 647                 size_t size = 256 * sizeof(struct dmar_domain *);
 648                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 649         }
 650
 651         domains = iommu->domains[idx];
 652         if (WARN_ON(!domains))
 653                 return;
 654         else
 655                 domains[did & 0xff] = domain;
 656 }
 657
 658 static inline void *alloc_pgtable_page(int node)
 659 {
 660         struct page *page;
 661         void *vaddr = NULL;
 662
 663         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 664         if (page)
 665                 vaddr = page_address(page);
 666         return vaddr;
 667 }
 668
 669 static inline void free_pgtable_page(void *vaddr)
 670 {
 671         free_page((unsigned long)vaddr);
 672 }
 673
 674 static inline void *alloc_domain_mem(void)
 675 {
 676         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 677 }
 678
 679 static void free_domain_mem(void *vaddr)
 680 {
 681         kmem_cache_free(iommu_domain_cache, vaddr);
 682 }
 683
 684 static inline void * alloc_devinfo_mem(void)
 685 {
 686         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 687 }
 688
 689 static inline void free_devinfo_mem(void *vaddr)
 690 {
 691         kmem_cache_free(iommu_devinfo_cache, vaddr);
 692 }
 693
 694 static inline int domain_type_is_vm(struct dmar_domain *domain)
 695 {
 696         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 697 }
 698
 699 static inline int domain_type_is_si(struct dmar_domain *domain)
 700 {
 701         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 702 }
 703
 704 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 705 {
 706         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 707                                 DOMAIN_FLAG_STATIC_IDENTITY);
 708 }
 709
 710 static inline int domain_pfn_supported(struct dmar_domain *domain,
 711                                        unsigned long pfn)
 712 {
 713         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 714
 715         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 716 }
 717
 718 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 719 {
 720         unsigned long sagaw;
 721         int agaw = -1;
 722
 723         sagaw = cap_sagaw(iommu->cap);
 724         for (agaw = width_to_agaw(max_gaw);
 725              agaw >= 0; agaw--) {
 726                 if (test_bit(agaw, &sagaw))
 727                         break;
 728         }
 729
 730         return agaw;
 731 }
 732
 733 /*
 734  * Calculate max SAGAW for each iommu.
 735  */
 736 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 737 {
 738         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 739 }
 740
 741 /*
 742  * calculate agaw for each iommu.
 743  * "SAGAW" may be different across iommus, use a default agaw, and
 744  * get a supported less agaw for iommus that don't support the default agaw.
 745  */
 746 int iommu_calculate_agaw(struct intel_iommu *iommu)
 747 {
 748         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 749 }
 750
 751 /* This functionin only returns single iommu in a domain */
 752 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 753 {
 754         int iommu_id;
 755
 756         /* si_domain and vm domain should not get here. */
 757         BUG_ON(domain_type_is_vm_or_si(domain));
 758         for_each_domain_iommu(iommu_id, domain)
 759                 break;
 760
 761         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 762                 return NULL;
 763
 764         return g_iommus[iommu_id];
 765 }
 766
 767 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 768 {
 769         struct dmar_drhd_unit *drhd;
 770         struct intel_iommu *iommu;
 771         bool found = false;
 772         int i;
 773
 774         domain->iommu_coherency = 1;
 775
 776         for_each_domain_iommu(i, domain) {
 777                 found = true;
 778                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 779                         domain->iommu_coherency = 0;
 780                         break;
 781                 }
 782         }
 783         if (found)
 784                 return;
 785
 786         /* No hardware attached; use lowest common denominator */
 787         rcu_read_lock();
 788         for_each_active_iommu(iommu, drhd) {
 789                 if (!ecap_coherent(iommu->ecap)) {
 790                         domain->iommu_coherency = 0;
 791                         break;
 792                 }
 793         }
 794         rcu_read_unlock();
 795 }
 796
 797 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 798 {
 799         struct dmar_drhd_unit *drhd;
 800         struct intel_iommu *iommu;
 801         int ret = 1;
 802
 803         rcu_read_lock();
 804         for_each_active_iommu(iommu, drhd) {
 805                 if (iommu != skip) {
 806                         if (!ecap_sc_support(iommu->ecap)) {
 807                                 ret = 0;
 808                                 break;
 809                         }
 810                 }
 811         }
 812         rcu_read_unlock();
 813
 814         return ret;
 815 }
 816
 817 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 818 {
 819         struct dmar_drhd_unit *drhd;
 820         struct intel_iommu *iommu;
 821         int mask = 0xf;
 822
 823         if (!intel_iommu_superpage) {
 824                 return 0;
 825         }
 826
 827         /* set iommu_superpage to the smallest common denominator */
 828         rcu_read_lock();
 829         for_each_active_iommu(iommu, drhd) {
 830                 if (iommu != skip) {
 831                         mask &= cap_super_page_val(iommu->cap);
 832                         if (!mask)
 833                                 break;
 834                 }
 835         }
 836         rcu_read_unlock();
 837
 838         return fls(mask);
 839 }
 840
 841 /* Some capabilities may be different across iommus */
 842 static void domain_update_iommu_cap(struct dmar_domain *domain)
 843 {
 844         domain_update_iommu_coherency(domain);
 845         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 846         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 847 }
 848
 849 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 850                                                        u8 bus, u8 devfn, int alloc)
 851 {
 852         struct root_entry *root = &iommu->root_entry[bus];
 853         struct context_entry *context;
 854         u64 *entry;
 855
 856         entry = &root->lo;
 857         if (ecs_enabled(iommu)) {
 858                 if (devfn >= 0x80) {
 859                         devfn -= 0x80;
 860                         entry = &root->hi;
 861                 }
 862                 devfn *= 2;
 863         }
 864         if (*entry & 1)
 865                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 866         else {
 867                 unsigned long phy_addr;
 868                 if (!alloc)
 869                         return NULL;
 870
 871                 context = alloc_pgtable_page(iommu->node);
 872                 if (!context)
 873                         return NULL;
 874
 875                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 876                 phy_addr = virt_to_phys((void *)context);
 877                 *entry = phy_addr | 1;
 878                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 879         }
 880         return &context[devfn];
 881 }
 882
 883 static int iommu_dummy(struct device *dev)
 884 {
 885         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 886 }
 887
 888 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 889 {
 890         struct dmar_drhd_unit *drhd = NULL;
 891         struct intel_iommu *iommu;
 892         struct device *tmp;
 893         struct pci_dev *ptmp, *pdev = NULL;
 894         u16 segment = 0;
 895         int i;
 896
 897         if (iommu_dummy(dev))
 898                 return NULL;
 899
 900         if (dev_is_pci(dev)) {
 901                 struct pci_dev *pf_pdev;
 902
 903                 pdev = to_pci_dev(dev);
 904                 /* VFs aren't listed in scope tables; we need to look up
 905                  * the PF instead to find the IOMMU. */
 906                 pf_pdev = pci_physfn(pdev);
 907                 dev = &pf_pdev->dev;
 908                 segment = pci_domain_nr(pdev->bus);
 909         } else if (has_acpi_companion(dev))
 910                 dev = &ACPI_COMPANION(dev)->dev;
 911
 912         rcu_read_lock();
 913         for_each_active_iommu(iommu, drhd) {
 914                 if (pdev && segment != drhd->segment)
 915                         continue;
 916
 917                 for_each_active_dev_scope(drhd->devices,
 918                                           drhd->devices_cnt, i, tmp) {
 919                         if (tmp == dev) {
 920                                 /* For a VF use its original BDF# not that of the PF
 921                                  * which we used for the IOMMU lookup. Strictly speaking
 922                                  * we could do this for all PCI devices; we only need to
 923                                  * get the BDF# from the scope table for ACPI matches. */
 924                                 if (pdev && pdev->is_virtfn)
 925                                         goto got_pdev;
 926
 927                                 *bus = drhd->devices[i].bus;
 928                                 *devfn = drhd->devices[i].devfn;
 929                                 goto out;
 930                         }
 931
 932                         if (!pdev || !dev_is_pci(tmp))
 933                                 continue;
 934
 935                         ptmp = to_pci_dev(tmp);
 936                         if (ptmp->subordinate &&
 937                             ptmp->subordinate->number <= pdev->bus->number &&
 938                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 939                                 goto got_pdev;
 940                 }
 941
 942                 if (pdev && drhd->include_all) {
 943                 got_pdev:
 944                         *bus = pdev->bus->number;
 945                         *devfn = pdev->devfn;
 946                         goto out;
 947                 }
 948         }
 949         iommu = NULL;
 950  out:
 951         rcu_read_unlock();
 952
 953         return iommu;
 954 }
 955
 956 static void domain_flush_cache(struct dmar_domain *domain,
 957                                void *addr, int size)
 958 {
 959         if (!domain->iommu_coherency)
 960                 clflush_cache_range(addr, size);
 961 }
 962
 963 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 964 {
 965         struct context_entry *context;
 966         int ret = 0;
 967         unsigned long flags;
 968
 969         spin_lock_irqsave(&iommu->lock, flags);
 970         context = iommu_context_addr(iommu, bus, devfn, 0);
 971         if (context)
 972                 ret = context_present(context);
 973         spin_unlock_irqrestore(&iommu->lock, flags);
 974         return ret;
 975 }
 976
 977 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 978 {
 979         struct context_entry *context;
 980         unsigned long flags;
 981
 982         spin_lock_irqsave(&iommu->lock, flags);
 983         context = iommu_context_addr(iommu, bus, devfn, 0);
 984         if (context) {
 985                 context_clear_entry(context);
 986                 __iommu_flush_cache(iommu, context, sizeof(*context));
 987         }
 988         spin_unlock_irqrestore(&iommu->lock, flags);
 989 }
 990
 991 static void free_context_table(struct intel_iommu *iommu)
 992 {
 993         int i;
 994         unsigned long flags;
 995         struct context_entry *context;
 996
 997         spin_lock_irqsave(&iommu->lock, flags);
 998         if (!iommu->root_entry) {
 999                 goto out;
1000         }
1001         for (i = 0; i < ROOT_ENTRY_NR; i++) {
1002                 context = iommu_context_addr(iommu, i, 0, 0);
1003                 if (context)
1004                         free_pgtable_page(context);
1005
1006                 if (!ecs_enabled(iommu))
1007                         continue;
1008
1009                 context = iommu_context_addr(iommu, i, 0x80, 0);
1010                 if (context)
1011                         free_pgtable_page(context);
1012
1013         }
1014         free_pgtable_page(iommu->root_entry);
1015         iommu->root_entry = NULL;
1016 out:
1017         spin_unlock_irqrestore(&iommu->lock, flags);
1018 }
1019
1020 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1021                                       unsigned long pfn, int *target_level)
1022 {
1023         struct dma_pte *parent, *pte = NULL;
1024         int level = agaw_to_level(domain->agaw);
1025         int offset;
1026
1027         BUG_ON(!domain->pgd);
1028
1029         if (!domain_pfn_supported(domain, pfn))
1030                 /* Address beyond IOMMU's addressing capabilities. */
1031                 return NULL;
1032
1033         parent = domain->pgd;
1034
1035         while (1) {
1036                 void *tmp_page;
1037
1038                 offset = pfn_level_offset(pfn, level);
1039                 pte = &parent[offset];
1040                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1041                         break;
1042                 if (level == *target_level)
1043                         break;
1044
1045                 if (!dma_pte_present(pte)) {
1046                         uint64_t pteval;
1047
1048                         tmp_page = alloc_pgtable_page(domain->nid);
1049
1050                         if (!tmp_page)
1051                                 return NULL;
1052
1053                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1054                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1055                         if (cmpxchg64(&pte->val, 0ULL, pteval))
1056                                 /* Someone else set it while we were thinking; use theirs. */
1057                                 free_pgtable_page(tmp_page);
1058                         else
1059                                 domain_flush_cache(domain, pte, sizeof(*pte));
1060                 }
1061                 if (level == 1)
1062                         break;
1063
1064                 parent = phys_to_virt(dma_pte_addr(pte));
1065                 level--;
1066         }
1067
1068         if (!*target_level)
1069                 *target_level = level;
1070
1071         return pte;
1072 }
1073
1074
1075 /* return address's pte at specific level */
1076 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1077                                          unsigned long pfn,
1078                                          int level, int *large_page)
1079 {
1080         struct dma_pte *parent, *pte = NULL;
1081         int total = agaw_to_level(domain->agaw);
1082         int offset;
1083
1084         parent = domain->pgd;
1085         while (level <= total) {
1086                 offset = pfn_level_offset(pfn, total);
1087                 pte = &parent[offset];
1088                 if (level == total)
1089                         return pte;
1090
1091                 if (!dma_pte_present(pte)) {
1092                         *large_page = total;
1093                         break;
1094                 }
1095
1096                 if (dma_pte_superpage(pte)) {
1097                         *large_page = total;
1098                         return pte;
1099                 }
1100
1101                 parent = phys_to_virt(dma_pte_addr(pte));
1102                 total--;
1103         }
1104         return NULL;
1105 }
1106
1107 /* clear last level pte, a tlb flush should be followed */
1108 static void dma_pte_clear_range(struct dmar_domain *domain,
1109                                 unsigned long start_pfn,
1110                                 unsigned long last_pfn)
1111 {
1112         unsigned int large_page = 1;
1113         struct dma_pte *first_pte, *pte;
1114
1115         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1116         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1117         BUG_ON(start_pfn > last_pfn);
1118
1119         /* we don't need lock here; nobody else touches the iova range */
1120         do {
1121                 large_page = 1;
1122                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1123                 if (!pte) {
1124                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1125                         continue;
1126                 }
1127                 do {
1128                         dma_clear_pte(pte);
1129                         start_pfn += lvl_to_nr_pages(large_page);
1130                         pte++;
1131                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1132
1133                 domain_flush_cache(domain, first_pte,
1134                                    (void *)pte - (void *)first_pte);
1135
1136         } while (start_pfn && start_pfn <= last_pfn);
1137 }
1138
1139 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1140                                struct dma_pte *pte, unsigned long pfn,
1141                                unsigned long start_pfn, unsigned long last_pfn)
1142 {
1143         pfn = max(start_pfn, pfn);
1144         pte = &pte[pfn_level_offset(pfn, level)];
1145
1146         do {
1147                 unsigned long level_pfn;
1148                 struct dma_pte *level_pte;
1149
1150                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1151                         goto next;
1152
1153                 level_pfn = pfn & level_mask(level);
1154                 level_pte = phys_to_virt(dma_pte_addr(pte));
1155
1156                 if (level > 2)
1157                         dma_pte_free_level(domain, level - 1, level_pte,
1158                                            level_pfn, start_pfn, last_pfn);
1159
1160                 /* If range covers entire pagetable, free it */
1161                 if (!(start_pfn > level_pfn ||
1162                       last_pfn < level_pfn + level_size(level) - 1)) {
1163                         dma_clear_pte(pte);
1164                         domain_flush_cache(domain, pte, sizeof(*pte));
1165                         free_pgtable_page(level_pte);
1166                 }
1167 next:
1168                 pfn += level_size(level);
1169         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1170 }
1171
1172 /* clear last level (leaf) ptes and free page table pages. */
1173 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1174                                    unsigned long start_pfn,
1175                                    unsigned long last_pfn)
1176 {
1177         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1178         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1179         BUG_ON(start_pfn > last_pfn);
1180
1181         dma_pte_clear_range(domain, start_pfn, last_pfn);
1182
1183         /* We don't need lock here; nobody else touches the iova range */
1184         dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1185                            domain->pgd, 0, start_pfn, last_pfn);
1186
1187         /* free pgd */
1188         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1189                 free_pgtable_page(domain->pgd);
1190                 domain->pgd = NULL;
1191         }
1192 }
1193
1194 /* When a page at a given level is being unlinked from its parent, we don't
1195    need to *modify* it at all. All we need to do is make a list of all the
1196    pages which can be freed just as soon as we've flushed the IOTLB and we
1197    know the hardware page-walk will no longer touch them.
1198    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1199    be freed. */
1200 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1201                                             int level, struct dma_pte *pte,
1202                                             struct page *freelist)
1203 {
1204         struct page *pg;
1205
1206         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1207         pg->freelist = freelist;
1208         freelist = pg;
1209
1210         if (level == 1)
1211                 return freelist;
1212
1213         pte = page_address(pg);
1214         do {
1215                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1216                         freelist = dma_pte_list_pagetables(domain, level - 1,
1217                                                            pte, freelist);
1218                 pte++;
1219         } while (!first_pte_in_page(pte));
1220
1221         return freelist;
1222 }
1223
1224 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1225                                         struct dma_pte *pte, unsigned long pfn,
1226                                         unsigned long start_pfn,
1227                                         unsigned long last_pfn,
1228                                         struct page *freelist)
1229 {
1230         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1231
1232         pfn = max(start_pfn, pfn);
1233         pte = &pte[pfn_level_offset(pfn, level)];
1234
1235         do {
1236                 unsigned long level_pfn;
1237
1238                 if (!dma_pte_present(pte))
1239                         goto next;
1240
1241                 level_pfn = pfn & level_mask(level);
1242
1243                 /* If range covers entire pagetable, free it */
1244                 if (start_pfn <= level_pfn &&
1245                     last_pfn >= level_pfn + level_size(level) - 1) {
1246                         /* These suborbinate page tables are going away entirely. Don't
1247                            bother to clear them; we're just going to *free* them. */
1248                         if (level > 1 && !dma_pte_superpage(pte))
1249                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1250
1251                         dma_clear_pte(pte);
1252                         if (!first_pte)
1253                                 first_pte = pte;
1254                         last_pte = pte;
1255                 } else if (level > 1) {
1256                         /* Recurse down into a level that isn't *entirely* obsolete */
1257                         freelist = dma_pte_clear_level(domain, level - 1,
1258                                                        phys_to_virt(dma_pte_addr(pte)),
1259                                                        level_pfn, start_pfn, last_pfn,
1260                                                        freelist);
1261                 }
1262 next:
1263                 pfn += level_size(level);
1264         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1265
1266         if (first_pte)
1267                 domain_flush_cache(domain, first_pte,
1268                                    (void *)++last_pte - (void *)first_pte);
1269
1270         return freelist;
1271 }
1272
1273 /* We can't just free the pages because the IOMMU may still be walking
1274    the page tables, and may have cached the intermediate levels. The
1275    pages can only be freed after the IOTLB flush has been done. */
1276 static struct page *domain_unmap(struct dmar_domain *domain,
1277                                  unsigned long start_pfn,
1278                                  unsigned long last_pfn)
1279 {
1280         struct page *freelist = NULL;
1281
1282         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1283         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1284         BUG_ON(start_pfn > last_pfn);
1285
1286         /* we don't need lock here; nobody else touches the iova range */
1287         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1288                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1289
1290         /* free pgd */
1291         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1292                 struct page *pgd_page = virt_to_page(domain->pgd);
1293                 pgd_page->freelist = freelist;
1294                 freelist = pgd_page;
1295
1296                 domain->pgd = NULL;
1297         }
1298
1299         return freelist;
1300 }
1301
1302 static void dma_free_pagelist(struct page *freelist)
1303 {
1304         struct page *pg;
1305
1306         while ((pg = freelist)) {
1307                 freelist = pg->freelist;
1308                 free_pgtable_page(page_address(pg));
1309         }
1310 }
1311
1312 /* iommu handling */
1313 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1314 {
1315         struct root_entry *root;
1316         unsigned long flags;
1317
1318         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1319         if (!root) {
1320                 pr_err("Allocating root entry for %s failed\n",
1321                         iommu->name);
1322                 return -ENOMEM;
1323         }
1324
1325         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1326
1327         spin_lock_irqsave(&iommu->lock, flags);
1328         iommu->root_entry = root;
1329         spin_unlock_irqrestore(&iommu->lock, flags);
1330
1331         return 0;
1332 }
1333
1334 static void iommu_set_root_entry(struct intel_iommu *iommu)
1335 {
1336         u64 addr;
1337         u32 sts;
1338         unsigned long flag;
1339
1340         addr = virt_to_phys(iommu->root_entry);
1341         if (ecs_enabled(iommu))
1342                 addr |= DMA_RTADDR_RTT;
1343
1344         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1345         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1346
1347         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1348
1349         /* Make sure hardware complete it */
1350         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1351                       readl, (sts & DMA_GSTS_RTPS), sts);
1352
1353         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1354 }
1355
1356 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1357 {
1358         u32 val;
1359         unsigned long flag;
1360
1361         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1362                 return;
1363
1364         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1365         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1366
1367         /* Make sure hardware complete it */
1368         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1369                       readl, (!(val & DMA_GSTS_WBFS)), val);
1370
1371         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1372 }
1373
1374 /* return value determine if we need a write buffer flush */
1375 static void __iommu_flush_context(struct intel_iommu *iommu,
1376                                   u16 did, u16 source_id, u8 function_mask,
1377                                   u64 type)
1378 {
1379         u64 val = 0;
1380         unsigned long flag;
1381
1382         switch (type) {
1383         case DMA_CCMD_GLOBAL_INVL:
1384                 val = DMA_CCMD_GLOBAL_INVL;
1385                 break;
1386         case DMA_CCMD_DOMAIN_INVL:
1387                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1388                 break;
1389         case DMA_CCMD_DEVICE_INVL:
1390                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1391                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1392                 break;
1393         default:
1394                 BUG();
1395         }
1396         val |= DMA_CCMD_ICC;
1397
1398         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1399         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1400
1401         /* Make sure hardware complete it */
1402         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1403                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1404
1405         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1406 }
1407
1408 /* return value determine if we need a write buffer flush */
1409 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1410                                 u64 addr, unsigned int size_order, u64 type)
1411 {
1412         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1413         u64 val = 0, val_iva = 0;
1414         unsigned long flag;
1415
1416         switch (type) {
1417         case DMA_TLB_GLOBAL_FLUSH:
1418                 /* global flush doesn't need set IVA_REG */
1419                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1420                 break;
1421         case DMA_TLB_DSI_FLUSH:
1422                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1423                 break;
1424         case DMA_TLB_PSI_FLUSH:
1425                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1426                 /* IH bit is passed in as part of address */
1427                 val_iva = size_order | addr;
1428                 break;
1429         default:
1430                 BUG();
1431         }
1432         /* Note: set drain read/write */
1433 #if 0
1434         /*
1435          * This is probably to be super secure.. Looks like we can
1436          * ignore it without any impact.
1437          */
1438         if (cap_read_drain(iommu->cap))
1439                 val |= DMA_TLB_READ_DRAIN;
1440 #endif
1441         if (cap_write_drain(iommu->cap))
1442                 val |= DMA_TLB_WRITE_DRAIN;
1443
1444         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1445         /* Note: Only uses first TLB reg currently */
1446         if (val_iva)
1447                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1448         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1449
1450         /* Make sure hardware complete it */
1451         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1452                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1453
1454         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1455
1456         /* check IOTLB invalidation granularity */
1457         if (DMA_TLB_IAIG(val) == 0)
1458                 pr_err("Flush IOTLB failed\n");
1459         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1460                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1461                         (unsigned long long)DMA_TLB_IIRG(type),
1462                         (unsigned long long)DMA_TLB_IAIG(val));
1463 }
1464
1465 static struct device_domain_info *
1466 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1467                          u8 bus, u8 devfn)
1468 {
1469         struct device_domain_info *info;
1470
1471         assert_spin_locked(&device_domain_lock);
1472
1473         if (!iommu->qi)
1474                 return NULL;
1475
1476         list_for_each_entry(info, &domain->devices, link)
1477                 if (info->iommu == iommu && info->bus == bus &&
1478                     info->devfn == devfn) {
1479                         if (info->ats_supported && info->dev)
1480                                 return info;
1481                         break;
1482                 }
1483
1484         return NULL;
1485 }
1486
1487 static void domain_update_iotlb(struct dmar_domain *domain)
1488 {
1489         struct device_domain_info *info;
1490         bool has_iotlb_device = false;
1491
1492         assert_spin_locked(&device_domain_lock);
1493
1494         list_for_each_entry(info, &domain->devices, link) {
1495                 struct pci_dev *pdev;
1496
1497                 if (!info->dev || !dev_is_pci(info->dev))
1498                         continue;
1499
1500                 pdev = to_pci_dev(info->dev);
1501                 if (pdev->ats_enabled) {
1502                         has_iotlb_device = true;
1503                         break;
1504                 }
1505         }
1506
1507         domain->has_iotlb_device = has_iotlb_device;
1508 }
1509
1510 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1511 {
1512         struct pci_dev *pdev;
1513
1514         assert_spin_locked(&device_domain_lock);
1515
1516         if (!info || !dev_is_pci(info->dev))
1517                 return;
1518
1519         pdev = to_pci_dev(info->dev);
1520
1521 #ifdef CONFIG_INTEL_IOMMU_SVM
1522         /* The PCIe spec, in its wisdom, declares that the behaviour of
1523            the device if you enable PASID support after ATS support is
1524            undefined. So always enable PASID support on devices which
1525            have it, even if we can't yet know if we're ever going to
1526            use it. */
1527         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1528                 info->pasid_enabled = 1;
1529
1530         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1531                 info->pri_enabled = 1;
1532 #endif
1533         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1534                 info->ats_enabled = 1;
1535                 domain_update_iotlb(info->domain);
1536                 info->ats_qdep = pci_ats_queue_depth(pdev);
1537         }
1538 }
1539
1540 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1541 {
1542         struct pci_dev *pdev;
1543
1544         assert_spin_locked(&device_domain_lock);
1545
1546         if (!dev_is_pci(info->dev))
1547                 return;
1548
1549         pdev = to_pci_dev(info->dev);
1550
1551         if (info->ats_enabled) {
1552                 pci_disable_ats(pdev);
1553                 info->ats_enabled = 0;
1554                 domain_update_iotlb(info->domain);
1555         }
1556 #ifdef CONFIG_INTEL_IOMMU_SVM
1557         if (info->pri_enabled) {
1558                 pci_disable_pri(pdev);
1559                 info->pri_enabled = 0;
1560         }
1561         if (info->pasid_enabled) {
1562                 pci_disable_pasid(pdev);
1563                 info->pasid_enabled = 0;
1564         }
1565 #endif
1566 }
1567
1568 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1569                                   u64 addr, unsigned mask)
1570 {
1571         u16 sid, qdep;
1572         unsigned long flags;
1573         struct device_domain_info *info;
1574
1575         if (!domain->has_iotlb_device)
1576                 return;
1577
1578         spin_lock_irqsave(&device_domain_lock, flags);
1579         list_for_each_entry(info, &domain->devices, link) {
1580                 if (!info->ats_enabled)
1581                         continue;
1582
1583                 sid = info->bus << 8 | info->devfn;
1584                 qdep = info->ats_qdep;
1585                 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1586         }
1587         spin_unlock_irqrestore(&device_domain_lock, flags);
1588 }
1589
1590 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1591                                   struct dmar_domain *domain,
1592                                   unsigned long pfn, unsigned int pages,
1593                                   int ih, int map)
1594 {
1595         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1596         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1597         u16 did = domain->iommu_did[iommu->seq_id];
1598
1599         BUG_ON(pages == 0);
1600
1601         if (ih)
1602                 ih = 1 << 6;
1603         /*
1604          * Fallback to domain selective flush if no PSI support or the size is
1605          * too big.
1606          * PSI requires page size to be 2 ^ x, and the base address is naturally
1607          * aligned to the size
1608          */
1609         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1610                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1611                                                 DMA_TLB_DSI_FLUSH);
1612         else
1613                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1614                                                 DMA_TLB_PSI_FLUSH);
1615
1616         /*
1617          * In caching mode, changes of pages from non-present to present require
1618          * flush. However, device IOTLB doesn't need to be flushed in this case.
1619          */
1620         if (!cap_caching_mode(iommu->cap) || !map)
1621                 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1622                                       addr, mask);
1623 }
1624
1625 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1626 {
1627         u32 pmen;
1628         unsigned long flags;
1629
1630         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1631         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1632         pmen &= ~DMA_PMEN_EPM;
1633         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1634
1635         /* wait for the protected region status bit to clear */
1636         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1637                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1638
1639         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1640 }
1641
1642 static void iommu_enable_translation(struct intel_iommu *iommu)
1643 {
1644         u32 sts;
1645         unsigned long flags;
1646
1647         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1648         iommu->gcmd |= DMA_GCMD_TE;
1649         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1650
1651         /* Make sure hardware complete it */
1652         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1653                       readl, (sts & DMA_GSTS_TES), sts);
1654
1655         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1656 }
1657
1658 static void iommu_disable_translation(struct intel_iommu *iommu)
1659 {
1660         u32 sts;
1661         unsigned long flag;
1662
1663         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1664         iommu->gcmd &= ~DMA_GCMD_TE;
1665         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1666
1667         /* Make sure hardware complete it */
1668         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1669                       readl, (!(sts & DMA_GSTS_TES)), sts);
1670
1671         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1672 }
1673
1674
1675 static int iommu_init_domains(struct intel_iommu *iommu)
1676 {
1677         u32 ndomains, nlongs;
1678         size_t size;
1679
1680         ndomains = cap_ndoms(iommu->cap);
1681         pr_debug("%s: Number of Domains supported <%d>\n",
1682                  iommu->name, ndomains);
1683         nlongs = BITS_TO_LONGS(ndomains);
1684
1685         spin_lock_init(&iommu->lock);
1686
1687         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1688         if (!iommu->domain_ids) {
1689                 pr_err("%s: Allocating domain id array failed\n",
1690                        iommu->name);
1691                 return -ENOMEM;
1692         }
1693
1694         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1695         iommu->domains = kzalloc(size, GFP_KERNEL);
1696
1697         if (iommu->domains) {
1698                 size = 256 * sizeof(struct dmar_domain *);
1699                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1700         }
1701
1702         if (!iommu->domains || !iommu->domains[0]) {
1703                 pr_err("%s: Allocating domain array failed\n",
1704                        iommu->name);
1705                 kfree(iommu->domain_ids);
1706                 kfree(iommu->domains);
1707                 iommu->domain_ids = NULL;
1708                 iommu->domains    = NULL;
1709                 return -ENOMEM;
1710         }
1711
1712
1713
1714         /*
1715          * If Caching mode is set, then invalid translations are tagged
1716          * with domain-id 0, hence we need to pre-allocate it. We also
1717          * use domain-id 0 as a marker for non-allocated domain-id, so
1718          * make sure it is not used for a real domain.
1719          */
1720         set_bit(0, iommu->domain_ids);
1721
1722         return 0;
1723 }
1724
1725 static void disable_dmar_iommu(struct intel_iommu *iommu)
1726 {
1727         struct device_domain_info *info, *tmp;
1728         unsigned long flags;
1729
1730         if (!iommu->domains || !iommu->domain_ids)
1731                 return;
1732
1733 again:
1734         spin_lock_irqsave(&device_domain_lock, flags);
1735         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1736                 struct dmar_domain *domain;
1737
1738                 if (info->iommu != iommu)
1739                         continue;
1740
1741                 if (!info->dev || !info->domain)
1742                         continue;
1743
1744                 domain = info->domain;
1745
1746                 __dmar_remove_one_dev_info(info);
1747
1748                 if (!domain_type_is_vm_or_si(domain)) {
1749                         /*
1750                          * The domain_exit() function  can't be called under
1751                          * device_domain_lock, as it takes this lock itself.
1752                          * So release the lock here and re-run the loop
1753                          * afterwards.
1754                          */
1755                         spin_unlock_irqrestore(&device_domain_lock, flags);
1756                         domain_exit(domain);
1757                         goto again;
1758                 }
1759         }
1760         spin_unlock_irqrestore(&device_domain_lock, flags);
1761
1762         if (iommu->gcmd & DMA_GCMD_TE)
1763                 iommu_disable_translation(iommu);
1764 }
1765
1766 static void free_dmar_iommu(struct intel_iommu *iommu)
1767 {
1768         if ((iommu->domains) && (iommu->domain_ids)) {
1769                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1770                 int i;
1771
1772                 for (i = 0; i < elems; i++)
1773                         kfree(iommu->domains[i]);
1774                 kfree(iommu->domains);
1775                 kfree(iommu->domain_ids);
1776                 iommu->domains = NULL;
1777                 iommu->domain_ids = NULL;
1778         }
1779
1780         g_iommus[iommu->seq_id] = NULL;
1781
1782         /* free context mapping */
1783         free_context_table(iommu);
1784
1785 #ifdef CONFIG_INTEL_IOMMU_SVM
1786         if (pasid_enabled(iommu)) {
1787                 if (ecap_prs(iommu->ecap))
1788                         intel_svm_finish_prq(iommu);
1789                 intel_svm_free_pasid_tables(iommu);
1790         }
1791 #endif
1792 }
1793
1794 static struct dmar_domain *alloc_domain(int flags)
1795 {
1796         struct dmar_domain *domain;
1797
1798         domain = alloc_domain_mem();
1799         if (!domain)
1800                 return NULL;
1801
1802         memset(domain, 0, sizeof(*domain));
1803         domain->nid = -1;
1804         domain->flags = flags;
1805         domain->has_iotlb_device = false;
1806         INIT_LIST_HEAD(&domain->devices);
1807
1808         return domain;
1809 }
1810
1811 /* Must be called with iommu->lock */
1812 static int domain_attach_iommu(struct dmar_domain *domain,
1813                                struct intel_iommu *iommu)
1814 {
1815         unsigned long ndomains;
1816         int num;
1817
1818         assert_spin_locked(&device_domain_lock);
1819         assert_spin_locked(&iommu->lock);
1820
1821         domain->iommu_refcnt[iommu->seq_id] += 1;
1822         domain->iommu_count += 1;
1823         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1824                 ndomains = cap_ndoms(iommu->cap);
1825                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1826
1827                 if (num >= ndomains) {
1828                         pr_err("%s: No free domain ids\n", iommu->name);
1829                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1830                         domain->iommu_count -= 1;
1831                         return -ENOSPC;
1832                 }
1833
1834                 set_bit(num, iommu->domain_ids);
1835                 set_iommu_domain(iommu, num, domain);
1836
1837                 domain->iommu_did[iommu->seq_id] = num;
1838                 domain->nid                      = iommu->node;
1839
1840                 domain_update_iommu_cap(domain);
1841         }
1842
1843         return 0;
1844 }
1845
1846 static int domain_detach_iommu(struct dmar_domain *domain,
1847                                struct intel_iommu *iommu)
1848 {
1849         int num, count = INT_MAX;
1850
1851         assert_spin_locked(&device_domain_lock);
1852         assert_spin_locked(&iommu->lock);
1853
1854         domain->iommu_refcnt[iommu->seq_id] -= 1;
1855         count = --domain->iommu_count;
1856         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1857                 num = domain->iommu_did[iommu->seq_id];
1858                 clear_bit(num, iommu->domain_ids);
1859                 set_iommu_domain(iommu, num, NULL);
1860
1861                 domain_update_iommu_cap(domain);
1862                 domain->iommu_did[iommu->seq_id] = 0;
1863         }
1864
1865         return count;
1866 }
1867
1868 static struct iova_domain reserved_iova_list;
1869 static struct lock_class_key reserved_rbtree_key;
1870
1871 static int dmar_init_reserved_ranges(void)
1872 {
1873         struct pci_dev *pdev = NULL;
1874         struct iova *iova;
1875         int i;
1876
1877         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1878                         DMA_32BIT_PFN);
1879
1880         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1881                 &reserved_rbtree_key);
1882
1883         /* IOAPIC ranges shouldn't be accessed by DMA */
1884         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1885                 IOVA_PFN(IOAPIC_RANGE_END));
1886         if (!iova) {
1887                 pr_err("Reserve IOAPIC range failed\n");
1888                 return -ENODEV;
1889         }
1890
1891         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1892         for_each_pci_dev(pdev) {
1893                 struct resource *r;
1894
1895                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1896                         r = &pdev->resource[i];
1897                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1898                                 continue;
1899                         iova = reserve_iova(&reserved_iova_list,
1900                                             IOVA_PFN(r->start),
1901                                             IOVA_PFN(r->end));
1902                         if (!iova) {
1903                                 pr_err("Reserve iova failed\n");
1904                                 return -ENODEV;
1905                         }
1906                 }
1907         }
1908         return 0;
1909 }
1910
1911 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1912 {
1913         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1914 }
1915
1916 static inline int guestwidth_to_adjustwidth(int gaw)
1917 {
1918         int agaw;
1919         int r = (gaw - 12) % 9;
1920
1921         if (r == 0)
1922                 agaw = gaw;
1923         else
1924                 agaw = gaw + 9 - r;
1925         if (agaw > 64)
1926                 agaw = 64;
1927         return agaw;
1928 }
1929
1930 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1931                        int guest_width)
1932 {
1933         int adjust_width, agaw;
1934         unsigned long sagaw;
1935
1936         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1937                         DMA_32BIT_PFN);
1938         domain_reserve_special_ranges(domain);
1939
1940         /* calculate AGAW */
1941         if (guest_width > cap_mgaw(iommu->cap))
1942                 guest_width = cap_mgaw(iommu->cap);
1943         domain->gaw = guest_width;
1944         adjust_width = guestwidth_to_adjustwidth(guest_width);
1945         agaw = width_to_agaw(adjust_width);
1946         sagaw = cap_sagaw(iommu->cap);
1947         if (!test_bit(agaw, &sagaw)) {
1948                 /* hardware doesn't support it, choose a bigger one */
1949                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1950                 agaw = find_next_bit(&sagaw, 5, agaw);
1951                 if (agaw >= 5)
1952                         return -ENODEV;
1953         }
1954         domain->agaw = agaw;
1955
1956         if (ecap_coherent(iommu->ecap))
1957                 domain->iommu_coherency = 1;
1958         else
1959                 domain->iommu_coherency = 0;
1960
1961         if (ecap_sc_support(iommu->ecap))
1962                 domain->iommu_snooping = 1;
1963         else
1964                 domain->iommu_snooping = 0;
1965
1966         if (intel_iommu_superpage)
1967                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1968         else
1969                 domain->iommu_superpage = 0;
1970
1971         domain->nid = iommu->node;
1972
1973         /* always allocate the top pgd */
1974         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1975         if (!domain->pgd)
1976                 return -ENOMEM;
1977         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1978         return 0;
1979 }
1980
1981 static void domain_exit(struct dmar_domain *domain)
1982 {
1983         struct page *freelist = NULL;
1984
1985         /* Domain 0 is reserved, so dont process it */
1986         if (!domain)
1987                 return;
1988
1989         /* Flush any lazy unmaps that may reference this domain */
1990         if (!intel_iommu_strict) {
1991                 int cpu;
1992
1993                 for_each_possible_cpu(cpu)
1994                         flush_unmaps_timeout(cpu);
1995         }
1996
1997         /* Remove associated devices and clear attached or cached domains */
1998         rcu_read_lock();
1999         domain_remove_dev_info(domain);
2000         rcu_read_unlock();
2001
2002         /* destroy iovas */
2003         put_iova_domain(&domain->iovad);
2004
2005         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2006
2007         dma_free_pagelist(freelist);
2008
2009         free_domain_mem(domain);
2010 }
2011
2012 static int domain_context_mapping_one(struct dmar_domain *domain,
2013                                       struct intel_iommu *iommu,
2014                                       u8 bus, u8 devfn)
2015 {
2016         u16 did = domain->iommu_did[iommu->seq_id];
2017         int translation = CONTEXT_TT_MULTI_LEVEL;
2018         struct device_domain_info *info = NULL;
2019         struct context_entry *context;
2020         unsigned long flags;
2021         struct dma_pte *pgd;
2022         int ret, agaw;
2023
2024         WARN_ON(did == 0);
2025
2026         if (hw_pass_through && domain_type_is_si(domain))
2027                 translation = CONTEXT_TT_PASS_THROUGH;
2028
2029         pr_debug("Set context mapping for %02x:%02x.%d\n",
2030                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2031
2032         BUG_ON(!domain->pgd);
2033
2034         spin_lock_irqsave(&device_domain_lock, flags);
2035         spin_lock(&iommu->lock);
2036
2037         ret = -ENOMEM;
2038         context = iommu_context_addr(iommu, bus, devfn, 1);
2039         if (!context)
2040                 goto out_unlock;
2041
2042         ret = 0;
2043         if (context_present(context))
2044                 goto out_unlock;
2045
2046         /*
2047          * For kdump cases, old valid entries may be cached due to the
2048          * in-flight DMA and copied pgtable, but there is no unmapping
2049          * behaviour for them, thus we need an explicit cache flush for
2050          * the newly-mapped device. For kdump, at this point, the device
2051          * is supposed to finish reset at its driver probe stage, so no
2052          * in-flight DMA will exist, and we don't need to worry anymore
2053          * hereafter.
2054          */
2055         if (context_copied(context)) {
2056                 u16 did_old = context_domain_id(context);
2057
2058                 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap))
2059                         iommu->flush.flush_context(iommu, did_old,
2060                                                    (((u16)bus) << 8) | devfn,
2061                                                    DMA_CCMD_MASK_NOBIT,
2062                                                    DMA_CCMD_DEVICE_INVL);
2063         }
2064
2065         pgd = domain->pgd;
2066
2067         context_clear_entry(context);
2068         context_set_domain_id(context, did);
2069
2070         /*
2071          * Skip top levels of page tables for iommu which has less agaw
2072          * than default.  Unnecessary for PT mode.
2073          */
2074         if (translation != CONTEXT_TT_PASS_THROUGH) {
2075                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2076                         ret = -ENOMEM;
2077                         pgd = phys_to_virt(dma_pte_addr(pgd));
2078                         if (!dma_pte_present(pgd))
2079                                 goto out_unlock;
2080                 }
2081
2082                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2083                 if (info && info->ats_supported)
2084                         translation = CONTEXT_TT_DEV_IOTLB;
2085                 else
2086                         translation = CONTEXT_TT_MULTI_LEVEL;
2087
2088                 context_set_address_root(context, virt_to_phys(pgd));
2089                 context_set_address_width(context, iommu->agaw);
2090         } else {
2091                 /*
2092                  * In pass through mode, AW must be programmed to
2093                  * indicate the largest AGAW value supported by
2094                  * hardware. And ASR is ignored by hardware.
2095                  */
2096                 context_set_address_width(context, iommu->msagaw);
2097         }
2098
2099         context_set_translation_type(context, translation);
2100         context_set_fault_enable(context);
2101         context_set_present(context);
2102         domain_flush_cache(domain, context, sizeof(*context));
2103
2104         /*
2105          * It's a non-present to present mapping. If hardware doesn't cache
2106          * non-present entry we only need to flush the write-buffer. If the
2107          * _does_ cache non-present entries, then it does so in the special
2108          * domain #0, which we have to flush:
2109          */
2110         if (cap_caching_mode(iommu->cap)) {
2111                 iommu->flush.flush_context(iommu, 0,
2112                                            (((u16)bus) << 8) | devfn,
2113                                            DMA_CCMD_MASK_NOBIT,
2114                                            DMA_CCMD_DEVICE_INVL);
2115                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2116         } else {
2117                 iommu_flush_write_buffer(iommu);
2118         }
2119         iommu_enable_dev_iotlb(info);
2120
2121         ret = 0;
2122
2123 out_unlock:
2124         spin_unlock(&iommu->lock);
2125         spin_unlock_irqrestore(&device_domain_lock, flags);
2126
2127         return ret;
2128 }
2129
2130 struct domain_context_mapping_data {
2131         struct dmar_domain *domain;
2132         struct intel_iommu *iommu;
2133 };
2134
2135 static int domain_context_mapping_cb(struct pci_dev *pdev,
2136                                      u16 alias, void *opaque)
2137 {
2138         struct domain_context_mapping_data *data = opaque;
2139
2140         return domain_context_mapping_one(data->domain, data->iommu,
2141                                           PCI_BUS_NUM(alias), alias & 0xff);
2142 }
2143
2144 static int
2145 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2146 {
2147         struct intel_iommu *iommu;
2148         u8 bus, devfn;
2149         struct domain_context_mapping_data data;
2150
2151         iommu = device_to_iommu(dev, &bus, &devfn);
2152         if (!iommu)
2153                 return -ENODEV;
2154
2155         if (!dev_is_pci(dev))
2156                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2157
2158         data.domain = domain;
2159         data.iommu = iommu;
2160
2161         return pci_for_each_dma_alias(to_pci_dev(dev),
2162                                       &domain_context_mapping_cb, &data);
2163 }
2164
2165 static int domain_context_mapped_cb(struct pci_dev *pdev,
2166                                     u16 alias, void *opaque)
2167 {
2168         struct intel_iommu *iommu = opaque;
2169
2170         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2171 }
2172
2173 static int domain_context_mapped(struct device *dev)
2174 {
2175         struct intel_iommu *iommu;
2176         u8 bus, devfn;
2177
2178         iommu = device_to_iommu(dev, &bus, &devfn);
2179         if (!iommu)
2180                 return -ENODEV;
2181
2182         if (!dev_is_pci(dev))
2183                 return device_context_mapped(iommu, bus, devfn);
2184
2185         return !pci_for_each_dma_alias(to_pci_dev(dev),
2186                                        domain_context_mapped_cb, iommu);
2187 }
2188
2189 /* Returns a number of VTD pages, but aligned to MM page size */
2190 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2191                                             size_t size)
2192 {
2193         host_addr &= ~PAGE_MASK;
2194         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2195 }
2196
2197 /* Return largest possible superpage level for a given mapping */
2198 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2199                                           unsigned long iov_pfn,
2200                                           unsigned long phy_pfn,
2201                                           unsigned long pages)
2202 {
2203         int support, level = 1;
2204         unsigned long pfnmerge;
2205
2206         support = domain->iommu_superpage;
2207
2208         /* To use a large page, the virtual *and* physical addresses
2209            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2210            of them will mean we have to use smaller pages. So just
2211            merge them and check both at once. */
2212         pfnmerge = iov_pfn | phy_pfn;
2213
2214         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2215                 pages >>= VTD_STRIDE_SHIFT;
2216                 if (!pages)
2217                         break;
2218                 pfnmerge >>= VTD_STRIDE_SHIFT;
2219                 level++;
2220                 support--;
2221         }
2222         return level;
2223 }
2224
2225 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2226                             struct scatterlist *sg, unsigned long phys_pfn,
2227                             unsigned long nr_pages, int prot)
2228 {
2229         struct dma_pte *first_pte = NULL, *pte = NULL;
2230         phys_addr_t uninitialized_var(pteval);
2231         unsigned long sg_res = 0;
2232         unsigned int largepage_lvl = 0;
2233         unsigned long lvl_pages = 0;
2234
2235         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2236
2237         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2238                 return -EINVAL;
2239
2240         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2241
2242         if (!sg) {
2243                 sg_res = nr_pages;
2244                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2245         }
2246
2247         while (nr_pages > 0) {
2248                 uint64_t tmp;
2249
2250                 if (!sg_res) {
2251                         sg_res = aligned_nrpages(sg->offset, sg->length);
2252                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2253                         sg->dma_length = sg->length;
2254                         pteval = page_to_phys(sg_page(sg)) | prot;
2255                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2256                 }
2257
2258                 if (!pte) {
2259                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2260
2261                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2262                         if (!pte)
2263                                 return -ENOMEM;
2264                         /* It is large page*/
2265                         if (largepage_lvl > 1) {
2266                                 unsigned long nr_superpages, end_pfn;
2267
2268                                 pteval |= DMA_PTE_LARGE_PAGE;
2269                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2270
2271                                 nr_superpages = sg_res / lvl_pages;
2272                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2273
2274                                 /*
2275                                  * Ensure that old small page tables are
2276                                  * removed to make room for superpage(s).
2277                                  */
2278                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2279                         } else {
2280                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2281                         }
2282
2283                 }
2284                 /* We don't need lock here, nobody else
2285                  * touches the iova range
2286                  */
2287                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2288                 if (tmp) {
2289                         static int dumps = 5;
2290                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2291                                 iov_pfn, tmp, (unsigned long long)pteval);
2292                         if (dumps) {
2293                                 dumps--;
2294                                 debug_dma_dump_mappings(NULL);
2295                         }
2296                         WARN_ON(1);
2297                 }
2298
2299                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2300
2301                 BUG_ON(nr_pages < lvl_pages);
2302                 BUG_ON(sg_res < lvl_pages);
2303
2304                 nr_pages -= lvl_pages;
2305                 iov_pfn += lvl_pages;
2306                 phys_pfn += lvl_pages;
2307                 pteval += lvl_pages * VTD_PAGE_SIZE;
2308                 sg_res -= lvl_pages;
2309
2310                 /* If the next PTE would be the first in a new page, then we
2311                    need to flush the cache on the entries we've just written.
2312                    And then we'll need to recalculate 'pte', so clear it and
2313                    let it get set again in the if (!pte) block above.
2314
2315                    If we're done (!nr_pages) we need to flush the cache too.
2316
2317                    Also if we've been setting superpages, we may need to
2318                    recalculate 'pte' and switch back to smaller pages for the
2319                    end of the mapping, if the trailing size is not enough to
2320                    use another superpage (i.e. sg_res < lvl_pages). */
2321                 pte++;
2322                 if (!nr_pages || first_pte_in_page(pte) ||
2323                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2324                         domain_flush_cache(domain, first_pte,
2325                                            (void *)pte - (void *)first_pte);
2326                         pte = NULL;
2327                 }
2328
2329                 if (!sg_res && nr_pages)
2330                         sg = sg_next(sg);
2331         }
2332         return 0;
2333 }
2334
2335 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2336                                     struct scatterlist *sg, unsigned long nr_pages,
2337                                     int prot)
2338 {
2339         return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2340 }
2341
2342 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2343                                      unsigned long phys_pfn, unsigned long nr_pages,
2344                                      int prot)
2345 {
2346         return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2347 }
2348
2349 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2350 {
2351         if (!iommu)
2352                 return;
2353
2354         clear_context_table(iommu, bus, devfn);
2355         iommu->flush.flush_context(iommu, 0, 0, 0,
2356                                            DMA_CCMD_GLOBAL_INVL);
2357         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2358 }
2359
2360 static inline void unlink_domain_info(struct device_domain_info *info)
2361 {
2362         assert_spin_locked(&device_domain_lock);
2363         list_del(&info->link);
2364         list_del(&info->global);
2365         if (info->dev)
2366                 info->dev->archdata.iommu = NULL;
2367 }
2368
2369 static void domain_remove_dev_info(struct dmar_domain *domain)
2370 {
2371         struct device_domain_info *info, *tmp;
2372         unsigned long flags;
2373
2374         spin_lock_irqsave(&device_domain_lock, flags);
2375         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2376                 __dmar_remove_one_dev_info(info);
2377         spin_unlock_irqrestore(&device_domain_lock, flags);
2378 }
2379
2380 /*
2381  * find_domain
2382  * Note: we use struct device->archdata.iommu stores the info
2383  */
2384 static struct dmar_domain *find_domain(struct device *dev)
2385 {
2386         struct device_domain_info *info;
2387
2388         /* No lock here, assumes no domain exit in normal case */
2389         info = dev->archdata.iommu;
2390         if (likely(info))
2391                 return info->domain;
2392         return NULL;
2393 }
2394
2395 static inline struct device_domain_info *
2396 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2397 {
2398         struct device_domain_info *info;
2399
2400         list_for_each_entry(info, &device_domain_list, global)
2401                 if (info->iommu->segment == segment && info->bus == bus &&
2402                     info->devfn == devfn)
2403                         return info;
2404
2405         return NULL;
2406 }
2407
2408 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2409                                                     int bus, int devfn,
2410                                                     struct device *dev,
2411                                                     struct dmar_domain *domain)
2412 {
2413         struct dmar_domain *found = NULL;
2414         struct device_domain_info *info;
2415         unsigned long flags;
2416         int ret;
2417
2418         info = alloc_devinfo_mem();
2419         if (!info)
2420                 return NULL;
2421
2422         info->bus = bus;
2423         info->devfn = devfn;
2424         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2425         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2426         info->ats_qdep = 0;
2427         info->dev = dev;
2428         info->domain = domain;
2429         info->iommu = iommu;
2430
2431         if (dev && dev_is_pci(dev)) {
2432                 struct pci_dev *pdev = to_pci_dev(info->dev);
2433
2434                 if (ecap_dev_iotlb_support(iommu->ecap) &&
2435                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2436                     dmar_find_matched_atsr_unit(pdev))
2437                         info->ats_supported = 1;
2438
2439                 if (ecs_enabled(iommu)) {
2440                         if (pasid_enabled(iommu)) {
2441                                 int features = pci_pasid_features(pdev);
2442                                 if (features >= 0)
2443                                         info->pasid_supported = features | 1;
2444                         }
2445
2446                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2447                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2448                                 info->pri_supported = 1;
2449                 }
2450         }
2451
2452         spin_lock_irqsave(&device_domain_lock, flags);
2453         if (dev)
2454                 found = find_domain(dev);
2455
2456         if (!found) {
2457                 struct device_domain_info *info2;
2458                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2459                 if (info2) {
2460                         found      = info2->domain;
2461                         info2->dev = dev;
2462                 }
2463         }
2464
2465         if (found) {
2466                 spin_unlock_irqrestore(&device_domain_lock, flags);
2467                 free_devinfo_mem(info);
2468                 /* Caller must free the original domain */
2469                 return found;
2470         }
2471
2472         spin_lock(&iommu->lock);
2473         ret = domain_attach_iommu(domain, iommu);
2474         spin_unlock(&iommu->lock);
2475
2476         if (ret) {
2477                 spin_unlock_irqrestore(&device_domain_lock, flags);
2478                 free_devinfo_mem(info);
2479                 return NULL;
2480         }
2481
2482         list_add(&info->link, &domain->devices);
2483         list_add(&info->global, &device_domain_list);
2484         if (dev)
2485                 dev->archdata.iommu = info;
2486         spin_unlock_irqrestore(&device_domain_lock, flags);
2487
2488         if (dev && domain_context_mapping(domain, dev)) {
2489                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2490                 dmar_remove_one_dev_info(domain, dev);
2491                 return NULL;
2492         }
2493
2494         return domain;
2495 }
2496
2497 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2498 {
2499         *(u16 *)opaque = alias;
2500         return 0;
2501 }
2502
2503 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2504 {
2505         struct device_domain_info *info = NULL;
2506         struct dmar_domain *domain = NULL;
2507         struct intel_iommu *iommu;
2508         u16 req_id, dma_alias;
2509         unsigned long flags;
2510         u8 bus, devfn;
2511
2512         iommu = device_to_iommu(dev, &bus, &devfn);
2513         if (!iommu)
2514                 return NULL;
2515
2516         req_id = ((u16)bus << 8) | devfn;
2517
2518         if (dev_is_pci(dev)) {
2519                 struct pci_dev *pdev = to_pci_dev(dev);
2520
2521                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2522
2523                 spin_lock_irqsave(&device_domain_lock, flags);
2524                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2525                                                       PCI_BUS_NUM(dma_alias),
2526                                                       dma_alias & 0xff);
2527                 if (info) {
2528                         iommu = info->iommu;
2529                         domain = info->domain;
2530                 }
2531                 spin_unlock_irqrestore(&device_domain_lock, flags);
2532
2533                 /* DMA alias already has a domain, use it */
2534                 if (info)
2535                         goto out;
2536         }
2537
2538         /* Allocate and initialize new domain for the device */
2539         domain = alloc_domain(0);
2540         if (!domain)
2541                 return NULL;
2542         if (domain_init(domain, iommu, gaw)) {
2543                 domain_exit(domain);
2544                 return NULL;
2545         }
2546
2547 out:
2548
2549         return domain;
2550 }
2551
2552 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2553                                               struct dmar_domain *domain)
2554 {
2555         struct intel_iommu *iommu;
2556         struct dmar_domain *tmp;
2557         u16 req_id, dma_alias;
2558         u8 bus, devfn;
2559
2560         iommu = device_to_iommu(dev, &bus, &devfn);
2561         if (!iommu)
2562                 return NULL;
2563
2564         req_id = ((u16)bus << 8) | devfn;
2565
2566         if (dev_is_pci(dev)) {
2567                 struct pci_dev *pdev = to_pci_dev(dev);
2568
2569                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2570
2571                 /* register PCI DMA alias device */
2572                 if (req_id != dma_alias) {
2573                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2574                                         dma_alias & 0xff, NULL, domain);
2575
2576                         if (!tmp || tmp != domain)
2577                                 return tmp;
2578                 }
2579         }
2580
2581         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2582         if (!tmp || tmp != domain)
2583                 return tmp;
2584
2585         return domain;
2586 }
2587
2588 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2589 {
2590         struct dmar_domain *domain, *tmp;
2591
2592         domain = find_domain(dev);
2593         if (domain)
2594                 goto out;
2595
2596         domain = find_or_alloc_domain(dev, gaw);
2597         if (!domain)
2598                 goto out;
2599
2600         tmp = set_domain_for_dev(dev, domain);
2601         if (!tmp || domain != tmp) {
2602                 domain_exit(domain);
2603                 domain = tmp;
2604         }
2605
2606 out:
2607
2608         return domain;
2609 }
2610
2611 static int iommu_domain_identity_map(struct dmar_domain *domain,
2612                                      unsigned long long start,
2613                                      unsigned long long end)
2614 {
2615         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2616         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2617
2618         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2619                           dma_to_mm_pfn(last_vpfn))) {
2620                 pr_err("Reserving iova failed\n");
2621                 return -ENOMEM;
2622         }
2623
2624         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2625         /*
2626          * RMRR range might have overlap with physical memory range,
2627          * clear it first
2628          */
2629         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2630
2631         return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2632                                   last_vpfn - first_vpfn + 1,
2633                                   DMA_PTE_READ|DMA_PTE_WRITE);
2634 }
2635
2636 static int domain_prepare_identity_map(struct device *dev,
2637                                        struct dmar_domain *domain,
2638                                        unsigned long long start,
2639                                        unsigned long long end)
2640 {
2641         /* For _hardware_ passthrough, don't bother. But for software
2642            passthrough, we do it anyway -- it may indicate a memory
2643            range which is reserved in E820, so which didn't get set
2644            up to start with in si_domain */
2645         if (domain == si_domain && hw_pass_through) {
2646                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2647                         dev_name(dev), start, end);
2648                 return 0;
2649         }
2650
2651         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2652                 dev_name(dev), start, end);
2653
2654         if (end < start) {
2655                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2656                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2657                         dmi_get_system_info(DMI_BIOS_VENDOR),
2658                         dmi_get_system_info(DMI_BIOS_VERSION),
2659                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2660                 return -EIO;
2661         }
2662
2663         if (end >> agaw_to_width(domain->agaw)) {
2664                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2665                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2666                      agaw_to_width(domain->agaw),
2667                      dmi_get_system_info(DMI_BIOS_VENDOR),
2668                      dmi_get_system_info(DMI_BIOS_VERSION),
2669                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2670                 return -EIO;
2671         }
2672
2673         return iommu_domain_identity_map(domain, start, end);
2674 }
2675
2676 static int iommu_prepare_identity_map(struct device *dev,
2677                                       unsigned long long start,
2678                                       unsigned long long end)
2679 {
2680         struct dmar_domain *domain;
2681         int ret;
2682
2683         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2684         if (!domain)
2685                 return -ENOMEM;
2686
2687         ret = domain_prepare_identity_map(dev, domain, start, end);
2688         if (ret)
2689                 domain_exit(domain);
2690
2691         return ret;
2692 }
2693
2694 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2695                                          struct device *dev)
2696 {
2697         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2698                 return 0;
2699         return iommu_prepare_identity_map(dev, rmrr->base_address,
2700                                           rmrr->end_address);
2701 }
2702
2703 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2704 static inline void iommu_prepare_isa(void)
2705 {
2706         struct pci_dev *pdev;
2707         int ret;
2708
2709         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2710         if (!pdev)
2711                 return;
2712
2713         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2714         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2715
2716         if (ret)
2717                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2718
2719         pci_dev_put(pdev);
2720 }
2721 #else
2722 static inline void iommu_prepare_isa(void)
2723 {
2724         return;
2725 }
2726 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2727
2728 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2729
2730 static int __init si_domain_init(int hw)
2731 {
2732         int nid, ret = 0;
2733
2734         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2735         if (!si_domain)
2736                 return -EFAULT;
2737
2738         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2739                 domain_exit(si_domain);
2740                 return -EFAULT;
2741         }
2742
2743         pr_debug("Identity mapping domain allocated\n");
2744
2745         if (hw)
2746                 return 0;
2747
2748         for_each_online_node(nid) {
2749                 unsigned long start_pfn, end_pfn;
2750                 int i;
2751
2752                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2753                         ret = iommu_domain_identity_map(si_domain,
2754                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2755                         if (ret)
2756                                 return ret;
2757                 }
2758         }
2759
2760         return 0;
2761 }
2762
2763 static int identity_mapping(struct device *dev)
2764 {
2765         struct device_domain_info *info;
2766
2767         if (likely(!iommu_identity_mapping))
2768                 return 0;
2769
2770         info = dev->archdata.iommu;
2771         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2772                 return (info->domain == si_domain);
2773
2774         return 0;
2775 }
2776
2777 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2778 {
2779         struct dmar_domain *ndomain;
2780         struct intel_iommu *iommu;
2781         u8 bus, devfn;
2782
2783         iommu = device_to_iommu(dev, &bus, &devfn);
2784         if (!iommu)
2785                 return -ENODEV;
2786
2787         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2788         if (ndomain != domain)
2789                 return -EBUSY;
2790
2791         return 0;
2792 }
2793
2794 static bool device_has_rmrr(struct device *dev)
2795 {
2796         struct dmar_rmrr_unit *rmrr;
2797         struct device *tmp;
2798         int i;
2799
2800         rcu_read_lock();
2801         for_each_rmrr_units(rmrr) {
2802                 /*
2803                  * Return TRUE if this RMRR contains the device that
2804                  * is passed in.
2805                  */
2806                 for_each_active_dev_scope(rmrr->devices,
2807                                           rmrr->devices_cnt, i, tmp)
2808                         if (tmp == dev) {
2809                                 rcu_read_unlock();
2810                                 return true;
2811                         }
2812         }
2813         rcu_read_unlock();
2814         return false;
2815 }
2816
2817 /*
2818  * There are a couple cases where we need to restrict the functionality of
2819  * devices associated with RMRRs.  The first is when evaluating a device for
2820  * identity mapping because problems exist when devices are moved in and out
2821  * of domains and their respective RMRR information is lost.  This means that
2822  * a device with associated RMRRs will never be in a "passthrough" domain.
2823  * The second is use of the device through the IOMMU API.  This interface
2824  * expects to have full control of the IOVA space for the device.  We cannot
2825  * satisfy both the requirement that RMRR access is maintained and have an
2826  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2827  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2828  * We therefore prevent devices associated with an RMRR from participating in
2829  * the IOMMU API, which eliminates them from device assignment.
2830  *
2831  * In both cases we assume that PCI USB devices with RMRRs have them largely
2832  * for historical reasons and that the RMRR space is not actively used post
2833  * boot.  This exclusion may change if vendors begin to abuse it.
2834  *
2835  * The same exception is made for graphics devices, with the requirement that
2836  * any use of the RMRR regions will be torn down before assigning the device
2837  * to a guest.
2838  */
2839 static bool device_is_rmrr_locked(struct device *dev)
2840 {
2841         if (!device_has_rmrr(dev))
2842                 return false;
2843
2844         if (dev_is_pci(dev)) {
2845                 struct pci_dev *pdev = to_pci_dev(dev);
2846
2847                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2848                         return false;
2849         }
2850
2851         return true;
2852 }
2853
2854 static int iommu_should_identity_map(struct device *dev, int startup)
2855 {
2856
2857         if (dev_is_pci(dev)) {
2858                 struct pci_dev *pdev = to_pci_dev(dev);
2859
2860                 if (device_is_rmrr_locked(dev))
2861                         return 0;
2862
2863                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2864                         return 1;
2865
2866                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2867                         return 1;
2868
2869                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2870                         return 0;
2871
2872                 /*
2873                  * We want to start off with all devices in the 1:1 domain, and
2874                  * take them out later if we find they can't access all of memory.
2875                  *
2876                  * However, we can't do this for PCI devices behind bridges,
2877                  * because all PCI devices behind the same bridge will end up
2878                  * with the same source-id on their transactions.
2879                  *
2880                  * Practically speaking, we can't change things around for these
2881                  * devices at run-time, because we can't be sure there'll be no
2882                  * DMA transactions in flight for any of their siblings.
2883                  *
2884                  * So PCI devices (unless they're on the root bus) as well as
2885                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2886                  * the 1:1 domain, just in _case_ one of their siblings turns out
2887                  * not to be able to map all of memory.
2888                  */
2889                 if (!pci_is_pcie(pdev)) {
2890                         if (!pci_is_root_bus(pdev->bus))
2891                                 return 0;
2892                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2893                                 return 0;
2894                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2895                         return 0;
2896         } else {
2897                 if (device_has_rmrr(dev))
2898                         return 0;
2899         }
2900
2901         /*
2902          * At boot time, we don't yet know if devices will be 64-bit capable.
2903          * Assume that they will — if they turn out not to be, then we can
2904          * take them out of the 1:1 domain later.
2905          */
2906         if (!startup) {
2907                 /*
2908                  * If the device's dma_mask is less than the system's memory
2909                  * size then this is not a candidate for identity mapping.
2910                  */
2911                 u64 dma_mask = *dev->dma_mask;
2912
2913                 if (dev->coherent_dma_mask &&
2914                     dev->coherent_dma_mask < dma_mask)
2915                         dma_mask = dev->coherent_dma_mask;
2916
2917                 return dma_mask >= dma_get_required_mask(dev);
2918         }
2919
2920         return 1;
2921 }
2922
2923 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2924 {
2925         int ret;
2926
2927         if (!iommu_should_identity_map(dev, 1))
2928                 return 0;
2929
2930         ret = domain_add_dev_info(si_domain, dev);
2931         if (!ret)
2932                 pr_info("%s identity mapping for device %s\n",
2933                         hw ? "Hardware" : "Software", dev_name(dev));
2934         else if (ret == -ENODEV)
2935                 /* device not associated with an iommu */
2936                 ret = 0;
2937
2938         return ret;
2939 }
2940
2941
2942 static int __init iommu_prepare_static_identity_mapping(int hw)
2943 {
2944         struct pci_dev *pdev = NULL;
2945         struct dmar_drhd_unit *drhd;
2946         struct intel_iommu *iommu;
2947         struct device *dev;
2948         int i;
2949         int ret = 0;
2950
2951         for_each_pci_dev(pdev) {
2952                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2953                 if (ret)
2954                         return ret;
2955         }
2956
2957         for_each_active_iommu(iommu, drhd)
2958                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2959                         struct acpi_device_physical_node *pn;
2960                         struct acpi_device *adev;
2961
2962                         if (dev->bus != &acpi_bus_type)
2963                                 continue;
2964
2965                         adev= to_acpi_device(dev);
2966                         mutex_lock(&adev->physical_node_lock);
2967                         list_for_each_entry(pn, &adev->physical_node_list, node) {
2968                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2969                                 if (ret)
2970                                         break;
2971                         }
2972                         mutex_unlock(&adev->physical_node_lock);
2973                         if (ret)
2974                                 return ret;
2975                 }
2976
2977         return 0;
2978 }
2979
2980 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2981 {
2982         /*
2983          * Start from the sane iommu hardware state.
2984          * If the queued invalidation is already initialized by us
2985          * (for example, while enabling interrupt-remapping) then
2986          * we got the things already rolling from a sane state.
2987          */
2988         if (!iommu->qi) {
2989                 /*
2990                  * Clear any previous faults.
2991                  */
2992                 dmar_fault(-1, iommu);
2993                 /*
2994                  * Disable queued invalidation if supported and already enabled
2995                  * before OS handover.
2996                  */
2997                 dmar_disable_qi(iommu);
2998         }
2999
3000         if (dmar_enable_qi(iommu)) {
3001                 /*
3002                  * Queued Invalidate not enabled, use Register Based Invalidate
3003                  */
3004                 iommu->flush.flush_context = __iommu_flush_context;
3005                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3006                 pr_info("%s: Using Register based invalidation\n",
3007                         iommu->name);
3008         } else {
3009                 iommu->flush.flush_context = qi_flush_context;
3010                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3011                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3012         }
3013 }
3014
3015 static int copy_context_table(struct intel_iommu *iommu,
3016                               struct root_entry *old_re,
3017                               struct context_entry **tbl,
3018                               int bus, bool ext)
3019 {
3020         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3021         struct context_entry *new_ce = NULL, ce;
3022         struct context_entry *old_ce = NULL;
3023         struct root_entry re;
3024         phys_addr_t old_ce_phys;
3025
3026         tbl_idx = ext ? bus * 2 : bus;
3027         memcpy(&re, old_re, sizeof(re));
3028
3029         for (devfn = 0; devfn < 256; devfn++) {
3030                 /* First calculate the correct index */
3031                 idx = (ext ? devfn * 2 : devfn) % 256;
3032
3033                 if (idx == 0) {
3034                         /* First save what we may have and clean up */
3035                         if (new_ce) {
3036                                 tbl[tbl_idx] = new_ce;
3037                                 __iommu_flush_cache(iommu, new_ce,
3038                                                     VTD_PAGE_SIZE);
3039                                 pos = 1;
3040                         }
3041
3042                         if (old_ce)
3043                                 iounmap(old_ce);
3044
3045                         ret = 0;
3046                         if (devfn < 0x80)
3047                                 old_ce_phys = root_entry_lctp(&re);
3048                         else
3049                                 old_ce_phys = root_entry_uctp(&re);
3050
3051                         if (!old_ce_phys) {
3052                                 if (ext && devfn == 0) {
3053                                         /* No LCTP, try UCTP */
3054                                         devfn = 0x7f;
3055                                         continue;
3056                                 } else {
3057                                         goto out;
3058                                 }
3059                         }
3060
3061                         ret = -ENOMEM;
3062                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3063                                         MEMREMAP_WB);
3064                         if (!old_ce)
3065                                 goto out;
3066
3067                         new_ce = alloc_pgtable_page(iommu->node);
3068                         if (!new_ce)
3069                                 goto out_unmap;
3070
3071                         ret = 0;
3072                 }
3073
3074                 /* Now copy the context entry */
3075                 memcpy(&ce, old_ce + idx, sizeof(ce));
3076
3077                 if (!__context_present(&ce))
3078                         continue;
3079
3080                 did = context_domain_id(&ce);
3081                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3082                         set_bit(did, iommu->domain_ids);
3083
3084                 /*
3085                  * We need a marker for copied context entries. This
3086                  * marker needs to work for the old format as well as
3087                  * for extended context entries.
3088                  *
3089                  * Bit 67 of the context entry is used. In the old
3090                  * format this bit is available to software, in the
3091                  * extended format it is the PGE bit, but PGE is ignored
3092                  * by HW if PASIDs are disabled (and thus still
3093                  * available).
3094                  *
3095                  * So disable PASIDs first and then mark the entry
3096                  * copied. This means that we don't copy PASID
3097                  * translations from the old kernel, but this is fine as
3098                  * faults there are not fatal.
3099                  */
3100                 context_clear_pasid_enable(&ce);
3101                 context_set_copied(&ce);
3102
3103                 new_ce[idx] = ce;
3104         }
3105
3106         tbl[tbl_idx + pos] = new_ce;
3107
3108         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3109
3110 out_unmap:
3111         memunmap(old_ce);
3112
3113 out:
3114         return ret;
3115 }
3116
3117 static int copy_translation_tables(struct intel_iommu *iommu)
3118 {
3119         struct context_entry **ctxt_tbls;
3120         struct root_entry *old_rt;
3121         phys_addr_t old_rt_phys;
3122         int ctxt_table_entries;
3123         unsigned long flags;
3124         u64 rtaddr_reg;
3125         int bus, ret;
3126         bool new_ext, ext;
3127
3128         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3129         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3130         new_ext    = !!ecap_ecs(iommu->ecap);
3131
3132         /*
3133          * The RTT bit can only be changed when translation is disabled,
3134          * but disabling translation means to open a window for data
3135          * corruption. So bail out and don't copy anything if we would
3136          * have to change the bit.
3137          */
3138         if (new_ext != ext)
3139                 return -EINVAL;
3140
3141         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3142         if (!old_rt_phys)
3143                 return -EINVAL;
3144
3145         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3146         if (!old_rt)
3147                 return -ENOMEM;
3148
3149         /* This is too big for the stack - allocate it from slab */
3150         ctxt_table_entries = ext ? 512 : 256;
3151         ret = -ENOMEM;
3152         ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3153         if (!ctxt_tbls)
3154                 goto out_unmap;
3155
3156         for (bus = 0; bus < 256; bus++) {
3157                 ret = copy_context_table(iommu, &old_rt[bus],
3158                                          ctxt_tbls, bus, ext);
3159                 if (ret) {
3160                         pr_err("%s: Failed to copy context table for bus %d\n",
3161                                 iommu->name, bus);
3162                         continue;
3163                 }
3164         }
3165
3166         spin_lock_irqsave(&iommu->lock, flags);
3167
3168         /* Context tables are copied, now write them to the root_entry table */
3169         for (bus = 0; bus < 256; bus++) {
3170                 int idx = ext ? bus * 2 : bus;
3171                 u64 val;
3172
3173                 if (ctxt_tbls[idx]) {
3174                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3175                         iommu->root_entry[bus].lo = val;
3176                 }
3177
3178                 if (!ext || !ctxt_tbls[idx + 1])
3179                         continue;
3180
3181                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3182                 iommu->root_entry[bus].hi = val;
3183         }
3184
3185         spin_unlock_irqrestore(&iommu->lock, flags);
3186
3187         kfree(ctxt_tbls);
3188
3189         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3190
3191         ret = 0;
3192
3193 out_unmap:
3194         memunmap(old_rt);
3195
3196         return ret;
3197 }
3198
3199 static int __init init_dmars(void)
3200 {
3201         struct dmar_drhd_unit *drhd;
3202         struct dmar_rmrr_unit *rmrr;
3203         bool copied_tables = false;
3204         struct device *dev;
3205         struct intel_iommu *iommu;
3206         int i, ret, cpu;
3207
3208         /*
3209          * for each drhd
3210          *    allocate root
3211          *    initialize and program root entry to not present
3212          * endfor
3213          */
3214         for_each_drhd_unit(drhd) {
3215                 /*
3216                  * lock not needed as this is only incremented in the single
3217                  * threaded kernel __init code path all other access are read
3218                  * only
3219                  */
3220                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3221                         g_num_of_iommus++;
3222                         continue;
3223                 }
3224                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3225         }
3226
3227         /* Preallocate enough resources for IOMMU hot-addition */
3228         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3229                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3230
3231         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3232                         GFP_KERNEL);
3233         if (!g_iommus) {
3234                 pr_err("Allocating global iommu array failed\n");
3235                 ret = -ENOMEM;
3236                 goto error;
3237         }
3238
3239         for_each_possible_cpu(cpu) {
3240                 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3241                                                               cpu);
3242
3243                 dfd->tables = kzalloc(g_num_of_iommus *
3244                                       sizeof(struct deferred_flush_table),
3245                                       GFP_KERNEL);
3246                 if (!dfd->tables) {
3247                         ret = -ENOMEM;
3248                         goto free_g_iommus;
3249                 }
3250
3251                 spin_lock_init(&dfd->lock);
3252                 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3253         }
3254
3255         for_each_active_iommu(iommu, drhd) {
3256                 g_iommus[iommu->seq_id] = iommu;
3257
3258                 intel_iommu_init_qi(iommu);
3259
3260                 ret = iommu_init_domains(iommu);
3261                 if (ret)
3262                         goto free_iommu;
3263
3264                 init_translation_status(iommu);
3265
3266                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3267                         iommu_disable_translation(iommu);
3268                         clear_translation_pre_enabled(iommu);
3269                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3270                                 iommu->name);
3271                 }
3272
3273                 /*
3274                  * TBD:
3275                  * we could share the same root & context tables
3276                  * among all IOMMU's. Need to Split it later.
3277                  */
3278                 ret = iommu_alloc_root_entry(iommu);
3279                 if (ret)
3280                         goto free_iommu;
3281
3282                 if (translation_pre_enabled(iommu)) {
3283                         pr_info("Translation already enabled - trying to copy translation structures\n");
3284
3285                         ret = copy_translation_tables(iommu);
3286                         if (ret) {
3287                                 /*
3288                                  * We found the IOMMU with translation
3289                                  * enabled - but failed to copy over the
3290                                  * old root-entry table. Try to proceed
3291                                  * by disabling translation now and
3292                                  * allocating a clean root-entry table.
3293                                  * This might cause DMAR faults, but
3294                                  * probably the dump will still succeed.
3295                                  */
3296                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3297                                        iommu->name);
3298                                 iommu_disable_translation(iommu);
3299                                 clear_translation_pre_enabled(iommu);
3300                         } else {
3301                                 pr_info("Copied translation tables from previous kernel for %s\n",
3302                                         iommu->name);
3303                                 copied_tables = true;
3304                         }
3305                 }
3306
3307                 if (!ecap_pass_through(iommu->ecap))
3308                         hw_pass_through = 0;
3309 #ifdef CONFIG_INTEL_IOMMU_SVM
3310                 if (pasid_enabled(iommu))
3311                         intel_svm_alloc_pasid_tables(iommu);
3312 #endif
3313         }
3314
3315         /*
3316          * Now that qi is enabled on all iommus, set the root entry and flush
3317          * caches. This is required on some Intel X58 chipsets, otherwise the
3318          * flush_context function will loop forever and the boot hangs.
3319          */
3320         for_each_active_iommu(iommu, drhd) {
3321                 iommu_flush_write_buffer(iommu);
3322                 iommu_set_root_entry(iommu);
3323                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3324                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3325         }
3326
3327         if (iommu_pass_through)
3328                 iommu_identity_mapping |= IDENTMAP_ALL;
3329
3330 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3331         iommu_identity_mapping |= IDENTMAP_GFX;
3332 #endif
3333
3334         check_tylersburg_isoch();
3335
3336         if (iommu_identity_mapping) {
3337                 ret = si_domain_init(hw_pass_through);
3338                 if (ret)
3339                         goto free_iommu;
3340         }
3341
3342
3343         /*
3344          * If we copied translations from a previous kernel in the kdump
3345          * case, we can not assign the devices to domains now, as that
3346          * would eliminate the old mappings. So skip this part and defer
3347          * the assignment to device driver initialization time.
3348          */
3349         if (copied_tables)
3350                 goto domains_done;
3351
3352         /*
3353          * If pass through is not set or not enabled, setup context entries for
3354          * identity mappings for rmrr, gfx, and isa and may fall back to static
3355          * identity mapping if iommu_identity_mapping is set.
3356          */
3357         if (iommu_identity_mapping) {
3358                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3359                 if (ret) {
3360                         pr_crit("Failed to setup IOMMU pass-through\n");
3361                         goto free_iommu;
3362                 }
3363         }
3364         /*
3365          * For each rmrr
3366          *   for each dev attached to rmrr
3367          *   do
3368          *     locate drhd for dev, alloc domain for dev
3369          *     allocate free domain
3370          *     allocate page table entries for rmrr
3371          *     if context not allocated for bus
3372          *           allocate and init context
3373          *           set present in root table for this bus
3374          *     init context with domain, translation etc
3375          *    endfor
3376          * endfor
3377          */
3378         pr_info("Setting RMRR:\n");
3379         for_each_rmrr_units(rmrr) {
3380                 /* some BIOS lists non-exist devices in DMAR table. */
3381                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3382                                           i, dev) {
3383                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3384                         if (ret)
3385                                 pr_err("Mapping reserved region failed\n");
3386                 }
3387         }
3388
3389         iommu_prepare_isa();
3390
3391 domains_done:
3392
3393         /*
3394          * for each drhd
3395          *   enable fault log
3396          *   global invalidate context cache
3397          *   global invalidate iotlb
3398          *   enable translation
3399          */
3400         for_each_iommu(iommu, drhd) {
3401                 if (drhd->ignored) {
3402                         /*
3403                          * we always have to disable PMRs or DMA may fail on
3404                          * this device
3405                          */
3406                         if (force_on)
3407                                 iommu_disable_protect_mem_regions(iommu);
3408                         continue;
3409                 }
3410
3411                 iommu_flush_write_buffer(iommu);
3412
3413 #ifdef CONFIG_INTEL_IOMMU_SVM
3414                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3415                         ret = intel_svm_enable_prq(iommu);
3416                         if (ret)
3417                                 goto free_iommu;
3418                 }
3419 #endif
3420                 ret = dmar_set_interrupt(iommu);
3421                 if (ret)
3422                         goto free_iommu;
3423
3424                 if (!translation_pre_enabled(iommu))
3425                         iommu_enable_translation(iommu);
3426
3427                 iommu_disable_protect_mem_regions(iommu);
3428         }
3429
3430         return 0;
3431
3432 free_iommu:
3433         for_each_active_iommu(iommu, drhd) {
3434                 disable_dmar_iommu(iommu);
3435                 free_dmar_iommu(iommu);
3436         }
3437 free_g_iommus:
3438         for_each_possible_cpu(cpu)
3439                 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3440         kfree(g_iommus);
3441 error:
3442         return ret;
3443 }
3444
3445 /* This takes a number of _MM_ pages, not VTD pages */
3446 static unsigned long intel_alloc_iova(struct device *dev,
3447                                      struct dmar_domain *domain,
3448                                      unsigned long nrpages, uint64_t dma_mask)
3449 {
3450         unsigned long iova_pfn = 0;
3451
3452         /* Restrict dma_mask to the width that the iommu can handle */
3453         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3454         /* Ensure we reserve the whole size-aligned region */
3455         nrpages = __roundup_pow_of_two(nrpages);
3456
3457         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3458                 /*
3459                  * First try to allocate an io virtual address in
3460                  * DMA_BIT_MASK(32) and if that fails then try allocating
3461                  * from higher range
3462                  */
3463                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3464                                            IOVA_PFN(DMA_BIT_MASK(32)));
3465                 if (iova_pfn)
3466                         return iova_pfn;
3467         }
3468         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3469         if (unlikely(!iova_pfn)) {
3470                 pr_err("Allocating %ld-page iova for %s failed",
3471                        nrpages, dev_name(dev));
3472                 return 0;
3473         }
3474
3475         return iova_pfn;
3476 }
3477
3478 static struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3479 {
3480         struct dmar_domain *domain, *tmp;
3481         struct dmar_rmrr_unit *rmrr;
3482         struct device *i_dev;
3483         int i, ret;
3484
3485         domain = find_domain(dev);
3486         if (domain)
3487                 goto out;
3488
3489         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3490         if (!domain)
3491                 goto out;
3492
3493         /* We have a new domain - setup possible RMRRs for the device */
3494         rcu_read_lock();
3495         for_each_rmrr_units(rmrr) {
3496                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3497                                           i, i_dev) {
3498                         if (i_dev != dev)
3499                                 continue;
3500
3501                         ret = domain_prepare_identity_map(dev, domain,
3502                                                           rmrr->base_address,
3503                                                           rmrr->end_address);
3504                         if (ret)
3505                                 dev_err(dev, "Mapping reserved region failed\n");
3506                 }
3507         }
3508         rcu_read_unlock();
3509
3510         tmp = set_domain_for_dev(dev, domain);
3511         if (!tmp || domain != tmp) {
3512                 domain_exit(domain);
3513                 domain = tmp;
3514         }
3515
3516 out:
3517
3518         if (!domain)
3519                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3520
3521
3522         return domain;
3523 }
3524
3525 /* Check if the dev needs to go through non-identity map and unmap process.*/
3526 static int iommu_no_mapping(struct device *dev)
3527 {
3528         int found;
3529
3530         if (iommu_dummy(dev))
3531                 return 1;
3532
3533         if (!iommu_identity_mapping)
3534                 return 0;
3535
3536         found = identity_mapping(dev);
3537         if (found) {
3538                 if (iommu_should_identity_map(dev, 0))
3539                         return 1;
3540                 else {
3541                         /*
3542                          * 32 bit DMA is removed from si_domain and fall back
3543                          * to non-identity mapping.
3544                          */
3545                         dmar_remove_one_dev_info(si_domain, dev);
3546                         pr_info("32bit %s uses non-identity mapping\n",
3547                                 dev_name(dev));
3548                         return 0;
3549                 }
3550         } else {
3551                 /*
3552                  * In case of a detached 64 bit DMA device from vm, the device
3553                  * is put into si_domain for identity mapping.
3554                  */
3555                 if (iommu_should_identity_map(dev, 0)) {
3556                         int ret;
3557                         ret = domain_add_dev_info(si_domain, dev);
3558                         if (!ret) {
3559                                 pr_info("64bit %s uses identity mapping\n",
3560                                         dev_name(dev));
3561                                 return 1;
3562                         }
3563                 }
3564         }
3565
3566         return 0;
3567 }
3568
3569 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3570                                      size_t size, int dir, u64 dma_mask)
3571 {
3572         struct dmar_domain *domain;
3573         phys_addr_t start_paddr;
3574         unsigned long iova_pfn;
3575         int prot = 0;
3576         int ret;
3577         struct intel_iommu *iommu;
3578         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3579
3580         BUG_ON(dir == DMA_NONE);
3581
3582         if (iommu_no_mapping(dev))
3583                 return paddr;
3584
3585         domain = get_valid_domain_for_dev(dev);
3586         if (!domain)
3587                 return 0;
3588
3589         iommu = domain_get_iommu(domain);
3590         size = aligned_nrpages(paddr, size);
3591
3592         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3593         if (!iova_pfn)
3594                 goto error;
3595
3596         /*
3597          * Check if DMAR supports zero-length reads on write only
3598          * mappings..
3599          */
3600         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3601                         !cap_zlr(iommu->cap))
3602                 prot |= DMA_PTE_READ;
3603         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3604                 prot |= DMA_PTE_WRITE;
3605         /*
3606          * paddr - (paddr + size) might be partial page, we should map the whole
3607          * page.  Note: if two part of one page are separately mapped, we
3608          * might have two guest_addr mapping to the same host paddr, but this
3609          * is not a big problem
3610          */
3611         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3612                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3613         if (ret)
3614                 goto error;
3615
3616         /* it's a non-present to present mapping. Only flush if caching mode */
3617         if (cap_caching_mode(iommu->cap))
3618                 iommu_flush_iotlb_psi(iommu, domain,
3619                                       mm_to_dma_pfn(iova_pfn),
3620                                       size, 0, 1);
3621         else
3622                 iommu_flush_write_buffer(iommu);
3623
3624         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3625         start_paddr += paddr & ~PAGE_MASK;
3626         return start_paddr;
3627
3628 error:
3629         if (iova_pfn)
3630                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3631         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3632                 dev_name(dev), size, (unsigned long long)paddr, dir);
3633         return 0;
3634 }
3635
3636 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3637                                  unsigned long offset, size_t size,
3638                                  enum dma_data_direction dir,
3639                                  unsigned long attrs)
3640 {
3641         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3642                                   dir, *dev->dma_mask);
3643 }
3644
3645 static void flush_unmaps(struct deferred_flush_data *flush_data)
3646 {
3647         int i, j;
3648
3649         flush_data->timer_on = 0;
3650
3651         /* just flush them all */
3652         for (i = 0; i < g_num_of_iommus; i++) {
3653                 struct intel_iommu *iommu = g_iommus[i];
3654                 struct deferred_flush_table *flush_table =
3655                                 &flush_data->tables[i];
3656                 if (!iommu)
3657                         continue;
3658
3659                 if (!flush_table->next)
3660                         continue;
3661
3662                 /* In caching mode, global flushes turn emulation expensive */
3663                 if (!cap_caching_mode(iommu->cap))
3664                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3665                                          DMA_TLB_GLOBAL_FLUSH);
3666                 for (j = 0; j < flush_table->next; j++) {
3667                         unsigned long mask;
3668                         struct deferred_flush_entry *entry =
3669                                                 &flush_table->entries[j];
3670                         unsigned long iova_pfn = entry->iova_pfn;
3671                         unsigned long nrpages = entry->nrpages;
3672                         struct dmar_domain *domain = entry->domain;
3673                         struct page *freelist = entry->freelist;
3674
3675                         /* On real hardware multiple invalidations are expensive */
3676                         if (cap_caching_mode(iommu->cap))
3677                                 iommu_flush_iotlb_psi(iommu, domain,
3678                                         mm_to_dma_pfn(iova_pfn),
3679                                         nrpages, !freelist, 0);
3680                         else {
3681                                 mask = ilog2(nrpages);
3682                                 iommu_flush_dev_iotlb(domain,
3683                                                 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3684                         }
3685                         free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3686                         if (freelist)
3687                                 dma_free_pagelist(freelist);
3688                 }
3689                 flush_table->next = 0;
3690         }
3691
3692         flush_data->size = 0;
3693 }
3694
3695 static void flush_unmaps_timeout(unsigned long cpuid)
3696 {
3697         struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3698         unsigned long flags;
3699
3700         spin_lock_irqsave(&flush_data->lock, flags);
3701         flush_unmaps(flush_data);
3702         spin_unlock_irqrestore(&flush_data->lock, flags);
3703 }
3704
3705 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3706                       unsigned long nrpages, struct page *freelist)
3707 {
3708         unsigned long flags;
3709         int entry_id, iommu_id;
3710         struct intel_iommu *iommu;
3711         struct deferred_flush_entry *entry;
3712         struct deferred_flush_data *flush_data;
3713
3714         flush_data = raw_cpu_ptr(&deferred_flush);
3715
3716         /* Flush all CPUs' entries to avoid deferring too much.  If
3717          * this becomes a bottleneck, can just flush us, and rely on
3718          * flush timer for the rest.
3719          */
3720         if (flush_data->size == HIGH_WATER_MARK) {
3721                 int cpu;
3722
3723                 for_each_online_cpu(cpu)
3724                         flush_unmaps_timeout(cpu);
3725         }
3726
3727         spin_lock_irqsave(&flush_data->lock, flags);
3728
3729         iommu = domain_get_iommu(dom);
3730         iommu_id = iommu->seq_id;
3731
3732         entry_id = flush_data->tables[iommu_id].next;
3733         ++(flush_data->tables[iommu_id].next);
3734
3735         entry = &flush_data->tables[iommu_id].entries[entry_id];
3736         entry->domain = dom;
3737         entry->iova_pfn = iova_pfn;
3738         entry->nrpages = nrpages;
3739         entry->freelist = freelist;
3740
3741         if (!flush_data->timer_on) {
3742                 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3743                 flush_data->timer_on = 1;
3744         }
3745         flush_data->size++;
3746         spin_unlock_irqrestore(&flush_data->lock, flags);
3747 }
3748
3749 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3750 {
3751         struct dmar_domain *domain;
3752         unsigned long start_pfn, last_pfn;
3753         unsigned long nrpages;
3754         unsigned long iova_pfn;
3755         struct intel_iommu *iommu;
3756         struct page *freelist;
3757
3758         if (iommu_no_mapping(dev))
3759                 return;
3760
3761         domain = find_domain(dev);
3762         BUG_ON(!domain);
3763
3764         iommu = domain_get_iommu(domain);
3765
3766         iova_pfn = IOVA_PFN(dev_addr);
3767
3768         nrpages = aligned_nrpages(dev_addr, size);
3769         start_pfn = mm_to_dma_pfn(iova_pfn);
3770         last_pfn = start_pfn + nrpages - 1;
3771
3772         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3773                  dev_name(dev), start_pfn, last_pfn);
3774
3775         freelist = domain_unmap(domain, start_pfn, last_pfn);
3776
3777         if (intel_iommu_strict) {
3778                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3779                                       nrpages, !freelist, 0);
3780                 /* free iova */
3781                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3782                 dma_free_pagelist(freelist);
3783         } else {
3784                 add_unmap(domain, iova_pfn, nrpages, freelist);
3785                 /*
3786                  * queue up the release of the unmap to save the 1/6th of the
3787                  * cpu used up by the iotlb flush operation...
3788                  */
3789         }
3790 }
3791
3792 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3793                              size_t size, enum dma_data_direction dir,
3794                              unsigned long attrs)
3795 {
3796         intel_unmap(dev, dev_addr, size);
3797 }
3798
3799 static void *intel_alloc_coherent(struct device *dev, size_t size,
3800                                   dma_addr_t *dma_handle, gfp_t flags,
3801                                   unsigned long attrs)
3802 {
3803         struct page *page = NULL;
3804         int order;
3805
3806         size = PAGE_ALIGN(size);
3807         order = get_order(size);
3808
3809         if (!iommu_no_mapping(dev))
3810                 flags &= ~(GFP_DMA | GFP_DMA32);
3811         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3812                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3813                         flags |= GFP_DMA;
3814                 else
3815                         flags |= GFP_DMA32;
3816         }
3817
3818         if (gfpflags_allow_blocking(flags)) {
3819                 unsigned int count = size >> PAGE_SHIFT;
3820
3821                 page = dma_alloc_from_contiguous(dev, count, order, flags);
3822                 if (page && iommu_no_mapping(dev) &&
3823                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3824                         dma_release_from_contiguous(dev, page, count);
3825                         page = NULL;
3826                 }
3827         }
3828
3829         if (!page)
3830                 page = alloc_pages(flags, order);
3831         if (!page)
3832                 return NULL;
3833         memset(page_address(page), 0, size);
3834
3835         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3836                                          DMA_BIDIRECTIONAL,
3837                                          dev->coherent_dma_mask);
3838         if (*dma_handle)
3839                 return page_address(page);
3840         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3841                 __free_pages(page, order);
3842
3843         return NULL;
3844 }
3845
3846 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3847                                 dma_addr_t dma_handle, unsigned long attrs)
3848 {
3849         int order;
3850         struct page *page = virt_to_page(vaddr);
3851
3852         size = PAGE_ALIGN(size);
3853         order = get_order(size);
3854
3855         intel_unmap(dev, dma_handle, size);
3856         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3857                 __free_pages(page, order);
3858 }
3859
3860 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3861                            int nelems, enum dma_data_direction dir,
3862                            unsigned long attrs)
3863 {
3864         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3865         unsigned long nrpages = 0;
3866         struct scatterlist *sg;
3867         int i;
3868
3869         for_each_sg(sglist, sg, nelems, i) {
3870                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3871         }
3872
3873         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3874 }
3875
3876 static int intel_nontranslate_map_sg(struct device *hddev,
3877         struct scatterlist *sglist, int nelems, int dir)
3878 {
3879         int i;
3880         struct scatterlist *sg;
3881
3882         for_each_sg(sglist, sg, nelems, i) {
3883                 BUG_ON(!sg_page(sg));
3884                 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3885                 sg->dma_length = sg->length;
3886         }
3887         return nelems;
3888 }
3889
3890 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3891                         enum dma_data_direction dir, unsigned long attrs)
3892 {
3893         int i;
3894         struct dmar_domain *domain;
3895         size_t size = 0;
3896         int prot = 0;
3897         unsigned long iova_pfn;
3898         int ret;
3899         struct scatterlist *sg;
3900         unsigned long start_vpfn;
3901         struct intel_iommu *iommu;
3902
3903         BUG_ON(dir == DMA_NONE);
3904         if (iommu_no_mapping(dev))
3905                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3906
3907         domain = get_valid_domain_for_dev(dev);
3908         if (!domain)
3909                 return 0;
3910
3911         iommu = domain_get_iommu(domain);
3912
3913         for_each_sg(sglist, sg, nelems, i)
3914                 size += aligned_nrpages(sg->offset, sg->length);
3915
3916         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3917                                 *dev->dma_mask);
3918         if (!iova_pfn) {
3919                 sglist->dma_length = 0;
3920                 return 0;
3921         }
3922
3923         /*
3924          * Check if DMAR supports zero-length reads on write only
3925          * mappings..
3926          */
3927         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3928                         !cap_zlr(iommu->cap))
3929                 prot |= DMA_PTE_READ;
3930         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3931                 prot |= DMA_PTE_WRITE;
3932
3933         start_vpfn = mm_to_dma_pfn(iova_pfn);
3934
3935         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3936         if (unlikely(ret)) {
3937                 dma_pte_free_pagetable(domain, start_vpfn,
3938                                        start_vpfn + size - 1);
3939                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3940                 return 0;
3941         }
3942
3943         /* it's a non-present to present mapping. Only flush if caching mode */
3944         if (cap_caching_mode(iommu->cap))
3945                 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3946         else
3947                 iommu_flush_write_buffer(iommu);
3948
3949         return nelems;
3950 }
3951
3952 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3953 {
3954         return !dma_addr;
3955 }
3956
3957 const struct dma_map_ops intel_dma_ops = {
3958         .alloc = intel_alloc_coherent,
3959         .free = intel_free_coherent,
3960         .map_sg = intel_map_sg,
3961         .unmap_sg = intel_unmap_sg,
3962         .map_page = intel_map_page,
3963         .unmap_page = intel_unmap_page,
3964         .mapping_error = intel_mapping_error,
3965 };
3966
3967 static inline int iommu_domain_cache_init(void)
3968 {
3969         int ret = 0;
3970
3971         iommu_domain_cache = kmem_cache_create("iommu_domain",
3972                                          sizeof(struct dmar_domain),
3973                                          0,
3974                                          SLAB_HWCACHE_ALIGN,
3975
3976                                          NULL);
3977         if (!iommu_domain_cache) {
3978                 pr_err("Couldn't create iommu_domain cache\n");
3979                 ret = -ENOMEM;
3980         }
3981
3982         return ret;
3983 }
3984
3985 static inline int iommu_devinfo_cache_init(void)
3986 {
3987         int ret = 0;
3988
3989         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3990                                          sizeof(struct device_domain_info),
3991                                          0,
3992                                          SLAB_HWCACHE_ALIGN,
3993                                          NULL);
3994         if (!iommu_devinfo_cache) {
3995                 pr_err("Couldn't create devinfo cache\n");
3996                 ret = -ENOMEM;
3997         }
3998
3999         return ret;
4000 }
4001
4002 static int __init iommu_init_mempool(void)
4003 {
4004         int ret;
4005         ret = iova_cache_get();
4006         if (ret)
4007                 return ret;
4008
4009         ret = iommu_domain_cache_init();
4010         if (ret)
4011                 goto domain_error;
4012
4013         ret = iommu_devinfo_cache_init();
4014         if (!ret)
4015                 return ret;
4016
4017         kmem_cache_destroy(iommu_domain_cache);
4018 domain_error:
4019         iova_cache_put();
4020
4021         return -ENOMEM;
4022 }
4023
4024 static void __init iommu_exit_mempool(void)
4025 {
4026         kmem_cache_destroy(iommu_devinfo_cache);
4027         kmem_cache_destroy(iommu_domain_cache);
4028         iova_cache_put();
4029 }
4030
4031 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4032 {
4033         struct dmar_drhd_unit *drhd;
4034         u32 vtbar;
4035         int rc;
4036
4037         /* We know that this device on this chipset has its own IOMMU.
4038          * If we find it under a different IOMMU, then the BIOS is lying
4039          * to us. Hope that the IOMMU for this device is actually
4040          * disabled, and it needs no translation...
4041          */
4042         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4043         if (rc) {
4044                 /* "can't" happen */
4045                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4046                 return;
4047         }
4048         vtbar &= 0xffff0000;
4049
4050         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4051         drhd = dmar_find_matched_drhd_unit(pdev);
4052         if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4053                             TAINT_FIRMWARE_WORKAROUND,
4054                             "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4055                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4056 }
4057 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4058
4059 static void __init init_no_remapping_devices(void)
4060 {
4061         struct dmar_drhd_unit *drhd;
4062         struct device *dev;
4063         int i;
4064
4065         for_each_drhd_unit(drhd) {
4066                 if (!drhd->include_all) {
4067                         for_each_active_dev_scope(drhd->devices,
4068                                                   drhd->devices_cnt, i, dev)
4069                                 break;
4070                         /* ignore DMAR unit if no devices exist */
4071                         if (i == drhd->devices_cnt)
4072                                 drhd->ignored = 1;
4073                 }
4074         }
4075
4076         for_each_active_drhd_unit(drhd) {
4077                 if (drhd->include_all)
4078                         continue;
4079
4080                 for_each_active_dev_scope(drhd->devices,
4081                                           drhd->devices_cnt, i, dev)
4082                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4083                                 break;
4084                 if (i < drhd->devices_cnt)
4085                         continue;
4086
4087                 /* This IOMMU has *only* gfx devices. Either bypass it or
4088                    set the gfx_mapped flag, as appropriate */
4089                 if (dmar_map_gfx) {
4090                         intel_iommu_gfx_mapped = 1;
4091                 } else {
4092                         drhd->ignored = 1;
4093                         for_each_active_dev_scope(drhd->devices,
4094                                                   drhd->devices_cnt, i, dev)
4095                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4096                 }
4097         }
4098 }
4099
4100 #ifdef CONFIG_SUSPEND
4101 static int init_iommu_hw(void)
4102 {
4103         struct dmar_drhd_unit *drhd;
4104         struct intel_iommu *iommu = NULL;
4105
4106         for_each_active_iommu(iommu, drhd)
4107                 if (iommu->qi)
4108                         dmar_reenable_qi(iommu);
4109
4110         for_each_iommu(iommu, drhd) {
4111                 if (drhd->ignored) {
4112                         /*
4113                          * we always have to disable PMRs or DMA may fail on
4114                          * this device
4115                          */
4116                         if (force_on)
4117                                 iommu_disable_protect_mem_regions(iommu);
4118                         continue;
4119                 }
4120
4121                 iommu_flush_write_buffer(iommu);
4122
4123                 iommu_set_root_entry(iommu);
4124
4125                 iommu->flush.flush_context(iommu, 0, 0, 0,
4126                                            DMA_CCMD_GLOBAL_INVL);
4127                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4128                 iommu_enable_translation(iommu);
4129                 iommu_disable_protect_mem_regions(iommu);
4130         }
4131
4132         return 0;
4133 }
4134
4135 static void iommu_flush_all(void)
4136 {
4137         struct dmar_drhd_unit *drhd;
4138         struct intel_iommu *iommu;
4139
4140         for_each_active_iommu(iommu, drhd) {
4141                 iommu->flush.flush_context(iommu, 0, 0, 0,
4142                                            DMA_CCMD_GLOBAL_INVL);
4143                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4144                                          DMA_TLB_GLOBAL_FLUSH);
4145         }
4146 }
4147
4148 static int iommu_suspend(void)
4149 {
4150         struct dmar_drhd_unit *drhd;
4151         struct intel_iommu *iommu = NULL;
4152         unsigned long flag;
4153
4154         for_each_active_iommu(iommu, drhd) {
4155                 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4156                                                  GFP_ATOMIC);
4157                 if (!iommu->iommu_state)
4158                         goto nomem;
4159         }
4160
4161         iommu_flush_all();
4162
4163         for_each_active_iommu(iommu, drhd) {
4164                 iommu_disable_translation(iommu);
4165
4166                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4167
4168                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4169                         readl(iommu->reg + DMAR_FECTL_REG);
4170                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4171                         readl(iommu->reg + DMAR_FEDATA_REG);
4172                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4173                         readl(iommu->reg + DMAR_FEADDR_REG);
4174                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4175                         readl(iommu->reg + DMAR_FEUADDR_REG);
4176
4177                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4178         }
4179         return 0;
4180
4181 nomem:
4182         for_each_active_iommu(iommu, drhd)
4183                 kfree(iommu->iommu_state);
4184
4185         return -ENOMEM;
4186 }
4187
4188 static void iommu_resume(void)
4189 {
4190         struct dmar_drhd_unit *drhd;
4191         struct intel_iommu *iommu = NULL;
4192         unsigned long flag;
4193
4194         if (init_iommu_hw()) {
4195                 if (force_on)
4196                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4197                 else
4198                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4199                 return;
4200         }
4201
4202         for_each_active_iommu(iommu, drhd) {
4203
4204                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4205
4206                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4207                         iommu->reg + DMAR_FECTL_REG);
4208                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4209                         iommu->reg + DMAR_FEDATA_REG);
4210                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4211                         iommu->reg + DMAR_FEADDR_REG);
4212                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4213                         iommu->reg + DMAR_FEUADDR_REG);
4214
4215                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4216         }
4217
4218         for_each_active_iommu(iommu, drhd)
4219                 kfree(iommu->iommu_state);
4220 }
4221
4222 static struct syscore_ops iommu_syscore_ops = {
4223         .resume         = iommu_resume,
4224         .suspend        = iommu_suspend,
4225 };
4226
4227 static void __init init_iommu_pm_ops(void)
4228 {
4229         register_syscore_ops(&iommu_syscore_ops);
4230 }
4231
4232 #else
4233 static inline void init_iommu_pm_ops(void) {}
4234 #endif  /* CONFIG_PM */
4235
4236
4237 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4238 {
4239         struct acpi_dmar_reserved_memory *rmrr;
4240         int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4241         struct dmar_rmrr_unit *rmrru;
4242         size_t length;
4243
4244         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4245         if (!rmrru)
4246                 goto out;
4247
4248         rmrru->hdr = header;
4249         rmrr = (struct acpi_dmar_reserved_memory *)header;
4250         rmrru->base_address = rmrr->base_address;
4251         rmrru->end_address = rmrr->end_address;
4252
4253         length = rmrr->end_address - rmrr->base_address + 1;
4254         rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4255                                               IOMMU_RESV_DIRECT);
4256         if (!rmrru->resv)
4257                 goto free_rmrru;
4258
4259         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4260                                 ((void *)rmrr) + rmrr->header.length,
4261                                 &rmrru->devices_cnt);
4262         if (rmrru->devices_cnt && rmrru->devices == NULL)
4263                 goto free_all;
4264
4265         list_add(&rmrru->list, &dmar_rmrr_units);
4266
4267         return 0;
4268 free_all:
4269         kfree(rmrru->resv);
4270 free_rmrru:
4271         kfree(rmrru);
4272 out:
4273         return -ENOMEM;
4274 }
4275
4276 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4277 {
4278         struct dmar_atsr_unit *atsru;
4279         struct acpi_dmar_atsr *tmp;
4280
4281         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4282                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4283                 if (atsr->segment != tmp->segment)
4284                         continue;
4285                 if (atsr->header.length != tmp->header.length)
4286                         continue;
4287                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4288                         return atsru;
4289         }
4290
4291         return NULL;
4292 }
4293
4294 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4295 {
4296         struct acpi_dmar_atsr *atsr;
4297         struct dmar_atsr_unit *atsru;
4298
4299         if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4300                 return 0;
4301
4302         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4303         atsru = dmar_find_atsr(atsr);
4304         if (atsru)
4305                 return 0;
4306
4307         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4308         if (!atsru)
4309                 return -ENOMEM;
4310
4311         /*
4312          * If memory is allocated from slab by ACPI _DSM method, we need to
4313          * copy the memory content because the memory buffer will be freed
4314          * on return.
4315          */
4316         atsru->hdr = (void *)(atsru + 1);
4317         memcpy(atsru->hdr, hdr, hdr->length);
4318         atsru->include_all = atsr->flags & 0x1;
4319         if (!atsru->include_all) {
4320                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4321                                 (void *)atsr + atsr->header.length,
4322                                 &atsru->devices_cnt);
4323                 if (atsru->devices_cnt && atsru->devices == NULL) {
4324                         kfree(atsru);
4325                         return -ENOMEM;
4326                 }
4327         }
4328
4329         list_add_rcu(&atsru->list, &dmar_atsr_units);
4330
4331         return 0;
4332 }
4333
4334 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4335 {
4336         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4337         kfree(atsru);
4338 }
4339
4340 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4341 {
4342         struct acpi_dmar_atsr *atsr;
4343         struct dmar_atsr_unit *atsru;
4344
4345         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4346         atsru = dmar_find_atsr(atsr);
4347         if (atsru) {
4348                 list_del_rcu(&atsru->list);
4349                 synchronize_rcu();
4350                 intel_iommu_free_atsr(atsru);
4351         }
4352
4353         return 0;
4354 }
4355
4356 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4357 {
4358         int i;
4359         struct device *dev;
4360         struct acpi_dmar_atsr *atsr;
4361         struct dmar_atsr_unit *atsru;
4362
4363         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4364         atsru = dmar_find_atsr(atsr);
4365         if (!atsru)
4366                 return 0;
4367
4368         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4369                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4370                                           i, dev)
4371                         return -EBUSY;
4372         }
4373
4374         return 0;
4375 }
4376
4377 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4378 {
4379         int sp, ret = 0;
4380         struct intel_iommu *iommu = dmaru->iommu;
4381
4382         if (g_iommus[iommu->seq_id])
4383                 return 0;
4384
4385         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4386                 pr_warn("%s: Doesn't support hardware pass through.\n",
4387                         iommu->name);
4388                 return -ENXIO;
4389         }
4390         if (!ecap_sc_support(iommu->ecap) &&
4391             domain_update_iommu_snooping(iommu)) {
4392                 pr_warn("%s: Doesn't support snooping.\n",
4393                         iommu->name);
4394                 return -ENXIO;
4395         }
4396         sp = domain_update_iommu_superpage(iommu) - 1;
4397         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4398                 pr_warn("%s: Doesn't support large page.\n",
4399                         iommu->name);
4400                 return -ENXIO;
4401         }
4402
4403         /*
4404          * Disable translation if already enabled prior to OS handover.
4405          */
4406         if (iommu->gcmd & DMA_GCMD_TE)
4407                 iommu_disable_translation(iommu);
4408
4409         g_iommus[iommu->seq_id] = iommu;
4410         ret = iommu_init_domains(iommu);
4411         if (ret == 0)
4412                 ret = iommu_alloc_root_entry(iommu);
4413         if (ret)
4414                 goto out;
4415
4416 #ifdef CONFIG_INTEL_IOMMU_SVM
4417         if (pasid_enabled(iommu))
4418                 intel_svm_alloc_pasid_tables(iommu);
4419 #endif
4420
4421         if (dmaru->ignored) {
4422                 /*
4423                  * we always have to disable PMRs or DMA may fail on this device
4424                  */
4425                 if (force_on)
4426                         iommu_disable_protect_mem_regions(iommu);
4427                 return 0;
4428         }
4429
4430         intel_iommu_init_qi(iommu);
4431         iommu_flush_write_buffer(iommu);
4432
4433 #ifdef CONFIG_INTEL_IOMMU_SVM
4434         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4435                 ret = intel_svm_enable_prq(iommu);
4436                 if (ret)
4437                         goto disable_iommu;
4438         }
4439 #endif
4440         ret = dmar_set_interrupt(iommu);
4441         if (ret)
4442                 goto disable_iommu;
4443
4444         iommu_set_root_entry(iommu);
4445         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4446         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4447         iommu_enable_translation(iommu);
4448
4449         iommu_disable_protect_mem_regions(iommu);
4450         return 0;
4451
4452 disable_iommu:
4453         disable_dmar_iommu(iommu);
4454 out:
4455         free_dmar_iommu(iommu);
4456         return ret;
4457 }
4458
4459 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4460 {
4461         int ret = 0;
4462         struct intel_iommu *iommu = dmaru->iommu;
4463
4464         if (!intel_iommu_enabled)
4465                 return 0;
4466         if (iommu == NULL)
4467                 return -EINVAL;
4468
4469         if (insert) {
4470                 ret = intel_iommu_add(dmaru);
4471         } else {
4472                 disable_dmar_iommu(iommu);
4473                 free_dmar_iommu(iommu);
4474         }
4475
4476         return ret;
4477 }
4478
4479 static void intel_iommu_free_dmars(void)
4480 {
4481         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4482         struct dmar_atsr_unit *atsru, *atsr_n;
4483
4484         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4485                 list_del(&rmrru->list);
4486                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4487                 kfree(rmrru->resv);
4488                 kfree(rmrru);
4489         }
4490
4491         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4492                 list_del(&atsru->list);
4493                 intel_iommu_free_atsr(atsru);
4494         }
4495 }
4496
4497 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4498 {
4499         int i, ret = 1;
4500         struct pci_bus *bus;
4501         struct pci_dev *bridge = NULL;
4502         struct device *tmp;
4503         struct acpi_dmar_atsr *atsr;
4504         struct dmar_atsr_unit *atsru;
4505
4506         dev = pci_physfn(dev);
4507         for (bus = dev->bus; bus; bus = bus->parent) {
4508                 bridge = bus->self;
4509                 /* If it's an integrated device, allow ATS */
4510                 if (!bridge)
4511                         return 1;
4512                 /* Connected via non-PCIe: no ATS */
4513                 if (!pci_is_pcie(bridge) ||
4514                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4515                         return 0;
4516                 /* If we found the root port, look it up in the ATSR */
4517                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4518                         break;
4519         }
4520
4521         rcu_read_lock();
4522         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4523                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4524                 if (atsr->segment != pci_domain_nr(dev->bus))
4525                         continue;
4526
4527                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4528                         if (tmp == &bridge->dev)
4529                                 goto out;
4530
4531                 if (atsru->include_all)
4532                         goto out;
4533         }
4534         ret = 0;
4535 out:
4536         rcu_read_unlock();
4537
4538         return ret;
4539 }
4540
4541 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4542 {
4543         int ret = 0;
4544         struct dmar_rmrr_unit *rmrru;
4545         struct dmar_atsr_unit *atsru;
4546         struct acpi_dmar_atsr *atsr;
4547         struct acpi_dmar_reserved_memory *rmrr;
4548
4549         if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4550                 return 0;
4551
4552         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4553                 rmrr = container_of(rmrru->hdr,
4554                                     struct acpi_dmar_reserved_memory, header);
4555                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4556                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4557                                 ((void *)rmrr) + rmrr->header.length,
4558                                 rmrr->segment, rmrru->devices,
4559                                 rmrru->devices_cnt);
4560                         if(ret < 0)
4561                                 return ret;
4562                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4563                         dmar_remove_dev_scope(info, rmrr->segment,
4564                                 rmrru->devices, rmrru->devices_cnt);
4565                 }
4566         }
4567
4568         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4569                 if (atsru->include_all)
4570                         continue;
4571
4572                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4573                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4574                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4575                                         (void *)atsr + atsr->header.length,
4576                                         atsr->segment, atsru->devices,
4577                                         atsru->devices_cnt);
4578                         if (ret > 0)
4579                                 break;
4580                         else if(ret < 0)
4581                                 return ret;
4582                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4583                         if (dmar_remove_dev_scope(info, atsr->segment,
4584                                         atsru->devices, atsru->devices_cnt))
4585                                 break;
4586                 }
4587         }
4588
4589         return 0;
4590 }
4591
4592 /*
4593  * Here we only respond to action of unbound device from driver.
4594  *
4595  * Added device is not attached to its DMAR domain here yet. That will happen
4596  * when mapping the device to iova.
4597  */
4598 static int device_notifier(struct notifier_block *nb,
4599                                   unsigned long action, void *data)
4600 {
4601         struct device *dev = data;
4602         struct dmar_domain *domain;
4603
4604         if (iommu_dummy(dev))
4605                 return 0;
4606
4607         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4608                 return 0;
4609
4610         domain = find_domain(dev);
4611         if (!domain)
4612                 return 0;
4613
4614         dmar_remove_one_dev_info(domain, dev);
4615         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4616                 domain_exit(domain);
4617
4618         return 0;
4619 }
4620
4621 static struct notifier_block device_nb = {
4622         .notifier_call = device_notifier,
4623 };
4624
4625 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4626                                        unsigned long val, void *v)
4627 {
4628         struct memory_notify *mhp = v;
4629         unsigned long long start, end;
4630         unsigned long start_vpfn, last_vpfn;
4631
4632         switch (val) {
4633         case MEM_GOING_ONLINE:
4634                 start = mhp->start_pfn << PAGE_SHIFT;
4635                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4636                 if (iommu_domain_identity_map(si_domain, start, end)) {
4637                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4638                                 start, end);
4639                         return NOTIFY_BAD;
4640                 }
4641                 break;
4642
4643         case MEM_OFFLINE:
4644         case MEM_CANCEL_ONLINE:
4645                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4646                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4647                 while (start_vpfn <= last_vpfn) {
4648                         struct iova *iova;
4649                         struct dmar_drhd_unit *drhd;
4650                         struct intel_iommu *iommu;
4651                         struct page *freelist;
4652
4653                         iova = find_iova(&si_domain->iovad, start_vpfn);
4654                         if (iova == NULL) {
4655                                 pr_debug("Failed get IOVA for PFN %lx\n",
4656                                          start_vpfn);
4657                                 break;
4658                         }
4659
4660                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4661                                                      start_vpfn, last_vpfn);
4662                         if (iova == NULL) {
4663                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4664                                         start_vpfn, last_vpfn);
4665                                 return NOTIFY_BAD;
4666                         }
4667
4668                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4669                                                iova->pfn_hi);
4670
4671                         rcu_read_lock();
4672                         for_each_active_iommu(iommu, drhd)
4673                                 iommu_flush_iotlb_psi(iommu, si_domain,
4674                                         iova->pfn_lo, iova_size(iova),
4675                                         !freelist, 0);
4676                         rcu_read_unlock();
4677                         dma_free_pagelist(freelist);
4678
4679                         start_vpfn = iova->pfn_hi + 1;
4680                         free_iova_mem(iova);
4681                 }
4682                 break;
4683         }
4684
4685         return NOTIFY_OK;
4686 }
4687
4688 static struct notifier_block intel_iommu_memory_nb = {
4689         .notifier_call = intel_iommu_memory_notifier,
4690         .priority = 0
4691 };
4692
4693 static void free_all_cpu_cached_iovas(unsigned int cpu)
4694 {
4695         int i;
4696
4697         for (i = 0; i < g_num_of_iommus; i++) {
4698                 struct intel_iommu *iommu = g_iommus[i];
4699                 struct dmar_domain *domain;
4700                 int did;
4701
4702                 if (!iommu)
4703                         continue;
4704
4705                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4706                         domain = get_iommu_domain(iommu, (u16)did);
4707
4708                         if (!domain)
4709                                 continue;
4710                         free_cpu_cached_iovas(cpu, &domain->iovad);
4711                 }
4712         }
4713 }
4714
4715 static int intel_iommu_cpu_dead(unsigned int cpu)
4716 {
4717         free_all_cpu_cached_iovas(cpu);
4718         flush_unmaps_timeout(cpu);
4719         return 0;
4720 }
4721
4722 static void intel_disable_iommus(void)
4723 {
4724         struct intel_iommu *iommu = NULL;
4725         struct dmar_drhd_unit *drhd;
4726
4727         for_each_iommu(iommu, drhd)
4728                 iommu_disable_translation(iommu);
4729 }
4730
4731 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4732 {
4733         return container_of(dev, struct intel_iommu, iommu.dev);
4734 }
4735
4736 static ssize_t intel_iommu_show_version(struct device *dev,
4737                                         struct device_attribute *attr,
4738                                         char *buf)
4739 {
4740         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4741         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4742         return sprintf(buf, "%d:%d\n",
4743                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4744 }
4745 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4746
4747 static ssize_t intel_iommu_show_address(struct device *dev,
4748                                         struct device_attribute *attr,
4749                                         char *buf)
4750 {
4751         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4752         return sprintf(buf, "%llx\n", iommu->reg_phys);
4753 }
4754 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4755
4756 static ssize_t intel_iommu_show_cap(struct device *dev,
4757                                     struct device_attribute *attr,
4758                                     char *buf)
4759 {
4760         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4761         return sprintf(buf, "%llx\n", iommu->cap);
4762 }
4763 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4764
4765 static ssize_t intel_iommu_show_ecap(struct device *dev,
4766                                     struct device_attribute *attr,
4767                                     char *buf)
4768 {
4769         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4770         return sprintf(buf, "%llx\n", iommu->ecap);
4771 }
4772 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4773
4774 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4775                                       struct device_attribute *attr,
4776                                       char *buf)
4777 {
4778         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4779         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4780 }
4781 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4782
4783 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4784                                            struct device_attribute *attr,
4785                                            char *buf)
4786 {
4787         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4788         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4789                                                   cap_ndoms(iommu->cap)));
4790 }
4791 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4792
4793 static struct attribute *intel_iommu_attrs[] = {
4794         &dev_attr_version.attr,
4795         &dev_attr_address.attr,
4796         &dev_attr_cap.attr,
4797         &dev_attr_ecap.attr,
4798         &dev_attr_domains_supported.attr,
4799         &dev_attr_domains_used.attr,
4800         NULL,
4801 };
4802
4803 static struct attribute_group intel_iommu_group = {
4804         .name = "intel-iommu",
4805         .attrs = intel_iommu_attrs,
4806 };
4807
4808 const struct attribute_group *intel_iommu_groups[] = {
4809         &intel_iommu_group,
4810         NULL,
4811 };
4812
4813 int __init intel_iommu_init(void)
4814 {
4815         int ret = -ENODEV;
4816         struct dmar_drhd_unit *drhd;
4817         struct intel_iommu *iommu;
4818
4819         /* VT-d is required for a TXT/tboot launch, so enforce that */
4820         force_on = tboot_force_iommu();
4821
4822         if (iommu_init_mempool()) {
4823                 if (force_on)
4824                         panic("tboot: Failed to initialize iommu memory\n");
4825                 return -ENOMEM;
4826         }
4827
4828         down_write(&dmar_global_lock);
4829         if (dmar_table_init()) {
4830                 if (force_on)
4831                         panic("tboot: Failed to initialize DMAR table\n");
4832                 goto out_free_dmar;
4833         }
4834
4835         if (dmar_dev_scope_init() < 0) {
4836                 if (force_on)
4837                         panic("tboot: Failed to initialize DMAR device scope\n");
4838                 goto out_free_dmar;
4839         }
4840
4841         if (no_iommu || dmar_disabled) {
4842                 /*
4843                  * We exit the function here to ensure IOMMU's remapping and
4844                  * mempool aren't setup, which means that the IOMMU's PMRs
4845                  * won't be disabled via the call to init_dmars(). So disable
4846                  * it explicitly here. The PMRs were setup by tboot prior to
4847                  * calling SENTER, but the kernel is expected to reset/tear
4848                  * down the PMRs.
4849                  */
4850                 if (intel_iommu_tboot_noforce) {
4851                         for_each_iommu(iommu, drhd)
4852                                 iommu_disable_protect_mem_regions(iommu);
4853                 }
4854
4855                 /*
4856                  * Make sure the IOMMUs are switched off, even when we
4857                  * boot into a kexec kernel and the previous kernel left
4858                  * them enabled
4859                  */
4860                 intel_disable_iommus();
4861                 goto out_free_dmar;
4862         }
4863
4864         if (list_empty(&dmar_rmrr_units))
4865                 pr_info("No RMRR found\n");
4866
4867         if (list_empty(&dmar_atsr_units))
4868                 pr_info("No ATSR found\n");
4869
4870         if (dmar_init_reserved_ranges()) {
4871                 if (force_on)
4872                         panic("tboot: Failed to reserve iommu ranges\n");
4873                 goto out_free_reserved_range;
4874         }
4875
4876         init_no_remapping_devices();
4877
4878         ret = init_dmars();
4879         if (ret) {
4880                 if (force_on)
4881                         panic("tboot: Failed to initialize DMARs\n");
4882                 pr_err("Initialization failed\n");
4883                 goto out_free_reserved_range;
4884         }
4885         up_write(&dmar_global_lock);
4886         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4887
4888 #ifdef CONFIG_SWIOTLB
4889         swiotlb = 0;
4890 #endif
4891         dma_ops = &intel_dma_ops;
4892
4893         init_iommu_pm_ops();
4894
4895         for_each_active_iommu(iommu, drhd) {
4896                 iommu_device_sysfs_add(&iommu->iommu, NULL,
4897                                        intel_iommu_groups,
4898                                        "%s", iommu->name);
4899                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4900                 iommu_device_register(&iommu->iommu);
4901         }
4902
4903         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4904         bus_register_notifier(&pci_bus_type, &device_nb);
4905         if (si_domain && !hw_pass_through)
4906                 register_memory_notifier(&intel_iommu_memory_nb);
4907         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4908                           intel_iommu_cpu_dead);
4909         intel_iommu_enabled = 1;
4910
4911         return 0;
4912
4913 out_free_reserved_range:
4914         put_iova_domain(&reserved_iova_list);
4915 out_free_dmar:
4916         intel_iommu_free_dmars();
4917         up_write(&dmar_global_lock);
4918         iommu_exit_mempool();
4919         return ret;
4920 }
4921
4922 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4923 {
4924         struct intel_iommu *iommu = opaque;
4925
4926         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4927         return 0;
4928 }
4929
4930 /*
4931  * NB - intel-iommu lacks any sort of reference counting for the users of
4932  * dependent devices.  If multiple endpoints have intersecting dependent
4933  * devices, unbinding the driver from any one of them will possibly leave
4934  * the others unable to operate.
4935  */
4936 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4937 {
4938         if (!iommu || !dev || !dev_is_pci(dev))
4939                 return;
4940
4941         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4942 }
4943
4944 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4945 {
4946         struct intel_iommu *iommu;
4947         unsigned long flags;
4948
4949         assert_spin_locked(&device_domain_lock);
4950
4951         if (WARN_ON(!info))
4952                 return;
4953
4954         iommu = info->iommu;
4955
4956         if (info->dev) {
4957                 iommu_disable_dev_iotlb(info);
4958                 domain_context_clear(iommu, info->dev);
4959         }
4960
4961         unlink_domain_info(info);
4962
4963         spin_lock_irqsave(&iommu->lock, flags);
4964         domain_detach_iommu(info->domain, iommu);
4965         spin_unlock_irqrestore(&iommu->lock, flags);
4966
4967         free_devinfo_mem(info);
4968 }
4969
4970 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4971                                      struct device *dev)
4972 {
4973         struct device_domain_info *info;
4974         unsigned long flags;
4975
4976         spin_lock_irqsave(&device_domain_lock, flags);
4977         info = dev->archdata.iommu;
4978         __dmar_remove_one_dev_info(info);
4979         spin_unlock_irqrestore(&device_domain_lock, flags);
4980 }
4981
4982 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4983 {
4984         int adjust_width;
4985
4986         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4987                         DMA_32BIT_PFN);
4988         domain_reserve_special_ranges(domain);
4989
4990         /* calculate AGAW */
4991         domain->gaw = guest_width;
4992         adjust_width = guestwidth_to_adjustwidth(guest_width);
4993         domain->agaw = width_to_agaw(adjust_width);
4994
4995         domain->iommu_coherency = 0;
4996         domain->iommu_snooping = 0;
4997         domain->iommu_superpage = 0;
4998         domain->max_addr = 0;
4999
5000         /* always allocate the top pgd */
5001         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5002         if (!domain->pgd)
5003                 return -ENOMEM;
5004         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5005         return 0;
5006 }
5007
5008 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5009 {
5010         struct dmar_domain *dmar_domain;
5011         struct iommu_domain *domain;
5012
5013         if (type != IOMMU_DOMAIN_UNMANAGED)
5014                 return NULL;
5015
5016         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5017         if (!dmar_domain) {
5018                 pr_err("Can't allocate dmar_domain\n");
5019                 return NULL;
5020         }
5021         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5022                 pr_err("Domain initialization failed\n");
5023                 domain_exit(dmar_domain);
5024                 return NULL;
5025         }
5026         domain_update_iommu_cap(dmar_domain);
5027
5028         domain = &dmar_domain->domain;
5029         domain->geometry.aperture_start = 0;
5030         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5031         domain->geometry.force_aperture = true;
5032
5033         return domain;
5034 }
5035
5036 static void intel_iommu_domain_free(struct iommu_domain *domain)
5037 {
5038         domain_exit(to_dmar_domain(domain));
5039 }
5040
5041 static int intel_iommu_attach_device(struct iommu_domain *domain,
5042                                      struct device *dev)
5043 {
5044         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5045         struct intel_iommu *iommu;
5046         int addr_width;
5047         u8 bus, devfn;
5048
5049         if (device_is_rmrr_locked(dev)) {
5050                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5051                 return -EPERM;
5052         }
5053
5054         /* normally dev is not mapped */
5055         if (unlikely(domain_context_mapped(dev))) {
5056                 struct dmar_domain *old_domain;
5057
5058                 old_domain = find_domain(dev);
5059                 if (old_domain) {
5060                         rcu_read_lock();
5061                         dmar_remove_one_dev_info(old_domain, dev);
5062                         rcu_read_unlock();
5063
5064                         if (!domain_type_is_vm_or_si(old_domain) &&
5065                              list_empty(&old_domain->devices))
5066                                 domain_exit(old_domain);
5067                 }
5068         }
5069
5070         iommu = device_to_iommu(dev, &bus, &devfn);
5071         if (!iommu)
5072                 return -ENODEV;
5073
5074         /* check if this iommu agaw is sufficient for max mapped address */
5075         addr_width = agaw_to_width(iommu->agaw);
5076         if (addr_width > cap_mgaw(iommu->cap))
5077                 addr_width = cap_mgaw(iommu->cap);
5078
5079         if (dmar_domain->max_addr > (1LL << addr_width)) {
5080                 pr_err("%s: iommu width (%d) is not "
5081                        "sufficient for the mapped address (%llx)\n",
5082                        __func__, addr_width, dmar_domain->max_addr);
5083                 return -EFAULT;
5084         }
5085         dmar_domain->gaw = addr_width;
5086
5087         /*
5088          * Knock out extra levels of page tables if necessary
5089          */
5090         while (iommu->agaw < dmar_domain->agaw) {
5091                 struct dma_pte *pte;
5092
5093                 pte = dmar_domain->pgd;
5094                 if (dma_pte_present(pte)) {
5095                         dmar_domain->pgd = (struct dma_pte *)
5096                                 phys_to_virt(dma_pte_addr(pte));
5097                         free_pgtable_page(pte);
5098                 }
5099                 dmar_domain->agaw--;
5100         }
5101
5102         return domain_add_dev_info(dmar_domain, dev);
5103 }
5104
5105 static void intel_iommu_detach_device(struct iommu_domain *domain,
5106                                       struct device *dev)
5107 {
5108         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5109 }
5110
5111 static int intel_iommu_map(struct iommu_domain *domain,
5112                            unsigned long iova, phys_addr_t hpa,
5113                            size_t size, int iommu_prot)
5114 {
5115         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5116         u64 max_addr;
5117         int prot = 0;
5118         int ret;
5119
5120         if (iommu_prot & IOMMU_READ)
5121                 prot |= DMA_PTE_READ;
5122         if (iommu_prot & IOMMU_WRITE)
5123                 prot |= DMA_PTE_WRITE;
5124         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5125                 prot |= DMA_PTE_SNP;
5126
5127         max_addr = iova + size;
5128         if (dmar_domain->max_addr < max_addr) {
5129                 u64 end;
5130
5131                 /* check if minimum agaw is sufficient for mapped address */
5132                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5133                 if (end < max_addr) {
5134                         pr_err("%s: iommu width (%d) is not "
5135                                "sufficient for the mapped address (%llx)\n",
5136                                __func__, dmar_domain->gaw, max_addr);
5137                         return -EFAULT;
5138                 }
5139                 dmar_domain->max_addr = max_addr;
5140         }
5141         /* Round up size to next multiple of PAGE_SIZE, if it and
5142            the low bits of hpa would take us onto the next page */
5143         size = aligned_nrpages(hpa, size);
5144         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5145                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5146         return ret;
5147 }
5148
5149 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5150                                 unsigned long iova, size_t size)
5151 {
5152         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5153         struct page *freelist = NULL;
5154         struct intel_iommu *iommu;
5155         unsigned long start_pfn, last_pfn;
5156         unsigned int npages;
5157         int iommu_id, level = 0;
5158
5159         /* Cope with horrid API which requires us to unmap more than the
5160            size argument if it happens to be a large-page mapping. */
5161         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5162
5163         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5164                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5165
5166         start_pfn = iova >> VTD_PAGE_SHIFT;
5167         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5168
5169         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5170
5171         npages = last_pfn - start_pfn + 1;
5172
5173         for_each_domain_iommu(iommu_id, dmar_domain) {
5174                 iommu = g_iommus[iommu_id];
5175
5176                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5177                                       start_pfn, npages, !freelist, 0);
5178         }
5179
5180         dma_free_pagelist(freelist);
5181
5182         if (dmar_domain->max_addr == iova + size)
5183                 dmar_domain->max_addr = iova;
5184
5185         return size;
5186 }
5187
5188 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5189                                             dma_addr_t iova)
5190 {
5191         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5192         struct dma_pte *pte;
5193         int level = 0;
5194         u64 phys = 0;
5195
5196         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5197         if (pte)
5198                 phys = dma_pte_addr(pte);
5199
5200         return phys;
5201 }
5202
5203 static bool intel_iommu_capable(enum iommu_cap cap)
5204 {
5205         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5206                 return domain_update_iommu_snooping(NULL) == 1;
5207         if (cap == IOMMU_CAP_INTR_REMAP)
5208                 return irq_remapping_enabled == 1;
5209
5210         return false;
5211 }
5212
5213 static int intel_iommu_add_device(struct device *dev)
5214 {
5215         struct intel_iommu *iommu;
5216         struct iommu_group *group;
5217         u8 bus, devfn;
5218
5219         iommu = device_to_iommu(dev, &bus, &devfn);
5220         if (!iommu)
5221                 return -ENODEV;
5222
5223         iommu_device_link(&iommu->iommu, dev);
5224
5225         group = iommu_group_get_for_dev(dev);
5226
5227         if (IS_ERR(group))
5228                 return PTR_ERR(group);
5229
5230         iommu_group_put(group);
5231         return 0;
5232 }
5233
5234 static void intel_iommu_remove_device(struct device *dev)
5235 {
5236         struct intel_iommu *iommu;
5237         u8 bus, devfn;
5238
5239         iommu = device_to_iommu(dev, &bus, &devfn);
5240         if (!iommu)
5241                 return;
5242
5243         iommu_group_remove_device(dev);
5244
5245         iommu_device_unlink(&iommu->iommu, dev);
5246 }
5247
5248 static void intel_iommu_get_resv_regions(struct device *device,
5249                                          struct list_head *head)
5250 {
5251         struct iommu_resv_region *reg;
5252         struct dmar_rmrr_unit *rmrr;
5253         struct device *i_dev;
5254         int i;
5255
5256         rcu_read_lock();
5257         for_each_rmrr_units(rmrr) {
5258                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5259                                           i, i_dev) {
5260                         if (i_dev != device)
5261                                 continue;
5262
5263                         list_add_tail(&rmrr->resv->list, head);
5264                 }
5265         }
5266         rcu_read_unlock();
5267
5268         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5269                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5270                                       0, IOMMU_RESV_MSI);
5271         if (!reg)
5272                 return;
5273         list_add_tail(&reg->list, head);
5274 }
5275
5276 static void intel_iommu_put_resv_regions(struct device *dev,
5277                                          struct list_head *head)
5278 {
5279         struct iommu_resv_region *entry, *next;
5280
5281         list_for_each_entry_safe(entry, next, head, list) {
5282                 if (entry->type == IOMMU_RESV_RESERVED)
5283                         kfree(entry);
5284         }
5285 }
5286
5287 #ifdef CONFIG_INTEL_IOMMU_SVM
5288 #define MAX_NR_PASID_BITS (20)
5289 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5290 {
5291         /*
5292          * Convert ecap_pss to extend context entry pts encoding, also
5293          * respect the soft pasid_max value set by the iommu.
5294          * - number of PASID bits = ecap_pss + 1
5295          * - number of PASID table entries = 2^(pts + 5)
5296          * Therefore, pts = ecap_pss - 4
5297          * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5298          */
5299         if (ecap_pss(iommu->ecap) < 5)
5300                 return 0;
5301
5302         /* pasid_max is encoded as actual number of entries not the bits */
5303         return find_first_bit((unsigned long *)&iommu->pasid_max,
5304                         MAX_NR_PASID_BITS) - 5;
5305 }
5306
5307 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5308 {
5309         struct device_domain_info *info;
5310         struct context_entry *context;
5311         struct dmar_domain *domain;
5312         unsigned long flags;
5313         u64 ctx_lo;
5314         int ret;
5315
5316         domain = get_valid_domain_for_dev(sdev->dev);
5317         if (!domain)
5318                 return -EINVAL;
5319
5320         spin_lock_irqsave(&device_domain_lock, flags);
5321         spin_lock(&iommu->lock);
5322
5323         ret = -EINVAL;
5324         info = sdev->dev->archdata.iommu;
5325         if (!info || !info->pasid_supported)
5326                 goto out;
5327
5328         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5329         if (WARN_ON(!context))
5330                 goto out;
5331
5332         ctx_lo = context[0].lo;
5333
5334         sdev->did = domain->iommu_did[iommu->seq_id];
5335         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5336
5337         if (!(ctx_lo & CONTEXT_PASIDE)) {
5338                 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5339                 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5340                         intel_iommu_get_pts(iommu);
5341
5342                 wmb();
5343                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5344                  * extended to permit requests-with-PASID if the PASIDE bit
5345                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5346                  * however, the PASIDE bit is ignored and requests-with-PASID
5347                  * are unconditionally blocked. Which makes less sense.
5348                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5349                  * "guest mode" translation types depending on whether ATS
5350                  * is available or not. Annoyingly, we can't use the new
5351                  * modes *unless* PASIDE is set. */
5352                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5353                         ctx_lo &= ~CONTEXT_TT_MASK;
5354                         if (info->ats_supported)
5355                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5356                         else
5357                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5358                 }
5359                 ctx_lo |= CONTEXT_PASIDE;
5360                 if (iommu->pasid_state_table)
5361                         ctx_lo |= CONTEXT_DINVE;
5362                 if (info->pri_supported)
5363                         ctx_lo |= CONTEXT_PRS;
5364                 context[0].lo = ctx_lo;
5365                 wmb();
5366                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5367                                            DMA_CCMD_MASK_NOBIT,
5368                                            DMA_CCMD_DEVICE_INVL);
5369         }
5370
5371         /* Enable PASID support in the device, if it wasn't already */
5372         if (!info->pasid_enabled)
5373                 iommu_enable_dev_iotlb(info);
5374
5375         if (info->ats_enabled) {
5376                 sdev->dev_iotlb = 1;
5377                 sdev->qdep = info->ats_qdep;
5378                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5379                         sdev->qdep = 0;
5380         }
5381         ret = 0;
5382
5383  out:
5384         spin_unlock(&iommu->lock);
5385         spin_unlock_irqrestore(&device_domain_lock, flags);
5386
5387         return ret;
5388 }
5389
5390 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5391 {
5392         struct intel_iommu *iommu;
5393         u8 bus, devfn;
5394
5395         if (iommu_dummy(dev)) {
5396                 dev_warn(dev,
5397                          "No IOMMU translation for device; cannot enable SVM\n");
5398                 return NULL;
5399         }
5400
5401         iommu = device_to_iommu(dev, &bus, &devfn);
5402         if ((!iommu)) {
5403                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5404                 return NULL;
5405         }
5406
5407         if (!iommu->pasid_table) {
5408                 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5409                 return NULL;
5410         }
5411
5412         return iommu;
5413 }
5414 #endif /* CONFIG_INTEL_IOMMU_SVM */
5415
5416 const struct iommu_ops intel_iommu_ops = {
5417         .capable                = intel_iommu_capable,
5418         .domain_alloc           = intel_iommu_domain_alloc,
5419         .domain_free            = intel_iommu_domain_free,
5420         .attach_dev             = intel_iommu_attach_device,
5421         .detach_dev             = intel_iommu_detach_device,
5422         .map                    = intel_iommu_map,
5423         .unmap                  = intel_iommu_unmap,
5424         .map_sg                 = default_iommu_map_sg,
5425         .iova_to_phys           = intel_iommu_iova_to_phys,
5426         .add_device             = intel_iommu_add_device,
5427         .remove_device          = intel_iommu_remove_device,
5428         .get_resv_regions       = intel_iommu_get_resv_regions,
5429         .put_resv_regions       = intel_iommu_put_resv_regions,
5430         .device_group           = pci_device_group,
5431         .pgsize_bitmap          = INTEL_IOMMU_PGSIZES,
5432 };
5433
5434 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5435 {
5436         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5437         pr_info("Disabling IOMMU for graphics on this chipset\n");
5438         dmar_map_gfx = 0;
5439 }
5440
5441 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5442 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5443 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5444 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5445 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5446 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5447 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5448
5449 static void quirk_iommu_rwbf(struct pci_dev *dev)
5450 {
5451         /*
5452          * Mobile 4 Series Chipset neglects to set RWBF capability,
5453          * but needs it. Same seems to hold for the desktop versions.
5454          */
5455         pr_info("Forcing write-buffer flush capability\n");
5456         rwbf_quirk = 1;
5457 }
5458
5459 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5460 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5461 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5462 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5463 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5464 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5465 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5466
5467 #define GGC 0x52
5468 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5469 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5470 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5471 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5472 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5473 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5474 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5475 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5476
5477 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5478 {
5479         unsigned short ggc;
5480
5481         if (pci_read_config_word(dev, GGC, &ggc))
5482                 return;
5483
5484         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5485                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5486                 dmar_map_gfx = 0;
5487         } else if (dmar_map_gfx) {
5488                 /* we have to ensure the gfx device is idle before we flush */
5489                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5490                 intel_iommu_strict = 1;
5491        }
5492 }
5493 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5494 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5495 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5496 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5497
5498 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5499    ISOCH DMAR unit for the Azalia sound device, but not give it any
5500    TLB entries, which causes it to deadlock. Check for that.  We do
5501    this in a function called from init_dmars(), instead of in a PCI
5502    quirk, because we don't want to print the obnoxious "BIOS broken"
5503    message if VT-d is actually disabled.
5504 */
5505 static void __init check_tylersburg_isoch(void)
5506 {
5507         struct pci_dev *pdev;
5508         uint32_t vtisochctrl;
5509
5510         /* If there's no Azalia in the system anyway, forget it. */
5511         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5512         if (!pdev)
5513                 return;
5514         pci_dev_put(pdev);
5515
5516         /* System Management Registers. Might be hidden, in which case
5517            we can't do the sanity check. But that's OK, because the
5518            known-broken BIOSes _don't_ actually hide it, so far. */
5519         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5520         if (!pdev)
5521                 return;
5522
5523         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5524                 pci_dev_put(pdev);
5525                 return;
5526         }
5527
5528         pci_dev_put(pdev);
5529
5530         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5531         if (vtisochctrl & 1)
5532                 return;
5533
5534         /* Drop all bits other than the number of TLB entries */
5535         vtisochctrl &= 0x1c;
5536
5537         /* If we have the recommended number of TLB entries (16), fine. */
5538         if (vtisochctrl == 0x10)
5539                 return;
5540
5541         /* Zero TLB entries? You get to ride the short bus to school. */
5542         if (!vtisochctrl) {
5543                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5544                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5545                      dmi_get_system_info(DMI_BIOS_VENDOR),
5546                      dmi_get_system_info(DMI_BIOS_VERSION),
5547                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5548                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5549                 return;
5550         }
5551
5552         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5553                vtisochctrl);
5554 }