2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
26 * These routines are used by both DMA-remapping and Interrupt-remapping
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <asm/irq_remapping.h>
42 #include <asm/iommu_table.h>
44 #include "irq_remapping.h"
46 /* No locks are needed as DMA remapping hardware unit
47 * list is constructed at boot time and hotplug of
48 * these units are not supported by the architecture.
50 LIST_HEAD(dmar_drhd_units);
52 struct acpi_table_header * __initdata dmar_tbl;
53 static acpi_size dmar_tbl_size;
55 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
58 * add INCLUDE_ALL at the tail, so scan the list will find it at
61 if (drhd->include_all)
62 list_add_tail(&drhd->list, &dmar_drhd_units);
64 list_add(&drhd->list, &dmar_drhd_units);
67 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
68 struct pci_dev **dev, u16 segment)
71 struct pci_dev *pdev = NULL;
72 struct acpi_dmar_pci_path *path;
75 bus = pci_find_bus(segment, scope->bus);
76 path = (struct acpi_dmar_pci_path *)(scope + 1);
77 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
78 / sizeof(struct acpi_dmar_pci_path);
84 * Some BIOSes list non-exist devices in DMAR table, just
88 pr_warn("Device scope bus [%d] not found\n", scope->bus);
91 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
93 /* warning will be printed below */
98 bus = pdev->subordinate;
101 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
110 pr_warn("Device scope type does not match for %s\n",
118 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119 struct pci_dev ***devices, u16 segment)
121 struct acpi_dmar_device_scope *scope;
127 while (start < end) {
129 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
132 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
133 scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
134 pr_warn("Unsupported device scope\n");
136 start += scope->length;
141 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
147 while (start < end) {
149 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
150 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
151 ret = dmar_parse_one_dev_scope(scope,
152 &(*devices)[index], segment);
159 start += scope->length;
166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
167 * structure which uniquely represent one DMA remapping hardware unit
168 * present in the platform
171 dmar_parse_one_drhd(struct acpi_dmar_header *header)
173 struct acpi_dmar_hardware_unit *drhd;
174 struct dmar_drhd_unit *dmaru;
177 drhd = (struct acpi_dmar_hardware_unit *)header;
178 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
183 dmaru->reg_base_addr = drhd->address;
184 dmaru->segment = drhd->segment;
185 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
187 ret = alloc_iommu(dmaru);
192 dmar_register_drhd_unit(dmaru);
196 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
198 struct acpi_dmar_hardware_unit *drhd;
201 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
203 if (dmaru->include_all)
206 ret = dmar_parse_dev_scope((void *)(drhd + 1),
207 ((void *)drhd) + drhd->header.length,
208 &dmaru->devices_cnt, &dmaru->devices,
211 list_del(&dmaru->list);
217 #ifdef CONFIG_ACPI_NUMA
219 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
221 struct acpi_dmar_rhsa *rhsa;
222 struct dmar_drhd_unit *drhd;
224 rhsa = (struct acpi_dmar_rhsa *)header;
225 for_each_drhd_unit(drhd) {
226 if (drhd->reg_base_addr == rhsa->base_address) {
227 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
229 if (!node_online(node))
231 drhd->iommu->node = node;
236 1, TAINT_FIRMWARE_WORKAROUND,
237 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
238 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
240 dmi_get_system_info(DMI_BIOS_VENDOR),
241 dmi_get_system_info(DMI_BIOS_VERSION),
242 dmi_get_system_info(DMI_PRODUCT_VERSION));
249 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
251 struct acpi_dmar_hardware_unit *drhd;
252 struct acpi_dmar_reserved_memory *rmrr;
253 struct acpi_dmar_atsr *atsr;
254 struct acpi_dmar_rhsa *rhsa;
256 switch (header->type) {
257 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
258 drhd = container_of(header, struct acpi_dmar_hardware_unit,
260 pr_info("DRHD base: %#016Lx flags: %#x\n",
261 (unsigned long long)drhd->address, drhd->flags);
263 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
264 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
266 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
267 (unsigned long long)rmrr->base_address,
268 (unsigned long long)rmrr->end_address);
270 case ACPI_DMAR_TYPE_ATSR:
271 atsr = container_of(header, struct acpi_dmar_atsr, header);
272 pr_info("ATSR flags: %#x\n", atsr->flags);
274 case ACPI_DMAR_HARDWARE_AFFINITY:
275 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
276 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
277 (unsigned long long)rhsa->base_address,
278 rhsa->proximity_domain);
284 * dmar_table_detect - checks to see if the platform supports DMAR devices
286 static int __init dmar_table_detect(void)
288 acpi_status status = AE_OK;
290 /* if we could find DMAR table, then there are DMAR devices */
291 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
292 (struct acpi_table_header **)&dmar_tbl,
295 if (ACPI_SUCCESS(status) && !dmar_tbl) {
296 pr_warn("Unable to map DMAR\n");
297 status = AE_NOT_FOUND;
300 return (ACPI_SUCCESS(status) ? 1 : 0);
304 * parse_dmar_table - parses the DMA reporting table
307 parse_dmar_table(void)
309 struct acpi_table_dmar *dmar;
310 struct acpi_dmar_header *entry_header;
314 * Do it again, earlier dmar_tbl mapping could be mapped with
320 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
321 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
323 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
325 dmar = (struct acpi_table_dmar *)dmar_tbl;
329 if (dmar->width < PAGE_SHIFT - 1) {
330 pr_warn("Invalid DMAR haw\n");
334 pr_info("Host address width %d\n", dmar->width + 1);
336 entry_header = (struct acpi_dmar_header *)(dmar + 1);
337 while (((unsigned long)entry_header) <
338 (((unsigned long)dmar) + dmar_tbl->length)) {
339 /* Avoid looping forever on bad ACPI tables */
340 if (entry_header->length == 0) {
341 pr_warn("Invalid 0-length structure\n");
346 dmar_table_print_dmar_entry(entry_header);
348 switch (entry_header->type) {
349 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
350 ret = dmar_parse_one_drhd(entry_header);
352 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
353 ret = dmar_parse_one_rmrr(entry_header);
355 case ACPI_DMAR_TYPE_ATSR:
356 ret = dmar_parse_one_atsr(entry_header);
358 case ACPI_DMAR_HARDWARE_AFFINITY:
359 #ifdef CONFIG_ACPI_NUMA
360 ret = dmar_parse_one_rhsa(entry_header);
364 pr_warn("Unknown DMAR structure type %d\n",
366 ret = 0; /* for forward compatibility */
372 entry_header = ((void *)entry_header + entry_header->length);
377 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
383 for (index = 0; index < cnt; index++)
384 if (dev == devices[index])
387 /* Check our parent */
388 dev = dev->bus->self;
394 struct dmar_drhd_unit *
395 dmar_find_matched_drhd_unit(struct pci_dev *dev)
397 struct dmar_drhd_unit *dmaru = NULL;
398 struct acpi_dmar_hardware_unit *drhd;
400 dev = pci_physfn(dev);
402 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
403 drhd = container_of(dmaru->hdr,
404 struct acpi_dmar_hardware_unit,
407 if (dmaru->include_all &&
408 drhd->segment == pci_domain_nr(dev->bus))
411 if (dmar_pci_device_match(dmaru->devices,
412 dmaru->devices_cnt, dev))
419 int __init dmar_dev_scope_init(void)
421 static int dmar_dev_scope_initialized;
422 struct dmar_drhd_unit *drhd, *drhd_n;
425 if (dmar_dev_scope_initialized)
426 return dmar_dev_scope_initialized;
428 if (list_empty(&dmar_drhd_units))
431 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
432 ret = dmar_parse_dev(drhd);
437 ret = dmar_parse_rmrr_atsr_dev();
441 dmar_dev_scope_initialized = 1;
445 dmar_dev_scope_initialized = ret;
450 int __init dmar_table_init(void)
452 static int dmar_table_initialized;
455 if (dmar_table_initialized)
458 dmar_table_initialized = 1;
460 ret = parse_dmar_table();
463 pr_info("parse DMAR table failure.\n");
467 if (list_empty(&dmar_drhd_units)) {
468 pr_info("No DMAR devices found\n");
475 static void warn_invalid_dmar(u64 addr, const char *message)
478 1, TAINT_FIRMWARE_WORKAROUND,
479 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
480 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
482 dmi_get_system_info(DMI_BIOS_VENDOR),
483 dmi_get_system_info(DMI_BIOS_VERSION),
484 dmi_get_system_info(DMI_PRODUCT_VERSION));
487 int __init check_zero_address(void)
489 struct acpi_table_dmar *dmar;
490 struct acpi_dmar_header *entry_header;
491 struct acpi_dmar_hardware_unit *drhd;
493 dmar = (struct acpi_table_dmar *)dmar_tbl;
494 entry_header = (struct acpi_dmar_header *)(dmar + 1);
496 while (((unsigned long)entry_header) <
497 (((unsigned long)dmar) + dmar_tbl->length)) {
498 /* Avoid looping forever on bad ACPI tables */
499 if (entry_header->length == 0) {
500 pr_warn("Invalid 0-length structure\n");
504 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
508 drhd = (void *)entry_header;
509 if (!drhd->address) {
510 warn_invalid_dmar(0, "");
514 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
516 printk("IOMMU: can't validate: %llx\n", drhd->address);
519 cap = dmar_readq(addr + DMAR_CAP_REG);
520 ecap = dmar_readq(addr + DMAR_ECAP_REG);
521 early_iounmap(addr, VTD_PAGE_SIZE);
522 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
523 warn_invalid_dmar(drhd->address,
524 " returns all ones");
529 entry_header = ((void *)entry_header + entry_header->length);
537 int __init detect_intel_iommu(void)
541 ret = dmar_table_detect();
543 ret = check_zero_address();
545 struct acpi_table_dmar *dmar;
547 dmar = (struct acpi_table_dmar *) dmar_tbl;
549 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
551 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
553 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
555 /* Make sure ACS will be enabled */
561 x86_init.iommu.iommu_init = intel_iommu_init;
564 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
567 return ret ? 1 : -ENODEV;
571 static void unmap_iommu(struct intel_iommu *iommu)
574 release_mem_region(iommu->reg_phys, iommu->reg_size);
578 * map_iommu: map the iommu's registers
579 * @iommu: the iommu to map
580 * @phys_addr: the physical address of the base resgister
582 * Memory map the iommu's registers. Start w/ a single page, and
583 * possibly expand if that turns out to be insufficent.
585 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
589 iommu->reg_phys = phys_addr;
590 iommu->reg_size = VTD_PAGE_SIZE;
592 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
593 pr_err("IOMMU: can't reserve memory\n");
598 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
600 pr_err("IOMMU: can't map the region\n");
605 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
606 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
608 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
610 warn_invalid_dmar(phys_addr, " returns all ones");
614 /* the registers might be more than one page */
615 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
616 cap_max_fault_reg_offset(iommu->cap));
617 map_size = VTD_PAGE_ALIGN(map_size);
618 if (map_size > iommu->reg_size) {
620 release_mem_region(iommu->reg_phys, iommu->reg_size);
621 iommu->reg_size = map_size;
622 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
624 pr_err("IOMMU: can't reserve memory\n");
628 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
630 pr_err("IOMMU: can't map the region\n");
641 release_mem_region(iommu->reg_phys, iommu->reg_size);
646 int alloc_iommu(struct dmar_drhd_unit *drhd)
648 struct intel_iommu *iommu;
650 static int iommu_allocated = 0;
655 if (!drhd->reg_base_addr) {
656 warn_invalid_dmar(0, "");
660 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
664 iommu->seq_id = iommu_allocated++;
665 sprintf (iommu->name, "dmar%d", iommu->seq_id);
667 err = map_iommu(iommu, drhd->reg_base_addr);
669 pr_err("IOMMU: failed to map %s\n", iommu->name);
674 agaw = iommu_calculate_agaw(iommu);
676 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
680 msagaw = iommu_calculate_max_sagaw(iommu);
682 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
687 iommu->msagaw = msagaw;
691 ver = readl(iommu->reg + DMAR_VER_REG);
692 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
694 (unsigned long long)drhd->reg_base_addr,
695 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
696 (unsigned long long)iommu->cap,
697 (unsigned long long)iommu->ecap);
699 raw_spin_lock_init(&iommu->register_lock);
711 void free_iommu(struct intel_iommu *iommu)
716 free_dmar_iommu(iommu);
725 * Reclaim all the submitted descriptors which have completed its work.
727 static inline void reclaim_free_desc(struct q_inval *qi)
729 while (qi->desc_status[qi->free_tail] == QI_DONE ||
730 qi->desc_status[qi->free_tail] == QI_ABORT) {
731 qi->desc_status[qi->free_tail] = QI_FREE;
732 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
737 static int qi_check_fault(struct intel_iommu *iommu, int index)
741 struct q_inval *qi = iommu->qi;
742 int wait_index = (index + 1) % QI_LENGTH;
744 if (qi->desc_status[wait_index] == QI_ABORT)
747 fault = readl(iommu->reg + DMAR_FSTS_REG);
750 * If IQE happens, the head points to the descriptor associated
751 * with the error. No new descriptors are fetched until the IQE
754 if (fault & DMA_FSTS_IQE) {
755 head = readl(iommu->reg + DMAR_IQH_REG);
756 if ((head >> DMAR_IQ_SHIFT) == index) {
757 pr_err("VT-d detected invalid descriptor: "
758 "low=%llx, high=%llx\n",
759 (unsigned long long)qi->desc[index].low,
760 (unsigned long long)qi->desc[index].high);
761 memcpy(&qi->desc[index], &qi->desc[wait_index],
762 sizeof(struct qi_desc));
763 __iommu_flush_cache(iommu, &qi->desc[index],
764 sizeof(struct qi_desc));
765 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
771 * If ITE happens, all pending wait_desc commands are aborted.
772 * No new descriptors are fetched until the ITE is cleared.
774 if (fault & DMA_FSTS_ITE) {
775 head = readl(iommu->reg + DMAR_IQH_REG);
776 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
778 tail = readl(iommu->reg + DMAR_IQT_REG);
779 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
781 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
784 if (qi->desc_status[head] == QI_IN_USE)
785 qi->desc_status[head] = QI_ABORT;
786 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
787 } while (head != tail);
789 if (qi->desc_status[wait_index] == QI_ABORT)
793 if (fault & DMA_FSTS_ICE)
794 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
800 * Submit the queued invalidation descriptor to the remapping
801 * hardware unit and wait for its completion.
803 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
806 struct q_inval *qi = iommu->qi;
807 struct qi_desc *hw, wait_desc;
808 int wait_index, index;
819 raw_spin_lock_irqsave(&qi->q_lock, flags);
820 while (qi->free_cnt < 3) {
821 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
823 raw_spin_lock_irqsave(&qi->q_lock, flags);
826 index = qi->free_head;
827 wait_index = (index + 1) % QI_LENGTH;
829 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
833 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
834 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
835 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
837 hw[wait_index] = wait_desc;
839 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
840 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
842 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
846 * update the HW tail register indicating the presence of
849 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
851 while (qi->desc_status[wait_index] != QI_DONE) {
853 * We will leave the interrupts disabled, to prevent interrupt
854 * context to queue another cmd while a cmd is already submitted
855 * and waiting for completion on this cpu. This is to avoid
856 * a deadlock where the interrupt context can wait indefinitely
857 * for free slots in the queue.
859 rc = qi_check_fault(iommu, index);
863 raw_spin_unlock(&qi->q_lock);
865 raw_spin_lock(&qi->q_lock);
868 qi->desc_status[index] = QI_DONE;
870 reclaim_free_desc(qi);
871 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
880 * Flush the global interrupt entry cache.
882 void qi_global_iec(struct intel_iommu *iommu)
886 desc.low = QI_IEC_TYPE;
889 /* should never fail */
890 qi_submit_sync(&desc, iommu);
893 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
898 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
899 | QI_CC_GRAN(type) | QI_CC_TYPE;
902 qi_submit_sync(&desc, iommu);
905 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
906 unsigned int size_order, u64 type)
913 if (cap_write_drain(iommu->cap))
916 if (cap_read_drain(iommu->cap))
919 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
920 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
921 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
922 | QI_IOTLB_AM(size_order);
924 qi_submit_sync(&desc, iommu);
927 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
928 u64 addr, unsigned mask)
933 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
934 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
935 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
937 desc.high = QI_DEV_IOTLB_ADDR(addr);
939 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
942 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
945 qi_submit_sync(&desc, iommu);
949 * Disable Queued Invalidation interface.
951 void dmar_disable_qi(struct intel_iommu *iommu)
955 cycles_t start_time = get_cycles();
957 if (!ecap_qis(iommu->ecap))
960 raw_spin_lock_irqsave(&iommu->register_lock, flags);
962 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
963 if (!(sts & DMA_GSTS_QIES))
967 * Give a chance to HW to complete the pending invalidation requests.
969 while ((readl(iommu->reg + DMAR_IQT_REG) !=
970 readl(iommu->reg + DMAR_IQH_REG)) &&
971 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
974 iommu->gcmd &= ~DMA_GCMD_QIE;
975 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
977 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
978 !(sts & DMA_GSTS_QIES), sts);
980 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
984 * Enable queued invalidation.
986 static void __dmar_enable_qi(struct intel_iommu *iommu)
990 struct q_inval *qi = iommu->qi;
992 qi->free_head = qi->free_tail = 0;
993 qi->free_cnt = QI_LENGTH;
995 raw_spin_lock_irqsave(&iommu->register_lock, flags);
997 /* write zero to the tail reg */
998 writel(0, iommu->reg + DMAR_IQT_REG);
1000 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1002 iommu->gcmd |= DMA_GCMD_QIE;
1003 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1005 /* Make sure hardware complete it */
1006 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1008 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1012 * Enable Queued Invalidation interface. This is a must to support
1013 * interrupt-remapping. Also used by DMA-remapping, which replaces
1014 * register based IOTLB invalidation.
1016 int dmar_enable_qi(struct intel_iommu *iommu)
1019 struct page *desc_page;
1021 if (!ecap_qis(iommu->ecap))
1025 * queued invalidation is already setup and enabled.
1030 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1037 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1044 qi->desc = page_address(desc_page);
1046 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1047 if (!qi->desc_status) {
1048 free_page((unsigned long) qi->desc);
1054 qi->free_head = qi->free_tail = 0;
1055 qi->free_cnt = QI_LENGTH;
1057 raw_spin_lock_init(&qi->q_lock);
1059 __dmar_enable_qi(iommu);
1064 /* iommu interrupt handling. Most stuff are MSI-like. */
1072 static const char *dma_remap_fault_reasons[] =
1075 "Present bit in root entry is clear",
1076 "Present bit in context entry is clear",
1077 "Invalid context entry",
1078 "Access beyond MGAW",
1079 "PTE Write access is not set",
1080 "PTE Read access is not set",
1081 "Next page table ptr is invalid",
1082 "Root table address invalid",
1083 "Context table ptr is invalid",
1084 "non-zero reserved fields in RTP",
1085 "non-zero reserved fields in CTP",
1086 "non-zero reserved fields in PTE",
1087 "PCE for translation request specifies blocking",
1090 static const char *irq_remap_fault_reasons[] =
1092 "Detected reserved fields in the decoded interrupt-remapped request",
1093 "Interrupt index exceeded the interrupt-remapping table size",
1094 "Present field in the IRTE entry is clear",
1095 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1096 "Detected reserved fields in the IRTE entry",
1097 "Blocked a compatibility format interrupt request",
1098 "Blocked an interrupt request due to source-id verification failure",
1101 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1103 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1105 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1106 ARRAY_SIZE(irq_remap_fault_reasons))) {
1107 *fault_type = INTR_REMAP;
1108 return irq_remap_fault_reasons[fault_reason - 0x20];
1109 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1110 *fault_type = DMA_REMAP;
1111 return dma_remap_fault_reasons[fault_reason];
1113 *fault_type = UNKNOWN;
1118 void dmar_msi_unmask(struct irq_data *data)
1120 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1124 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1125 writel(0, iommu->reg + DMAR_FECTL_REG);
1126 /* Read a reg to force flush the post write */
1127 readl(iommu->reg + DMAR_FECTL_REG);
1128 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1131 void dmar_msi_mask(struct irq_data *data)
1134 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1137 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1138 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1139 /* Read a reg to force flush the post write */
1140 readl(iommu->reg + DMAR_FECTL_REG);
1141 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1144 void dmar_msi_write(int irq, struct msi_msg *msg)
1146 struct intel_iommu *iommu = irq_get_handler_data(irq);
1149 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1150 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1151 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1152 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1153 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1156 void dmar_msi_read(int irq, struct msi_msg *msg)
1158 struct intel_iommu *iommu = irq_get_handler_data(irq);
1161 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1162 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1163 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1164 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1165 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1168 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1169 u8 fault_reason, u16 source_id, unsigned long long addr)
1174 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1176 if (fault_type == INTR_REMAP)
1177 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1178 "fault index %llx\n"
1179 "INTR-REMAP:[fault reason %02d] %s\n",
1180 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1181 PCI_FUNC(source_id & 0xFF), addr >> 48,
1182 fault_reason, reason);
1184 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1185 "fault addr %llx \n"
1186 "DMAR:[fault reason %02d] %s\n",
1187 (type ? "DMA Read" : "DMA Write"),
1188 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1189 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1193 #define PRIMARY_FAULT_REG_LEN (16)
1194 irqreturn_t dmar_fault(int irq, void *dev_id)
1196 struct intel_iommu *iommu = dev_id;
1197 int reg, fault_index;
1201 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1202 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1204 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1206 /* TBD: ignore advanced fault log currently */
1207 if (!(fault_status & DMA_FSTS_PPF))
1210 fault_index = dma_fsts_fault_record_index(fault_status);
1211 reg = cap_fault_reg_offset(iommu->cap);
1219 /* highest 32 bits */
1220 data = readl(iommu->reg + reg +
1221 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1222 if (!(data & DMA_FRCD_F))
1225 fault_reason = dma_frcd_fault_reason(data);
1226 type = dma_frcd_type(data);
1228 data = readl(iommu->reg + reg +
1229 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1230 source_id = dma_frcd_source_id(data);
1232 guest_addr = dmar_readq(iommu->reg + reg +
1233 fault_index * PRIMARY_FAULT_REG_LEN);
1234 guest_addr = dma_frcd_page_addr(guest_addr);
1235 /* clear the fault */
1236 writel(DMA_FRCD_F, iommu->reg + reg +
1237 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1239 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1241 dmar_fault_do_one(iommu, type, fault_reason,
1242 source_id, guest_addr);
1245 if (fault_index >= cap_num_fault_regs(iommu->cap))
1247 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1250 /* clear all the other faults */
1251 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1252 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1254 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1258 int dmar_set_interrupt(struct intel_iommu *iommu)
1263 * Check if the fault interrupt is already initialized.
1270 pr_err("IOMMU: no free vectors\n");
1274 irq_set_handler_data(irq, iommu);
1277 ret = arch_setup_dmar_msi(irq);
1279 irq_set_handler_data(irq, NULL);
1285 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1287 pr_err("IOMMU: can't request irq\n");
1291 int __init enable_drhd_fault_handling(void)
1293 struct dmar_drhd_unit *drhd;
1296 * Enable fault control interrupt.
1298 for_each_drhd_unit(drhd) {
1300 struct intel_iommu *iommu = drhd->iommu;
1301 ret = dmar_set_interrupt(iommu);
1304 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1305 (unsigned long long)drhd->reg_base_addr, ret);
1310 * Clear any previous faults.
1312 dmar_fault(iommu->irq, iommu);
1319 * Re-enable Queued Invalidation interface.
1321 int dmar_reenable_qi(struct intel_iommu *iommu)
1323 if (!ecap_qis(iommu->ecap))
1330 * First disable queued invalidation.
1332 dmar_disable_qi(iommu);
1334 * Then enable queued invalidation again. Since there is no pending
1335 * invalidation requests now, it's safe to re-enable queued
1338 __dmar_enable_qi(iommu);
1344 * Check interrupt remapping support in DMAR table description.
1346 int __init dmar_ir_support(void)
1348 struct acpi_table_dmar *dmar;
1349 dmar = (struct acpi_table_dmar *)dmar_tbl;
1352 return dmar->flags & 0x1;
1354 IOMMU_INIT_POST(detect_intel_iommu);