2 * IOMMU API for ARM architected SMMU implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * Copyright (C) 2013 ARM Limited
19 * Author: Will Deacon <will.deacon@arm.com>
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
29 #define pr_fmt(fmt) "arm-smmu: " fmt
31 #include <linux/delay.h>
32 #include <linux/dma-iommu.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/err.h>
35 #include <linux/interrupt.h>
37 #include <linux/iommu.h>
38 #include <linux/iopoll.h>
39 #include <linux/module.h>
41 #include <linux/of_address.h>
42 #include <linux/pci.h>
43 #include <linux/platform_device.h>
44 #include <linux/slab.h>
45 #include <linux/spinlock.h>
47 #include <linux/amba/bus.h>
49 #include "io-pgtable.h"
51 /* Maximum number of stream IDs assigned to a single device */
52 #define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
54 /* Maximum number of context banks per SMMU */
55 #define ARM_SMMU_MAX_CBS 128
57 /* Maximum number of mapping groups per SMMU */
58 #define ARM_SMMU_MAX_SMRS 128
60 /* SMMU global address space */
61 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
62 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
65 * SMMU global address space with conditional offset to access secure
66 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
69 #define ARM_SMMU_GR0_NS(smmu) \
71 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
75 #define smmu_writeq writeq_relaxed
77 #define smmu_writeq(reg64, addr) \
79 u64 __val = (reg64); \
80 void __iomem *__addr = (addr); \
81 writel_relaxed(__val >> 32, __addr + 4); \
82 writel_relaxed(__val, __addr); \
86 /* Configuration registers */
87 #define ARM_SMMU_GR0_sCR0 0x0
88 #define sCR0_CLIENTPD (1 << 0)
89 #define sCR0_GFRE (1 << 1)
90 #define sCR0_GFIE (1 << 2)
91 #define sCR0_GCFGFRE (1 << 4)
92 #define sCR0_GCFGFIE (1 << 5)
93 #define sCR0_USFCFG (1 << 10)
94 #define sCR0_VMIDPNE (1 << 11)
95 #define sCR0_PTM (1 << 12)
96 #define sCR0_FB (1 << 13)
97 #define sCR0_BSU_SHIFT 14
98 #define sCR0_BSU_MASK 0x3
100 /* Identification registers */
101 #define ARM_SMMU_GR0_ID0 0x20
102 #define ARM_SMMU_GR0_ID1 0x24
103 #define ARM_SMMU_GR0_ID2 0x28
104 #define ARM_SMMU_GR0_ID3 0x2c
105 #define ARM_SMMU_GR0_ID4 0x30
106 #define ARM_SMMU_GR0_ID5 0x34
107 #define ARM_SMMU_GR0_ID6 0x38
108 #define ARM_SMMU_GR0_ID7 0x3c
109 #define ARM_SMMU_GR0_sGFSR 0x48
110 #define ARM_SMMU_GR0_sGFSYNR0 0x50
111 #define ARM_SMMU_GR0_sGFSYNR1 0x54
112 #define ARM_SMMU_GR0_sGFSYNR2 0x58
114 #define ID0_S1TS (1 << 30)
115 #define ID0_S2TS (1 << 29)
116 #define ID0_NTS (1 << 28)
117 #define ID0_SMS (1 << 27)
118 #define ID0_ATOSNS (1 << 26)
119 #define ID0_CTTW (1 << 14)
120 #define ID0_NUMIRPT_SHIFT 16
121 #define ID0_NUMIRPT_MASK 0xff
122 #define ID0_NUMSIDB_SHIFT 9
123 #define ID0_NUMSIDB_MASK 0xf
124 #define ID0_NUMSMRG_SHIFT 0
125 #define ID0_NUMSMRG_MASK 0xff
127 #define ID1_PAGESIZE (1 << 31)
128 #define ID1_NUMPAGENDXB_SHIFT 28
129 #define ID1_NUMPAGENDXB_MASK 7
130 #define ID1_NUMS2CB_SHIFT 16
131 #define ID1_NUMS2CB_MASK 0xff
132 #define ID1_NUMCB_SHIFT 0
133 #define ID1_NUMCB_MASK 0xff
135 #define ID2_OAS_SHIFT 4
136 #define ID2_OAS_MASK 0xf
137 #define ID2_IAS_SHIFT 0
138 #define ID2_IAS_MASK 0xf
139 #define ID2_UBS_SHIFT 8
140 #define ID2_UBS_MASK 0xf
141 #define ID2_PTFS_4K (1 << 12)
142 #define ID2_PTFS_16K (1 << 13)
143 #define ID2_PTFS_64K (1 << 14)
145 /* Global TLB invalidation */
146 #define ARM_SMMU_GR0_TLBIVMID 0x64
147 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
148 #define ARM_SMMU_GR0_TLBIALLH 0x6c
149 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
150 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
151 #define sTLBGSTATUS_GSACTIVE (1 << 0)
152 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
154 /* Stream mapping registers */
155 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
156 #define SMR_VALID (1 << 31)
157 #define SMR_MASK_SHIFT 16
158 #define SMR_MASK_MASK 0x7fff
159 #define SMR_ID_SHIFT 0
160 #define SMR_ID_MASK 0x7fff
162 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
163 #define S2CR_CBNDX_SHIFT 0
164 #define S2CR_CBNDX_MASK 0xff
165 #define S2CR_TYPE_SHIFT 16
166 #define S2CR_TYPE_MASK 0x3
167 #define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT)
168 #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT)
169 #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT)
171 #define S2CR_PRIVCFG_SHIFT 24
172 #define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT)
174 /* Context bank attribute registers */
175 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
176 #define CBAR_VMID_SHIFT 0
177 #define CBAR_VMID_MASK 0xff
178 #define CBAR_S1_BPSHCFG_SHIFT 8
179 #define CBAR_S1_BPSHCFG_MASK 3
180 #define CBAR_S1_BPSHCFG_NSH 3
181 #define CBAR_S1_MEMATTR_SHIFT 12
182 #define CBAR_S1_MEMATTR_MASK 0xf
183 #define CBAR_S1_MEMATTR_WB 0xf
184 #define CBAR_TYPE_SHIFT 16
185 #define CBAR_TYPE_MASK 0x3
186 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
187 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
188 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
189 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
190 #define CBAR_IRPTNDX_SHIFT 24
191 #define CBAR_IRPTNDX_MASK 0xff
193 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
194 #define CBA2R_RW64_32BIT (0 << 0)
195 #define CBA2R_RW64_64BIT (1 << 0)
197 /* Translation context bank */
198 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
199 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
201 #define ARM_SMMU_CB_SCTLR 0x0
202 #define ARM_SMMU_CB_RESUME 0x8
203 #define ARM_SMMU_CB_TTBCR2 0x10
204 #define ARM_SMMU_CB_TTBR0 0x20
205 #define ARM_SMMU_CB_TTBR1 0x28
206 #define ARM_SMMU_CB_TTBCR 0x30
207 #define ARM_SMMU_CB_S1_MAIR0 0x38
208 #define ARM_SMMU_CB_S1_MAIR1 0x3c
209 #define ARM_SMMU_CB_PAR_LO 0x50
210 #define ARM_SMMU_CB_PAR_HI 0x54
211 #define ARM_SMMU_CB_FSR 0x58
212 #define ARM_SMMU_CB_FAR_LO 0x60
213 #define ARM_SMMU_CB_FAR_HI 0x64
214 #define ARM_SMMU_CB_FSYNR0 0x68
215 #define ARM_SMMU_CB_S1_TLBIVA 0x600
216 #define ARM_SMMU_CB_S1_TLBIASID 0x610
217 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
218 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
219 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
220 #define ARM_SMMU_CB_ATS1PR 0x800
221 #define ARM_SMMU_CB_ATSR 0x8f0
223 #define SCTLR_S1_ASIDPNE (1 << 12)
224 #define SCTLR_CFCFG (1 << 7)
225 #define SCTLR_CFIE (1 << 6)
226 #define SCTLR_CFRE (1 << 5)
227 #define SCTLR_E (1 << 4)
228 #define SCTLR_AFE (1 << 2)
229 #define SCTLR_TRE (1 << 1)
230 #define SCTLR_M (1 << 0)
231 #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE)
233 #define CB_PAR_F (1 << 0)
235 #define ATSR_ACTIVE (1 << 0)
237 #define RESUME_RETRY (0 << 0)
238 #define RESUME_TERMINATE (1 << 0)
240 #define TTBCR2_SEP_SHIFT 15
241 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
243 #define TTBRn_ASID_SHIFT 48
245 #define FSR_MULTI (1 << 31)
246 #define FSR_SS (1 << 30)
247 #define FSR_UUT (1 << 8)
248 #define FSR_ASF (1 << 7)
249 #define FSR_TLBLKF (1 << 6)
250 #define FSR_TLBMCF (1 << 5)
251 #define FSR_EF (1 << 4)
252 #define FSR_PF (1 << 3)
253 #define FSR_AFF (1 << 2)
254 #define FSR_TF (1 << 1)
256 #define FSR_IGN (FSR_AFF | FSR_ASF | \
257 FSR_TLBMCF | FSR_TLBLKF)
258 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
259 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
261 #define FSYNR0_WNR (1 << 4)
263 static int force_stage;
264 module_param(force_stage, int, S_IRUGO);
265 MODULE_PARM_DESC(force_stage,
266 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
267 static bool disable_bypass;
268 module_param(disable_bypass, bool, S_IRUGO);
269 MODULE_PARM_DESC(disable_bypass,
270 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
272 enum arm_smmu_arch_version {
277 struct arm_smmu_smr {
283 struct arm_smmu_master_cfg {
285 u16 streamids[MAX_MASTER_STREAMIDS];
286 struct arm_smmu_smr *smrs;
289 struct arm_smmu_master {
290 struct device_node *of_node;
292 struct arm_smmu_master_cfg cfg;
295 struct arm_smmu_device {
300 unsigned long pgshift;
302 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
303 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
304 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
305 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
306 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
307 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
310 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
312 enum arm_smmu_arch_version version;
314 u32 num_context_banks;
315 u32 num_s2_context_banks;
316 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
319 u32 num_mapping_groups;
320 DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
322 unsigned long va_size;
323 unsigned long ipa_size;
324 unsigned long pa_size;
327 u32 num_context_irqs;
330 struct list_head list;
331 struct rb_root masters;
334 struct arm_smmu_cfg {
339 #define INVALID_IRPTNDX 0xff
341 #define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx)
342 #define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1)
344 enum arm_smmu_domain_stage {
345 ARM_SMMU_DOMAIN_S1 = 0,
347 ARM_SMMU_DOMAIN_NESTED,
350 struct arm_smmu_domain {
351 struct arm_smmu_device *smmu;
352 struct io_pgtable_ops *pgtbl_ops;
353 spinlock_t pgtbl_lock;
354 struct arm_smmu_cfg cfg;
355 enum arm_smmu_domain_stage stage;
356 struct mutex init_mutex; /* Protects smmu pointer */
357 struct iommu_domain domain;
360 static struct iommu_ops arm_smmu_ops;
362 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
363 static LIST_HEAD(arm_smmu_devices);
365 struct arm_smmu_option_prop {
370 static struct arm_smmu_option_prop arm_smmu_options[] = {
371 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
375 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
377 return container_of(dom, struct arm_smmu_domain, domain);
380 static void parse_driver_options(struct arm_smmu_device *smmu)
385 if (of_property_read_bool(smmu->dev->of_node,
386 arm_smmu_options[i].prop)) {
387 smmu->options |= arm_smmu_options[i].opt;
388 dev_notice(smmu->dev, "option %s\n",
389 arm_smmu_options[i].prop);
391 } while (arm_smmu_options[++i].opt);
394 static struct device_node *dev_get_dev_node(struct device *dev)
396 if (dev_is_pci(dev)) {
397 struct pci_bus *bus = to_pci_dev(dev)->bus;
399 while (!pci_is_root_bus(bus))
401 return bus->bridge->parent->of_node;
407 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
408 struct device_node *dev_node)
410 struct rb_node *node = smmu->masters.rb_node;
413 struct arm_smmu_master *master;
415 master = container_of(node, struct arm_smmu_master, node);
417 if (dev_node < master->of_node)
418 node = node->rb_left;
419 else if (dev_node > master->of_node)
420 node = node->rb_right;
428 static struct arm_smmu_master_cfg *
429 find_smmu_master_cfg(struct device *dev)
431 struct arm_smmu_master_cfg *cfg = NULL;
432 struct iommu_group *group = iommu_group_get(dev);
435 cfg = iommu_group_get_iommudata(group);
436 iommu_group_put(group);
442 static int insert_smmu_master(struct arm_smmu_device *smmu,
443 struct arm_smmu_master *master)
445 struct rb_node **new, *parent;
447 new = &smmu->masters.rb_node;
450 struct arm_smmu_master *this
451 = container_of(*new, struct arm_smmu_master, node);
454 if (master->of_node < this->of_node)
455 new = &((*new)->rb_left);
456 else if (master->of_node > this->of_node)
457 new = &((*new)->rb_right);
462 rb_link_node(&master->node, parent, new);
463 rb_insert_color(&master->node, &smmu->masters);
467 static int register_smmu_master(struct arm_smmu_device *smmu,
469 struct of_phandle_args *masterspec)
472 struct arm_smmu_master *master;
474 master = find_smmu_master(smmu, masterspec->np);
477 "rejecting multiple registrations for master device %s\n",
478 masterspec->np->name);
482 if (masterspec->args_count > MAX_MASTER_STREAMIDS) {
484 "reached maximum number (%d) of stream IDs for master device %s\n",
485 MAX_MASTER_STREAMIDS, masterspec->np->name);
489 master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
493 master->of_node = masterspec->np;
494 master->cfg.num_streamids = masterspec->args_count;
496 for (i = 0; i < master->cfg.num_streamids; ++i) {
497 u16 streamid = masterspec->args[i];
499 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&
500 (streamid >= smmu->num_mapping_groups)) {
502 "stream ID for master device %s greater than maximum allowed (%d)\n",
503 masterspec->np->name, smmu->num_mapping_groups);
506 master->cfg.streamids[i] = streamid;
508 return insert_smmu_master(smmu, master);
511 static struct arm_smmu_device *find_smmu_for_device(struct device *dev)
513 struct arm_smmu_device *smmu;
514 struct arm_smmu_master *master = NULL;
515 struct device_node *dev_node = dev_get_dev_node(dev);
517 spin_lock(&arm_smmu_devices_lock);
518 list_for_each_entry(smmu, &arm_smmu_devices, list) {
519 master = find_smmu_master(smmu, dev_node);
523 spin_unlock(&arm_smmu_devices_lock);
525 return master ? smmu : NULL;
528 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
533 idx = find_next_zero_bit(map, end, start);
536 } while (test_and_set_bit(idx, map));
541 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
546 /* Wait for any pending TLB invalidations to complete */
547 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
550 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
552 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
553 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
554 & sTLBGSTATUS_GSACTIVE) {
556 if (++count == TLB_LOOP_TIMEOUT) {
557 dev_err_ratelimited(smmu->dev,
558 "TLB sync timed out -- SMMU may be deadlocked\n");
565 static void arm_smmu_tlb_sync(void *cookie)
567 struct arm_smmu_domain *smmu_domain = cookie;
568 __arm_smmu_tlb_sync(smmu_domain->smmu);
571 static void arm_smmu_tlb_inv_context(void *cookie)
573 struct arm_smmu_domain *smmu_domain = cookie;
574 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
575 struct arm_smmu_device *smmu = smmu_domain->smmu;
576 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
580 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
581 writel_relaxed(ARM_SMMU_CB_ASID(cfg),
582 base + ARM_SMMU_CB_S1_TLBIASID);
584 base = ARM_SMMU_GR0(smmu);
585 writel_relaxed(ARM_SMMU_CB_VMID(cfg),
586 base + ARM_SMMU_GR0_TLBIVMID);
589 __arm_smmu_tlb_sync(smmu);
592 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
593 size_t granule, bool leaf, void *cookie)
595 struct arm_smmu_domain *smmu_domain = cookie;
596 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
597 struct arm_smmu_device *smmu = smmu_domain->smmu;
598 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
602 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
603 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
605 if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {
607 iova |= ARM_SMMU_CB_ASID(cfg);
609 writel_relaxed(iova, reg);
611 } while (size -= granule);
615 iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48;
617 writeq_relaxed(iova, reg);
618 iova += granule >> 12;
619 } while (size -= granule);
623 } else if (smmu->version == ARM_SMMU_V2) {
624 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
625 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
626 ARM_SMMU_CB_S2_TLBIIPAS2;
629 writeq_relaxed(iova, reg);
630 iova += granule >> 12;
631 } while (size -= granule);
634 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
635 writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg);
639 static struct iommu_gather_ops arm_smmu_gather_ops = {
640 .tlb_flush_all = arm_smmu_tlb_inv_context,
641 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
642 .tlb_sync = arm_smmu_tlb_sync,
645 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
648 u32 fsr, far, fsynr, resume;
650 struct iommu_domain *domain = dev;
651 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
652 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
653 struct arm_smmu_device *smmu = smmu_domain->smmu;
654 void __iomem *cb_base;
656 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
657 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
659 if (!(fsr & FSR_FAULT))
663 dev_err_ratelimited(smmu->dev,
664 "Unexpected context fault (fsr 0x%x)\n",
667 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
668 flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
670 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO);
673 far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI);
674 iova |= ((unsigned long)far << 32);
677 if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {
679 resume = RESUME_RETRY;
681 dev_err_ratelimited(smmu->dev,
682 "Unhandled context fault: iova=0x%08lx, fsynr=0x%x, cb=%d\n",
683 iova, fsynr, cfg->cbndx);
685 resume = RESUME_TERMINATE;
688 /* Clear the faulting FSR */
689 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
691 /* Retry or terminate any stalled transactions */
693 writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME);
698 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
700 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
701 struct arm_smmu_device *smmu = dev;
702 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
704 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
705 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
706 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
707 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
712 dev_err_ratelimited(smmu->dev,
713 "Unexpected global fault, this could be serious\n");
714 dev_err_ratelimited(smmu->dev,
715 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
716 gfsr, gfsynr0, gfsynr1, gfsynr2);
718 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
722 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
723 struct io_pgtable_cfg *pgtbl_cfg)
728 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
729 struct arm_smmu_device *smmu = smmu_domain->smmu;
730 void __iomem *cb_base, *gr1_base;
732 gr1_base = ARM_SMMU_GR1(smmu);
733 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
734 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
736 if (smmu->version > ARM_SMMU_V1) {
739 * *Must* be initialised before CBAR thanks to VMID16
740 * architectural oversight affected some implementations.
743 reg = CBA2R_RW64_64BIT;
745 reg = CBA2R_RW64_32BIT;
747 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
752 if (smmu->version == ARM_SMMU_V1)
753 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
756 * Use the weakest shareability/memory types, so they are
757 * overridden by the ttbcr/pte.
760 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
761 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
763 reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT;
765 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
769 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
771 reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT;
772 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
774 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
775 reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT;
776 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1);
778 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
779 smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0);
784 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
785 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
786 if (smmu->version > ARM_SMMU_V1) {
787 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
788 reg |= TTBCR2_SEP_UPSTREAM;
789 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2);
792 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
793 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
796 /* MAIRs (stage-1 only) */
798 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
799 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
800 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
801 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1);
805 reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
807 reg |= SCTLR_S1_ASIDPNE;
811 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
814 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
815 struct arm_smmu_device *smmu)
817 int irq, start, ret = 0;
818 unsigned long ias, oas;
819 struct io_pgtable_ops *pgtbl_ops;
820 struct io_pgtable_cfg pgtbl_cfg;
821 enum io_pgtable_fmt fmt;
822 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
823 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
825 mutex_lock(&smmu_domain->init_mutex);
826 if (smmu_domain->smmu)
830 * Mapping the requested stage onto what we support is surprisingly
831 * complicated, mainly because the spec allows S1+S2 SMMUs without
832 * support for nested translation. That means we end up with the
835 * Requested Supported Actual
845 * Note that you can't actually request stage-2 mappings.
847 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
848 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
849 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
850 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
852 switch (smmu_domain->stage) {
853 case ARM_SMMU_DOMAIN_S1:
854 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
855 start = smmu->num_s2_context_banks;
857 oas = smmu->ipa_size;
858 if (IS_ENABLED(CONFIG_64BIT))
859 fmt = ARM_64_LPAE_S1;
861 fmt = ARM_32_LPAE_S1;
863 case ARM_SMMU_DOMAIN_NESTED:
865 * We will likely want to change this if/when KVM gets
868 case ARM_SMMU_DOMAIN_S2:
869 cfg->cbar = CBAR_TYPE_S2_TRANS;
871 ias = smmu->ipa_size;
873 if (IS_ENABLED(CONFIG_64BIT))
874 fmt = ARM_64_LPAE_S2;
876 fmt = ARM_32_LPAE_S2;
883 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
884 smmu->num_context_banks);
885 if (IS_ERR_VALUE(ret))
889 if (smmu->version == ARM_SMMU_V1) {
890 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
891 cfg->irptndx %= smmu->num_context_irqs;
893 cfg->irptndx = cfg->cbndx;
896 pgtbl_cfg = (struct io_pgtable_cfg) {
897 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
900 .tlb = &arm_smmu_gather_ops,
901 .iommu_dev = smmu->dev,
904 smmu_domain->smmu = smmu;
905 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
911 /* Update our support page sizes to reflect the page table format */
912 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
914 /* Initialise the context bank with our page table cfg */
915 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
918 * Request context fault interrupt. Do this last to avoid the
919 * handler seeing a half-initialised domain state.
921 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
922 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED,
923 "arm-smmu-context-fault", domain);
924 if (IS_ERR_VALUE(ret)) {
925 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
927 cfg->irptndx = INVALID_IRPTNDX;
930 mutex_unlock(&smmu_domain->init_mutex);
932 /* Publish page table ops for map/unmap */
933 smmu_domain->pgtbl_ops = pgtbl_ops;
937 smmu_domain->smmu = NULL;
939 mutex_unlock(&smmu_domain->init_mutex);
943 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
945 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
946 struct arm_smmu_device *smmu = smmu_domain->smmu;
947 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
948 void __iomem *cb_base;
955 * Disable the context bank and free the page tables before freeing
958 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
959 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
961 if (cfg->irptndx != INVALID_IRPTNDX) {
962 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
963 free_irq(irq, domain);
966 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
967 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
970 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
972 struct arm_smmu_domain *smmu_domain;
974 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
977 * Allocate the domain and initialise some of its data structures.
978 * We can't really do anything meaningful until we've added a
981 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
985 if (type == IOMMU_DOMAIN_DMA &&
986 iommu_get_dma_cookie(&smmu_domain->domain)) {
991 mutex_init(&smmu_domain->init_mutex);
992 spin_lock_init(&smmu_domain->pgtbl_lock);
994 return &smmu_domain->domain;
997 static void arm_smmu_domain_free(struct iommu_domain *domain)
999 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1002 * Free the domain resources. We assume that all devices have
1003 * already been detached.
1005 iommu_put_dma_cookie(domain);
1006 arm_smmu_destroy_domain_context(domain);
1010 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
1011 struct arm_smmu_master_cfg *cfg)
1014 struct arm_smmu_smr *smrs;
1015 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1017 if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))
1023 smrs = kmalloc_array(cfg->num_streamids, sizeof(*smrs), GFP_KERNEL);
1025 dev_err(smmu->dev, "failed to allocate %d SMRs\n",
1026 cfg->num_streamids);
1030 /* Allocate the SMRs on the SMMU */
1031 for (i = 0; i < cfg->num_streamids; ++i) {
1032 int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,
1033 smmu->num_mapping_groups);
1034 if (IS_ERR_VALUE(idx)) {
1035 dev_err(smmu->dev, "failed to allocate free SMR\n");
1039 smrs[i] = (struct arm_smmu_smr) {
1041 .mask = 0, /* We don't currently share SMRs */
1042 .id = cfg->streamids[i],
1046 /* It worked! Now, poke the actual hardware */
1047 for (i = 0; i < cfg->num_streamids; ++i) {
1048 u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
1049 smrs[i].mask << SMR_MASK_SHIFT;
1050 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
1058 __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);
1063 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,
1064 struct arm_smmu_master_cfg *cfg)
1067 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1068 struct arm_smmu_smr *smrs = cfg->smrs;
1073 /* Invalidate the SMRs before freeing back to the allocator */
1074 for (i = 0; i < cfg->num_streamids; ++i) {
1075 u8 idx = smrs[i].idx;
1077 writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx));
1078 __arm_smmu_free_bitmap(smmu->smr_map, idx);
1085 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1086 struct arm_smmu_master_cfg *cfg)
1089 struct arm_smmu_device *smmu = smmu_domain->smmu;
1090 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1092 /* Devices in an IOMMU group may already be configured */
1093 ret = arm_smmu_master_configure_smrs(smmu, cfg);
1095 return ret == -EEXIST ? 0 : ret;
1097 for (i = 0; i < cfg->num_streamids; ++i) {
1100 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1101 s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV |
1102 (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT);
1103 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1109 static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain,
1110 struct arm_smmu_master_cfg *cfg)
1113 struct arm_smmu_device *smmu = smmu_domain->smmu;
1114 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1116 /* An IOMMU group is torn down by the first device to be removed */
1117 if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)
1121 * We *must* clear the S2CR first, because freeing the SMR means
1122 * that it can be re-allocated immediately.
1124 for (i = 0; i < cfg->num_streamids; ++i) {
1125 u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i];
1126 u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1128 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1131 arm_smmu_master_free_smrs(smmu, cfg);
1134 static void arm_smmu_detach_dev(struct device *dev,
1135 struct arm_smmu_master_cfg *cfg)
1137 struct iommu_domain *domain = dev->archdata.iommu;
1138 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1140 dev->archdata.iommu = NULL;
1141 arm_smmu_domain_remove_master(smmu_domain, cfg);
1144 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1147 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1148 struct arm_smmu_device *smmu;
1149 struct arm_smmu_master_cfg *cfg;
1151 smmu = find_smmu_for_device(dev);
1153 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1157 /* Ensure that the domain is finalised */
1158 ret = arm_smmu_init_domain_context(domain, smmu);
1159 if (IS_ERR_VALUE(ret))
1163 * Sanity check the domain. We don't support domains across
1166 if (smmu_domain->smmu != smmu) {
1168 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1169 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1173 /* Looks ok, so add the device to the domain */
1174 cfg = find_smmu_master_cfg(dev);
1178 /* Detach the dev from its current domain */
1179 if (dev->archdata.iommu)
1180 arm_smmu_detach_dev(dev, cfg);
1182 ret = arm_smmu_domain_add_master(smmu_domain, cfg);
1184 dev->archdata.iommu = domain;
1188 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1189 phys_addr_t paddr, size_t size, int prot)
1192 unsigned long flags;
1193 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1194 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1199 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1200 ret = ops->map(ops, iova, paddr, size, prot);
1201 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1205 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1209 unsigned long flags;
1210 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1211 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1216 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1217 ret = ops->unmap(ops, iova, size);
1218 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1222 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1225 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1226 struct arm_smmu_device *smmu = smmu_domain->smmu;
1227 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1228 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1229 struct device *dev = smmu->dev;
1230 void __iomem *cb_base;
1235 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1237 /* ATS1 registers can only be written atomically */
1238 va = iova & ~0xfffUL;
1239 if (smmu->version == ARM_SMMU_V2)
1240 smmu_writeq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1242 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1244 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1245 !(tmp & ATSR_ACTIVE), 5, 50)) {
1247 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1249 return ops->iova_to_phys(ops, iova);
1252 phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO);
1253 phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32;
1255 if (phys & CB_PAR_F) {
1256 dev_err(dev, "translation fault!\n");
1257 dev_err(dev, "PAR = 0x%llx\n", phys);
1261 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1264 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1268 unsigned long flags;
1269 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1270 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1275 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1276 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1277 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1278 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1280 ret = ops->iova_to_phys(ops, iova);
1283 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1288 static bool arm_smmu_capable(enum iommu_cap cap)
1291 case IOMMU_CAP_CACHE_COHERENCY:
1293 * Return true here as the SMMU can always send out coherent
1297 case IOMMU_CAP_INTR_REMAP:
1298 return true; /* MSIs are just memory writes */
1299 case IOMMU_CAP_NOEXEC:
1306 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
1308 *((u16 *)data) = alias;
1309 return 0; /* Continue walking */
1312 static void __arm_smmu_release_pci_iommudata(void *data)
1317 static int arm_smmu_init_pci_device(struct pci_dev *pdev,
1318 struct iommu_group *group)
1320 struct arm_smmu_master_cfg *cfg;
1324 cfg = iommu_group_get_iommudata(group);
1326 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
1330 iommu_group_set_iommudata(group, cfg,
1331 __arm_smmu_release_pci_iommudata);
1334 if (cfg->num_streamids >= MAX_MASTER_STREAMIDS)
1338 * Assume Stream ID == Requester ID for now.
1339 * We need a way to describe the ID mappings in FDT.
1341 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1342 for (i = 0; i < cfg->num_streamids; ++i)
1343 if (cfg->streamids[i] == sid)
1346 /* Avoid duplicate SIDs, as this can lead to SMR conflicts */
1347 if (i == cfg->num_streamids)
1348 cfg->streamids[cfg->num_streamids++] = sid;
1353 static int arm_smmu_init_platform_device(struct device *dev,
1354 struct iommu_group *group)
1356 struct arm_smmu_device *smmu = find_smmu_for_device(dev);
1357 struct arm_smmu_master *master;
1362 master = find_smmu_master(smmu, dev->of_node);
1366 iommu_group_set_iommudata(group, &master->cfg, NULL);
1371 static int arm_smmu_add_device(struct device *dev)
1373 struct iommu_group *group;
1375 group = iommu_group_get_for_dev(dev);
1377 return PTR_ERR(group);
1379 iommu_group_put(group);
1383 static void arm_smmu_remove_device(struct device *dev)
1385 iommu_group_remove_device(dev);
1388 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1390 struct iommu_group *group;
1393 if (dev_is_pci(dev))
1394 group = pci_device_group(dev);
1396 group = generic_device_group(dev);
1401 if (dev_is_pci(dev))
1402 ret = arm_smmu_init_pci_device(to_pci_dev(dev), group);
1404 ret = arm_smmu_init_platform_device(dev, group);
1407 iommu_group_put(group);
1408 group = ERR_PTR(ret);
1414 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1415 enum iommu_attr attr, void *data)
1417 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1420 case DOMAIN_ATTR_NESTING:
1421 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1428 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1429 enum iommu_attr attr, void *data)
1432 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1434 mutex_lock(&smmu_domain->init_mutex);
1437 case DOMAIN_ATTR_NESTING:
1438 if (smmu_domain->smmu) {
1444 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1446 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1454 mutex_unlock(&smmu_domain->init_mutex);
1458 static struct iommu_ops arm_smmu_ops = {
1459 .capable = arm_smmu_capable,
1460 .domain_alloc = arm_smmu_domain_alloc,
1461 .domain_free = arm_smmu_domain_free,
1462 .attach_dev = arm_smmu_attach_dev,
1463 .map = arm_smmu_map,
1464 .unmap = arm_smmu_unmap,
1465 .map_sg = default_iommu_map_sg,
1466 .iova_to_phys = arm_smmu_iova_to_phys,
1467 .add_device = arm_smmu_add_device,
1468 .remove_device = arm_smmu_remove_device,
1469 .device_group = arm_smmu_device_group,
1470 .domain_get_attr = arm_smmu_domain_get_attr,
1471 .domain_set_attr = arm_smmu_domain_set_attr,
1472 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1475 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1477 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1478 void __iomem *cb_base;
1482 /* clear global FSR */
1483 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1484 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1486 /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */
1487 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS;
1488 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1489 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i));
1490 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i));
1493 /* Make sure all context banks are disabled and clear CB_FSR */
1494 for (i = 0; i < smmu->num_context_banks; ++i) {
1495 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1496 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1497 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1500 /* Invalidate the TLB, just in case */
1501 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1502 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1504 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1506 /* Enable fault reporting */
1507 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1509 /* Disable TLB broadcasting. */
1510 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1512 /* Enable client access, handling unmatched streams as appropriate */
1513 reg &= ~sCR0_CLIENTPD;
1517 reg &= ~sCR0_USFCFG;
1519 /* Disable forced broadcasting */
1522 /* Don't upgrade barriers */
1523 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1525 /* Push the button */
1526 __arm_smmu_tlb_sync(smmu);
1527 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1530 static int arm_smmu_id_size_to_bits(int size)
1549 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1552 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1554 bool cttw_dt, cttw_reg;
1556 dev_notice(smmu->dev, "probing hardware configuration...\n");
1557 dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);
1560 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1562 /* Restrict available stages based on module parameter */
1563 if (force_stage == 1)
1564 id &= ~(ID0_S2TS | ID0_NTS);
1565 else if (force_stage == 2)
1566 id &= ~(ID0_S1TS | ID0_NTS);
1568 if (id & ID0_S1TS) {
1569 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1570 dev_notice(smmu->dev, "\tstage 1 translation\n");
1573 if (id & ID0_S2TS) {
1574 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1575 dev_notice(smmu->dev, "\tstage 2 translation\n");
1579 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1580 dev_notice(smmu->dev, "\tnested translation\n");
1583 if (!(smmu->features &
1584 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1585 dev_err(smmu->dev, "\tno translation support!\n");
1589 if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {
1590 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1591 dev_notice(smmu->dev, "\taddress translation ops\n");
1595 * In order for DMA API calls to work properly, we must defer to what
1596 * the DT says about coherency, regardless of what the hardware claims.
1597 * Fortunately, this also opens up a workaround for systems where the
1598 * ID register value has ended up configured incorrectly.
1600 cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1601 cttw_reg = !!(id & ID0_CTTW);
1603 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1604 if (cttw_dt || cttw_reg)
1605 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1606 cttw_dt ? "" : "non-");
1607 if (cttw_dt != cttw_reg)
1608 dev_notice(smmu->dev,
1609 "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1614 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1615 smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
1617 if (smmu->num_mapping_groups == 0) {
1619 "stream-matching supported, but no SMRs present!\n");
1623 smr = SMR_MASK_MASK << SMR_MASK_SHIFT;
1624 smr |= (SMR_ID_MASK << SMR_ID_SHIFT);
1625 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1626 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1628 mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
1629 sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
1630 if ((mask & sid) != sid) {
1632 "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
1637 dev_notice(smmu->dev,
1638 "\tstream matching with %u register groups, mask 0x%x",
1639 smmu->num_mapping_groups, mask);
1641 smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &
1646 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1647 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1649 /* Check for size mismatch of SMMU address space from mapped region */
1650 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1651 size *= 2 << smmu->pgshift;
1652 if (smmu->size != size)
1654 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1657 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1658 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1659 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1660 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1663 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1664 smmu->num_context_banks, smmu->num_s2_context_banks);
1667 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1668 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1669 smmu->ipa_size = size;
1671 /* The output mask is also applied for bypass */
1672 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1673 smmu->pa_size = size;
1676 * What the page table walker can address actually depends on which
1677 * descriptor format is in use, but since a) we don't know that yet,
1678 * and b) it can vary per context bank, this will have to do...
1680 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1682 "failed to set DMA mask for table walker\n");
1684 if (smmu->version == ARM_SMMU_V1) {
1685 smmu->va_size = smmu->ipa_size;
1686 size = SZ_4K | SZ_2M | SZ_1G;
1688 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1689 smmu->va_size = arm_smmu_id_size_to_bits(size);
1690 #ifndef CONFIG_64BIT
1691 smmu->va_size = min(32UL, smmu->va_size);
1694 if (id & ID2_PTFS_4K)
1695 size |= SZ_4K | SZ_2M | SZ_1G;
1696 if (id & ID2_PTFS_16K)
1697 size |= SZ_16K | SZ_32M;
1698 if (id & ID2_PTFS_64K)
1699 size |= SZ_64K | SZ_512M;
1702 arm_smmu_ops.pgsize_bitmap &= size;
1703 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);
1705 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1706 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1707 smmu->va_size, smmu->ipa_size);
1709 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1710 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1711 smmu->ipa_size, smmu->pa_size);
1716 static const struct of_device_id arm_smmu_of_match[] = {
1717 { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 },
1718 { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 },
1719 { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 },
1720 { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 },
1721 { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 },
1724 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1726 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1728 const struct of_device_id *of_id;
1729 struct resource *res;
1730 struct arm_smmu_device *smmu;
1731 struct device *dev = &pdev->dev;
1732 struct rb_node *node;
1733 struct of_phandle_args masterspec;
1734 int num_irqs, i, err;
1736 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1738 dev_err(dev, "failed to allocate arm_smmu_device\n");
1743 of_id = of_match_node(arm_smmu_of_match, dev->of_node);
1744 smmu->version = (enum arm_smmu_arch_version)of_id->data;
1746 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1747 smmu->base = devm_ioremap_resource(dev, res);
1748 if (IS_ERR(smmu->base))
1749 return PTR_ERR(smmu->base);
1750 smmu->size = resource_size(res);
1752 if (of_property_read_u32(dev->of_node, "#global-interrupts",
1753 &smmu->num_global_irqs)) {
1754 dev_err(dev, "missing #global-interrupts property\n");
1759 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1761 if (num_irqs > smmu->num_global_irqs)
1762 smmu->num_context_irqs++;
1765 if (!smmu->num_context_irqs) {
1766 dev_err(dev, "found %d interrupts but expected at least %d\n",
1767 num_irqs, smmu->num_global_irqs + 1);
1771 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1774 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1778 for (i = 0; i < num_irqs; ++i) {
1779 int irq = platform_get_irq(pdev, i);
1782 dev_err(dev, "failed to get irq index %d\n", i);
1785 smmu->irqs[i] = irq;
1788 err = arm_smmu_device_cfg_probe(smmu);
1793 smmu->masters = RB_ROOT;
1794 while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters",
1795 "#stream-id-cells", i,
1797 err = register_smmu_master(smmu, dev, &masterspec);
1799 dev_err(dev, "failed to add master %s\n",
1800 masterspec.np->name);
1801 goto out_put_masters;
1806 dev_notice(dev, "registered %d master devices\n", i);
1808 parse_driver_options(smmu);
1810 if (smmu->version > ARM_SMMU_V1 &&
1811 smmu->num_context_banks != smmu->num_context_irqs) {
1813 "found only %d context interrupt(s) but %d required\n",
1814 smmu->num_context_irqs, smmu->num_context_banks);
1816 goto out_put_masters;
1819 for (i = 0; i < smmu->num_global_irqs; ++i) {
1820 err = request_irq(smmu->irqs[i],
1821 arm_smmu_global_fault,
1823 "arm-smmu global fault",
1826 dev_err(dev, "failed to request global IRQ %d (%u)\n",
1832 INIT_LIST_HEAD(&smmu->list);
1833 spin_lock(&arm_smmu_devices_lock);
1834 list_add(&smmu->list, &arm_smmu_devices);
1835 spin_unlock(&arm_smmu_devices_lock);
1837 arm_smmu_device_reset(smmu);
1842 free_irq(smmu->irqs[i], smmu);
1845 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1846 struct arm_smmu_master *master
1847 = container_of(node, struct arm_smmu_master, node);
1848 of_node_put(master->of_node);
1854 static int arm_smmu_device_remove(struct platform_device *pdev)
1857 struct device *dev = &pdev->dev;
1858 struct arm_smmu_device *curr, *smmu = NULL;
1859 struct rb_node *node;
1861 spin_lock(&arm_smmu_devices_lock);
1862 list_for_each_entry(curr, &arm_smmu_devices, list) {
1863 if (curr->dev == dev) {
1865 list_del(&smmu->list);
1869 spin_unlock(&arm_smmu_devices_lock);
1874 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
1875 struct arm_smmu_master *master
1876 = container_of(node, struct arm_smmu_master, node);
1877 of_node_put(master->of_node);
1880 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
1881 dev_err(dev, "removing device with active domains!\n");
1883 for (i = 0; i < smmu->num_global_irqs; ++i)
1884 free_irq(smmu->irqs[i], smmu);
1886 /* Turn the thing off */
1887 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1891 static struct platform_driver arm_smmu_driver = {
1894 .of_match_table = of_match_ptr(arm_smmu_of_match),
1896 .probe = arm_smmu_device_dt_probe,
1897 .remove = arm_smmu_device_remove,
1900 static int __init arm_smmu_init(void)
1902 struct device_node *np;
1906 * Play nice with systems that don't have an ARM SMMU by checking that
1907 * an ARM SMMU exists in the system before proceeding with the driver
1908 * and IOMMU bus operation registration.
1910 np = of_find_matching_node(NULL, arm_smmu_of_match);
1916 ret = platform_driver_register(&arm_smmu_driver);
1920 /* Oh, for a proper bus abstraction */
1921 if (!iommu_present(&platform_bus_type))
1922 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
1924 #ifdef CONFIG_ARM_AMBA
1925 if (!iommu_present(&amba_bustype))
1926 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
1930 if (!iommu_present(&pci_bus_type))
1931 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
1937 static void __exit arm_smmu_exit(void)
1939 return platform_driver_unregister(&arm_smmu_driver);
1942 subsys_initcall(arm_smmu_init);
1943 module_exit(arm_smmu_exit);
1945 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
1946 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
1947 MODULE_LICENSE("GPL v2");