2 * IOMMU API for ARM architected SMMU implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 * Copyright (C) 2013 ARM Limited
19 * Author: Will Deacon <will.deacon@arm.com>
21 * This driver currently supports:
22 * - SMMUv1 and v2 implementations
23 * - Stream-matching and stream-indexing
24 * - v7/v8 long-descriptor format
25 * - Non-secure access to the SMMU
26 * - Context fault reporting
27 * - Extended Stream ID (16 bit)
30 #define pr_fmt(fmt) "arm-smmu: " fmt
32 #include <linux/acpi.h>
33 #include <linux/acpi_iort.h>
34 #include <linux/atomic.h>
35 #include <linux/delay.h>
36 #include <linux/dma-iommu.h>
37 #include <linux/dma-mapping.h>
38 #include <linux/err.h>
39 #include <linux/interrupt.h>
41 #include <linux/io-64-nonatomic-hi-lo.h>
42 #include <linux/iommu.h>
43 #include <linux/iopoll.h>
44 #include <linux/module.h>
46 #include <linux/of_address.h>
47 #include <linux/of_device.h>
48 #include <linux/of_iommu.h>
49 #include <linux/pci.h>
50 #include <linux/platform_device.h>
51 #include <linux/slab.h>
52 #include <linux/spinlock.h>
54 #include <linux/amba/bus.h>
56 #include "io-pgtable.h"
58 /* Maximum number of context banks per SMMU */
59 #define ARM_SMMU_MAX_CBS 128
61 /* SMMU global address space */
62 #define ARM_SMMU_GR0(smmu) ((smmu)->base)
63 #define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift))
66 * SMMU global address space with conditional offset to access secure
67 * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
70 #define ARM_SMMU_GR0_NS(smmu) \
72 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
76 * Some 64-bit registers only make sense to write atomically, but in such
77 * cases all the data relevant to AArch32 formats lies within the lower word,
78 * therefore this actually makes more sense than it might first appear.
81 #define smmu_write_atomic_lq writeq_relaxed
83 #define smmu_write_atomic_lq writel_relaxed
86 /* Configuration registers */
87 #define ARM_SMMU_GR0_sCR0 0x0
88 #define sCR0_CLIENTPD (1 << 0)
89 #define sCR0_GFRE (1 << 1)
90 #define sCR0_GFIE (1 << 2)
91 #define sCR0_EXIDENABLE (1 << 3)
92 #define sCR0_GCFGFRE (1 << 4)
93 #define sCR0_GCFGFIE (1 << 5)
94 #define sCR0_USFCFG (1 << 10)
95 #define sCR0_VMIDPNE (1 << 11)
96 #define sCR0_PTM (1 << 12)
97 #define sCR0_FB (1 << 13)
98 #define sCR0_VMID16EN (1 << 31)
99 #define sCR0_BSU_SHIFT 14
100 #define sCR0_BSU_MASK 0x3
102 /* Auxiliary Configuration register */
103 #define ARM_SMMU_GR0_sACR 0x10
105 /* Identification registers */
106 #define ARM_SMMU_GR0_ID0 0x20
107 #define ARM_SMMU_GR0_ID1 0x24
108 #define ARM_SMMU_GR0_ID2 0x28
109 #define ARM_SMMU_GR0_ID3 0x2c
110 #define ARM_SMMU_GR0_ID4 0x30
111 #define ARM_SMMU_GR0_ID5 0x34
112 #define ARM_SMMU_GR0_ID6 0x38
113 #define ARM_SMMU_GR0_ID7 0x3c
114 #define ARM_SMMU_GR0_sGFSR 0x48
115 #define ARM_SMMU_GR0_sGFSYNR0 0x50
116 #define ARM_SMMU_GR0_sGFSYNR1 0x54
117 #define ARM_SMMU_GR0_sGFSYNR2 0x58
119 #define ID0_S1TS (1 << 30)
120 #define ID0_S2TS (1 << 29)
121 #define ID0_NTS (1 << 28)
122 #define ID0_SMS (1 << 27)
123 #define ID0_ATOSNS (1 << 26)
124 #define ID0_PTFS_NO_AARCH32 (1 << 25)
125 #define ID0_PTFS_NO_AARCH32S (1 << 24)
126 #define ID0_CTTW (1 << 14)
127 #define ID0_NUMIRPT_SHIFT 16
128 #define ID0_NUMIRPT_MASK 0xff
129 #define ID0_NUMSIDB_SHIFT 9
130 #define ID0_NUMSIDB_MASK 0xf
131 #define ID0_EXIDS (1 << 8)
132 #define ID0_NUMSMRG_SHIFT 0
133 #define ID0_NUMSMRG_MASK 0xff
135 #define ID1_PAGESIZE (1 << 31)
136 #define ID1_NUMPAGENDXB_SHIFT 28
137 #define ID1_NUMPAGENDXB_MASK 7
138 #define ID1_NUMS2CB_SHIFT 16
139 #define ID1_NUMS2CB_MASK 0xff
140 #define ID1_NUMCB_SHIFT 0
141 #define ID1_NUMCB_MASK 0xff
143 #define ID2_OAS_SHIFT 4
144 #define ID2_OAS_MASK 0xf
145 #define ID2_IAS_SHIFT 0
146 #define ID2_IAS_MASK 0xf
147 #define ID2_UBS_SHIFT 8
148 #define ID2_UBS_MASK 0xf
149 #define ID2_PTFS_4K (1 << 12)
150 #define ID2_PTFS_16K (1 << 13)
151 #define ID2_PTFS_64K (1 << 14)
152 #define ID2_VMID16 (1 << 15)
154 #define ID7_MAJOR_SHIFT 4
155 #define ID7_MAJOR_MASK 0xf
157 /* Global TLB invalidation */
158 #define ARM_SMMU_GR0_TLBIVMID 0x64
159 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68
160 #define ARM_SMMU_GR0_TLBIALLH 0x6c
161 #define ARM_SMMU_GR0_sTLBGSYNC 0x70
162 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74
163 #define sTLBGSTATUS_GSACTIVE (1 << 0)
164 #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */
166 /* Stream mapping registers */
167 #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
168 #define SMR_VALID (1 << 31)
169 #define SMR_MASK_SHIFT 16
170 #define SMR_ID_SHIFT 0
172 #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
173 #define S2CR_CBNDX_SHIFT 0
174 #define S2CR_CBNDX_MASK 0xff
175 #define S2CR_EXIDVALID (1 << 10)
176 #define S2CR_TYPE_SHIFT 16
177 #define S2CR_TYPE_MASK 0x3
178 enum arm_smmu_s2cr_type {
184 #define S2CR_PRIVCFG_SHIFT 24
185 #define S2CR_PRIVCFG_MASK 0x3
186 enum arm_smmu_s2cr_privcfg {
187 S2CR_PRIVCFG_DEFAULT,
193 /* Context bank attribute registers */
194 #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
195 #define CBAR_VMID_SHIFT 0
196 #define CBAR_VMID_MASK 0xff
197 #define CBAR_S1_BPSHCFG_SHIFT 8
198 #define CBAR_S1_BPSHCFG_MASK 3
199 #define CBAR_S1_BPSHCFG_NSH 3
200 #define CBAR_S1_MEMATTR_SHIFT 12
201 #define CBAR_S1_MEMATTR_MASK 0xf
202 #define CBAR_S1_MEMATTR_WB 0xf
203 #define CBAR_TYPE_SHIFT 16
204 #define CBAR_TYPE_MASK 0x3
205 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
206 #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
207 #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
208 #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
209 #define CBAR_IRPTNDX_SHIFT 24
210 #define CBAR_IRPTNDX_MASK 0xff
212 #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
213 #define CBA2R_RW64_32BIT (0 << 0)
214 #define CBA2R_RW64_64BIT (1 << 0)
215 #define CBA2R_VMID_SHIFT 16
216 #define CBA2R_VMID_MASK 0xffff
218 /* Translation context bank */
219 #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1))
220 #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift))
222 #define ARM_SMMU_CB_SCTLR 0x0
223 #define ARM_SMMU_CB_ACTLR 0x4
224 #define ARM_SMMU_CB_RESUME 0x8
225 #define ARM_SMMU_CB_TTBCR2 0x10
226 #define ARM_SMMU_CB_TTBR0 0x20
227 #define ARM_SMMU_CB_TTBR1 0x28
228 #define ARM_SMMU_CB_TTBCR 0x30
229 #define ARM_SMMU_CB_CONTEXTIDR 0x34
230 #define ARM_SMMU_CB_S1_MAIR0 0x38
231 #define ARM_SMMU_CB_S1_MAIR1 0x3c
232 #define ARM_SMMU_CB_PAR 0x50
233 #define ARM_SMMU_CB_FSR 0x58
234 #define ARM_SMMU_CB_FAR 0x60
235 #define ARM_SMMU_CB_FSYNR0 0x68
236 #define ARM_SMMU_CB_S1_TLBIVA 0x600
237 #define ARM_SMMU_CB_S1_TLBIASID 0x610
238 #define ARM_SMMU_CB_S1_TLBIVAL 0x620
239 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
240 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
241 #define ARM_SMMU_CB_ATS1PR 0x800
242 #define ARM_SMMU_CB_ATSR 0x8f0
244 #define SCTLR_S1_ASIDPNE (1 << 12)
245 #define SCTLR_CFCFG (1 << 7)
246 #define SCTLR_CFIE (1 << 6)
247 #define SCTLR_CFRE (1 << 5)
248 #define SCTLR_E (1 << 4)
249 #define SCTLR_AFE (1 << 2)
250 #define SCTLR_TRE (1 << 1)
251 #define SCTLR_M (1 << 0)
253 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
255 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
256 #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
258 #define CB_PAR_F (1 << 0)
260 #define ATSR_ACTIVE (1 << 0)
262 #define RESUME_RETRY (0 << 0)
263 #define RESUME_TERMINATE (1 << 0)
265 #define TTBCR2_SEP_SHIFT 15
266 #define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
267 #define TTBCR2_AS (1 << 4)
269 #define TTBRn_ASID_SHIFT 48
271 #define FSR_MULTI (1 << 31)
272 #define FSR_SS (1 << 30)
273 #define FSR_UUT (1 << 8)
274 #define FSR_ASF (1 << 7)
275 #define FSR_TLBLKF (1 << 6)
276 #define FSR_TLBMCF (1 << 5)
277 #define FSR_EF (1 << 4)
278 #define FSR_PF (1 << 3)
279 #define FSR_AFF (1 << 2)
280 #define FSR_TF (1 << 1)
282 #define FSR_IGN (FSR_AFF | FSR_ASF | \
283 FSR_TLBMCF | FSR_TLBLKF)
284 #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
285 FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
287 #define FSYNR0_WNR (1 << 4)
289 static int force_stage;
290 module_param(force_stage, int, S_IRUGO);
291 MODULE_PARM_DESC(force_stage,
292 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
293 static bool disable_bypass;
294 module_param(disable_bypass, bool, S_IRUGO);
295 MODULE_PARM_DESC(disable_bypass,
296 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
298 enum arm_smmu_arch_version {
304 enum arm_smmu_implementation {
310 struct arm_smmu_s2cr {
311 struct iommu_group *group;
313 enum arm_smmu_s2cr_type type;
314 enum arm_smmu_s2cr_privcfg privcfg;
318 #define s2cr_init_val (struct arm_smmu_s2cr){ \
319 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
322 struct arm_smmu_smr {
328 struct arm_smmu_master_cfg {
329 struct arm_smmu_device *smmu;
332 #define INVALID_SMENDX -1
333 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
334 #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu)
335 #define fwspec_smendx(fw, i) \
336 (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
337 #define for_each_cfg_sme(fw, i, idx) \
338 for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
340 struct arm_smmu_device {
345 unsigned long pgshift;
347 #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
348 #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
349 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
350 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
351 #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
352 #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
353 #define ARM_SMMU_FEAT_VMID16 (1 << 6)
354 #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
355 #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
356 #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
357 #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
358 #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
359 #define ARM_SMMU_FEAT_EXIDS (1 << 12)
362 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
364 enum arm_smmu_arch_version version;
365 enum arm_smmu_implementation model;
367 u32 num_context_banks;
368 u32 num_s2_context_banks;
369 DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
372 u32 num_mapping_groups;
375 struct arm_smmu_smr *smrs;
376 struct arm_smmu_s2cr *s2crs;
377 struct mutex stream_map_mutex;
379 unsigned long va_size;
380 unsigned long ipa_size;
381 unsigned long pa_size;
382 unsigned long pgsize_bitmap;
385 u32 num_context_irqs;
388 u32 cavium_id_base; /* Specific to Cavium */
391 enum arm_smmu_context_fmt {
392 ARM_SMMU_CTX_FMT_NONE,
393 ARM_SMMU_CTX_FMT_AARCH64,
394 ARM_SMMU_CTX_FMT_AARCH32_L,
395 ARM_SMMU_CTX_FMT_AARCH32_S,
398 struct arm_smmu_cfg {
402 enum arm_smmu_context_fmt fmt;
404 #define INVALID_IRPTNDX 0xff
406 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
407 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
409 enum arm_smmu_domain_stage {
410 ARM_SMMU_DOMAIN_S1 = 0,
412 ARM_SMMU_DOMAIN_NESTED,
415 struct arm_smmu_domain {
416 struct arm_smmu_device *smmu;
417 struct io_pgtable_ops *pgtbl_ops;
418 spinlock_t pgtbl_lock;
419 struct arm_smmu_cfg cfg;
420 enum arm_smmu_domain_stage stage;
421 struct mutex init_mutex; /* Protects smmu pointer */
422 struct iommu_domain domain;
425 struct arm_smmu_option_prop {
430 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
432 static bool using_legacy_binding, using_generic_binding;
434 static struct arm_smmu_option_prop arm_smmu_options[] = {
435 { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
439 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
441 return container_of(dom, struct arm_smmu_domain, domain);
444 static void parse_driver_options(struct arm_smmu_device *smmu)
449 if (of_property_read_bool(smmu->dev->of_node,
450 arm_smmu_options[i].prop)) {
451 smmu->options |= arm_smmu_options[i].opt;
452 dev_notice(smmu->dev, "option %s\n",
453 arm_smmu_options[i].prop);
455 } while (arm_smmu_options[++i].opt);
458 static struct device_node *dev_get_dev_node(struct device *dev)
460 if (dev_is_pci(dev)) {
461 struct pci_bus *bus = to_pci_dev(dev)->bus;
463 while (!pci_is_root_bus(bus))
465 return of_node_get(bus->bridge->parent->of_node);
468 return of_node_get(dev->of_node);
471 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
473 *((__be32 *)data) = cpu_to_be32(alias);
474 return 0; /* Continue walking */
477 static int __find_legacy_master_phandle(struct device *dev, void *data)
479 struct of_phandle_iterator *it = *(void **)data;
480 struct device_node *np = it->node;
483 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
484 "#stream-id-cells", 0)
485 if (it->node == np) {
486 *(void **)data = dev;
490 return err == -ENOENT ? 0 : err;
493 static struct platform_driver arm_smmu_driver;
494 static struct iommu_ops arm_smmu_ops;
496 static int arm_smmu_register_legacy_master(struct device *dev,
497 struct arm_smmu_device **smmu)
499 struct device *smmu_dev;
500 struct device_node *np;
501 struct of_phandle_iterator it;
507 np = dev_get_dev_node(dev);
508 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
514 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
515 __find_legacy_master_phandle);
523 if (dev_is_pci(dev)) {
524 /* "mmu-masters" assumes Stream ID == Requester ID */
525 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
531 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
536 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
540 *smmu = dev_get_drvdata(smmu_dev);
541 of_phandle_iterator_args(&it, sids, it.cur_count);
542 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
547 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
552 idx = find_next_zero_bit(map, end, start);
555 } while (test_and_set_bit(idx, map));
560 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
565 /* Wait for any pending TLB invalidations to complete */
566 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
569 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
571 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
572 while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
573 & sTLBGSTATUS_GSACTIVE) {
575 if (++count == TLB_LOOP_TIMEOUT) {
576 dev_err_ratelimited(smmu->dev,
577 "TLB sync timed out -- SMMU may be deadlocked\n");
584 static void arm_smmu_tlb_sync(void *cookie)
586 struct arm_smmu_domain *smmu_domain = cookie;
587 __arm_smmu_tlb_sync(smmu_domain->smmu);
590 static void arm_smmu_tlb_inv_context(void *cookie)
592 struct arm_smmu_domain *smmu_domain = cookie;
593 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
594 struct arm_smmu_device *smmu = smmu_domain->smmu;
595 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
599 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
600 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
601 base + ARM_SMMU_CB_S1_TLBIASID);
603 base = ARM_SMMU_GR0(smmu);
604 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
605 base + ARM_SMMU_GR0_TLBIVMID);
608 __arm_smmu_tlb_sync(smmu);
611 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
612 size_t granule, bool leaf, void *cookie)
614 struct arm_smmu_domain *smmu_domain = cookie;
615 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
616 struct arm_smmu_device *smmu = smmu_domain->smmu;
617 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
621 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
622 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
624 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
626 iova |= ARM_SMMU_CB_ASID(smmu, cfg);
628 writel_relaxed(iova, reg);
630 } while (size -= granule);
633 iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
635 writeq_relaxed(iova, reg);
636 iova += granule >> 12;
637 } while (size -= granule);
639 } else if (smmu->version == ARM_SMMU_V2) {
640 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
641 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
642 ARM_SMMU_CB_S2_TLBIIPAS2;
645 smmu_write_atomic_lq(iova, reg);
646 iova += granule >> 12;
647 } while (size -= granule);
649 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
650 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
654 static const struct iommu_gather_ops arm_smmu_gather_ops = {
655 .tlb_flush_all = arm_smmu_tlb_inv_context,
656 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
657 .tlb_sync = arm_smmu_tlb_sync,
660 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
664 struct iommu_domain *domain = dev;
665 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
666 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
667 struct arm_smmu_device *smmu = smmu_domain->smmu;
668 void __iomem *cb_base;
670 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
671 fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
673 if (!(fsr & FSR_FAULT))
676 fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
677 iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
679 dev_err_ratelimited(smmu->dev,
680 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
681 fsr, iova, fsynr, cfg->cbndx);
683 writel(fsr, cb_base + ARM_SMMU_CB_FSR);
687 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
689 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
690 struct arm_smmu_device *smmu = dev;
691 void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
693 gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
694 gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
695 gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
696 gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
701 dev_err_ratelimited(smmu->dev,
702 "Unexpected global fault, this could be serious\n");
703 dev_err_ratelimited(smmu->dev,
704 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
705 gfsr, gfsynr0, gfsynr1, gfsynr2);
707 writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
711 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
712 struct io_pgtable_cfg *pgtbl_cfg)
717 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
718 struct arm_smmu_device *smmu = smmu_domain->smmu;
719 void __iomem *cb_base, *gr1_base;
721 gr1_base = ARM_SMMU_GR1(smmu);
722 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
723 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
725 if (smmu->version > ARM_SMMU_V1) {
726 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
727 reg = CBA2R_RW64_64BIT;
729 reg = CBA2R_RW64_32BIT;
730 /* 16-bit VMIDs live in CBA2R */
731 if (smmu->features & ARM_SMMU_FEAT_VMID16)
732 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
734 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
739 if (smmu->version < ARM_SMMU_V2)
740 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
743 * Use the weakest shareability/memory types, so they are
744 * overridden by the ttbcr/pte.
747 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
748 (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
749 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
750 /* 8-bit VMIDs live in CBAR */
751 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
753 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
757 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
759 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
760 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
761 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
762 reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
763 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
764 writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
766 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
767 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
768 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
769 reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
770 reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
771 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
774 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
775 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
780 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
781 reg = pgtbl_cfg->arm_v7s_cfg.tcr;
784 reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
785 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
786 reg2 |= TTBCR2_SEP_UPSTREAM;
787 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
790 if (smmu->version > ARM_SMMU_V1)
791 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
793 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
795 writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
797 /* MAIRs (stage-1 only) */
799 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
800 reg = pgtbl_cfg->arm_v7s_cfg.prrr;
801 reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
803 reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
804 reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
806 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
807 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
811 reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
813 reg |= SCTLR_S1_ASIDPNE;
817 writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
820 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
821 struct arm_smmu_device *smmu)
823 int irq, start, ret = 0;
824 unsigned long ias, oas;
825 struct io_pgtable_ops *pgtbl_ops;
826 struct io_pgtable_cfg pgtbl_cfg;
827 enum io_pgtable_fmt fmt;
828 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
829 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
831 mutex_lock(&smmu_domain->init_mutex);
832 if (smmu_domain->smmu)
836 * Mapping the requested stage onto what we support is surprisingly
837 * complicated, mainly because the spec allows S1+S2 SMMUs without
838 * support for nested translation. That means we end up with the
841 * Requested Supported Actual
851 * Note that you can't actually request stage-2 mappings.
853 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
854 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
855 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
856 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
859 * Choosing a suitable context format is even more fiddly. Until we
860 * grow some way for the caller to express a preference, and/or move
861 * the decision into the io-pgtable code where it arguably belongs,
862 * just aim for the closest thing to the rest of the system, and hope
863 * that the hardware isn't esoteric enough that we can't assume AArch64
864 * support to be a superset of AArch32 support...
866 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
867 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
868 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
869 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
870 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
871 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
872 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
873 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
874 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
875 ARM_SMMU_FEAT_FMT_AARCH64_16K |
876 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
877 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
879 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
884 switch (smmu_domain->stage) {
885 case ARM_SMMU_DOMAIN_S1:
886 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
887 start = smmu->num_s2_context_banks;
889 oas = smmu->ipa_size;
890 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
891 fmt = ARM_64_LPAE_S1;
892 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
893 fmt = ARM_32_LPAE_S1;
894 ias = min(ias, 32UL);
895 oas = min(oas, 40UL);
898 ias = min(ias, 32UL);
899 oas = min(oas, 32UL);
902 case ARM_SMMU_DOMAIN_NESTED:
904 * We will likely want to change this if/when KVM gets
907 case ARM_SMMU_DOMAIN_S2:
908 cfg->cbar = CBAR_TYPE_S2_TRANS;
910 ias = smmu->ipa_size;
912 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
913 fmt = ARM_64_LPAE_S2;
915 fmt = ARM_32_LPAE_S2;
916 ias = min(ias, 40UL);
917 oas = min(oas, 40UL);
925 ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
926 smmu->num_context_banks);
931 if (smmu->version < ARM_SMMU_V2) {
932 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
933 cfg->irptndx %= smmu->num_context_irqs;
935 cfg->irptndx = cfg->cbndx;
938 pgtbl_cfg = (struct io_pgtable_cfg) {
939 .pgsize_bitmap = smmu->pgsize_bitmap,
942 .tlb = &arm_smmu_gather_ops,
943 .iommu_dev = smmu->dev,
946 smmu_domain->smmu = smmu;
947 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
953 /* Update the domain's page sizes to reflect the page table format */
954 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
955 domain->geometry.aperture_end = (1UL << ias) - 1;
956 domain->geometry.force_aperture = true;
958 /* Initialise the context bank with our page table cfg */
959 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
962 * Request context fault interrupt. Do this last to avoid the
963 * handler seeing a half-initialised domain state.
965 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
966 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
967 IRQF_SHARED, "arm-smmu-context-fault", domain);
969 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
971 cfg->irptndx = INVALID_IRPTNDX;
974 mutex_unlock(&smmu_domain->init_mutex);
976 /* Publish page table ops for map/unmap */
977 smmu_domain->pgtbl_ops = pgtbl_ops;
981 smmu_domain->smmu = NULL;
983 mutex_unlock(&smmu_domain->init_mutex);
987 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
989 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
990 struct arm_smmu_device *smmu = smmu_domain->smmu;
991 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
992 void __iomem *cb_base;
999 * Disable the context bank and free the page tables before freeing
1002 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1003 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1005 if (cfg->irptndx != INVALID_IRPTNDX) {
1006 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1007 devm_free_irq(smmu->dev, irq, domain);
1010 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1011 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1014 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1016 struct arm_smmu_domain *smmu_domain;
1018 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1021 * Allocate the domain and initialise some of its data structures.
1022 * We can't really do anything meaningful until we've added a
1025 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1029 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1030 iommu_get_dma_cookie(&smmu_domain->domain))) {
1035 mutex_init(&smmu_domain->init_mutex);
1036 spin_lock_init(&smmu_domain->pgtbl_lock);
1038 return &smmu_domain->domain;
1041 static void arm_smmu_domain_free(struct iommu_domain *domain)
1043 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1046 * Free the domain resources. We assume that all devices have
1047 * already been detached.
1049 iommu_put_dma_cookie(domain);
1050 arm_smmu_destroy_domain_context(domain);
1054 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1056 struct arm_smmu_smr *smr = smmu->smrs + idx;
1057 u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1059 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1061 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1064 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1066 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1067 u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1068 (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1069 (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1071 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1072 smmu->smrs[idx].valid)
1073 reg |= S2CR_EXIDVALID;
1074 writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1077 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1079 arm_smmu_write_s2cr(smmu, idx);
1081 arm_smmu_write_smr(smmu, idx);
1085 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1086 * should be called after sCR0 is written.
1088 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1090 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1097 * SMR.ID bits may not be preserved if the corresponding MASK
1098 * bits are set, so check each one separately. We can reject
1099 * masters later if they try to claim IDs outside these masks.
1101 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1102 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1103 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1104 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1106 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1107 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1108 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1109 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1112 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1114 struct arm_smmu_smr *smrs = smmu->smrs;
1115 int i, free_idx = -ENOSPC;
1117 /* Stream indexing is blissfully easy */
1121 /* Validating SMRs is... less so */
1122 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1123 if (!smrs[i].valid) {
1125 * Note the first free entry we come across, which
1126 * we'll claim in the end if nothing else matches.
1133 * If the new entry is _entirely_ matched by an existing entry,
1134 * then reuse that, with the guarantee that there also cannot
1135 * be any subsequent conflicting entries. In normal use we'd
1136 * expect simply identical entries for this case, but there's
1137 * no harm in accommodating the generalisation.
1139 if ((mask & smrs[i].mask) == mask &&
1140 !((id ^ smrs[i].id) & ~smrs[i].mask))
1143 * If the new entry has any other overlap with an existing one,
1144 * though, then there always exists at least one stream ID
1145 * which would cause a conflict, and we can't allow that risk.
1147 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1154 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1156 if (--smmu->s2crs[idx].count)
1159 smmu->s2crs[idx] = s2cr_init_val;
1161 smmu->smrs[idx].valid = false;
1166 static int arm_smmu_master_alloc_smes(struct device *dev)
1168 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1169 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1170 struct arm_smmu_device *smmu = cfg->smmu;
1171 struct arm_smmu_smr *smrs = smmu->smrs;
1172 struct iommu_group *group;
1175 mutex_lock(&smmu->stream_map_mutex);
1176 /* Figure out a viable stream map entry allocation */
1177 for_each_cfg_sme(fwspec, i, idx) {
1178 u16 sid = fwspec->ids[i];
1179 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1181 if (idx != INVALID_SMENDX) {
1186 ret = arm_smmu_find_sme(smmu, sid, mask);
1191 if (smrs && smmu->s2crs[idx].count == 0) {
1193 smrs[idx].mask = mask;
1194 smrs[idx].valid = true;
1196 smmu->s2crs[idx].count++;
1197 cfg->smendx[i] = (s16)idx;
1200 group = iommu_group_get_for_dev(dev);
1202 group = ERR_PTR(-ENOMEM);
1203 if (IS_ERR(group)) {
1204 ret = PTR_ERR(group);
1207 iommu_group_put(group);
1209 /* It worked! Now, poke the actual hardware */
1210 for_each_cfg_sme(fwspec, i, idx) {
1211 arm_smmu_write_sme(smmu, idx);
1212 smmu->s2crs[idx].group = group;
1215 mutex_unlock(&smmu->stream_map_mutex);
1220 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1221 cfg->smendx[i] = INVALID_SMENDX;
1223 mutex_unlock(&smmu->stream_map_mutex);
1227 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1229 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1230 struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1233 mutex_lock(&smmu->stream_map_mutex);
1234 for_each_cfg_sme(fwspec, i, idx) {
1235 if (arm_smmu_free_sme(smmu, idx))
1236 arm_smmu_write_sme(smmu, idx);
1237 cfg->smendx[i] = INVALID_SMENDX;
1239 mutex_unlock(&smmu->stream_map_mutex);
1242 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1243 struct iommu_fwspec *fwspec)
1245 struct arm_smmu_device *smmu = smmu_domain->smmu;
1246 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1247 enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1248 u8 cbndx = smmu_domain->cfg.cbndx;
1251 for_each_cfg_sme(fwspec, i, idx) {
1252 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1255 s2cr[idx].type = type;
1256 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1257 s2cr[idx].cbndx = cbndx;
1258 arm_smmu_write_s2cr(smmu, idx);
1263 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1266 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1267 struct arm_smmu_device *smmu;
1268 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1270 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1271 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1276 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1277 * domains between of_xlate() and add_device() - we have no way to cope
1278 * with that, so until ARM gets converted to rely on groups and default
1279 * domains, just say no (but more politely than by dereferencing NULL).
1280 * This should be at least a WARN_ON once that's sorted.
1282 if (!fwspec->iommu_priv)
1285 smmu = fwspec_smmu(fwspec);
1286 /* Ensure that the domain is finalised */
1287 ret = arm_smmu_init_domain_context(domain, smmu);
1292 * Sanity check the domain. We don't support domains across
1295 if (smmu_domain->smmu != smmu) {
1297 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1298 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1302 /* Looks ok, so add the device to the domain */
1303 return arm_smmu_domain_add_master(smmu_domain, fwspec);
1306 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1307 phys_addr_t paddr, size_t size, int prot)
1310 unsigned long flags;
1311 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1312 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1317 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1318 ret = ops->map(ops, iova, paddr, size, prot);
1319 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1323 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1327 unsigned long flags;
1328 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1329 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1334 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1335 ret = ops->unmap(ops, iova, size);
1336 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1340 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1343 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344 struct arm_smmu_device *smmu = smmu_domain->smmu;
1345 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1346 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1347 struct device *dev = smmu->dev;
1348 void __iomem *cb_base;
1353 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1355 /* ATS1 registers can only be written atomically */
1356 va = iova & ~0xfffUL;
1357 if (smmu->version == ARM_SMMU_V2)
1358 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1359 else /* Register is only 32-bit in v1 */
1360 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1362 if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1363 !(tmp & ATSR_ACTIVE), 5, 50)) {
1365 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1367 return ops->iova_to_phys(ops, iova);
1370 phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1371 if (phys & CB_PAR_F) {
1372 dev_err(dev, "translation fault!\n");
1373 dev_err(dev, "PAR = 0x%llx\n", phys);
1377 return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1380 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1384 unsigned long flags;
1385 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1386 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1391 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1392 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1393 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1394 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1396 ret = ops->iova_to_phys(ops, iova);
1399 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1404 static bool arm_smmu_capable(enum iommu_cap cap)
1407 case IOMMU_CAP_CACHE_COHERENCY:
1409 * Return true here as the SMMU can always send out coherent
1413 case IOMMU_CAP_INTR_REMAP:
1414 return true; /* MSIs are just memory writes */
1415 case IOMMU_CAP_NOEXEC:
1422 static int arm_smmu_match_node(struct device *dev, void *data)
1424 return dev->fwnode == data;
1428 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1430 struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1431 fwnode, arm_smmu_match_node);
1433 return dev ? dev_get_drvdata(dev) : NULL;
1436 static int arm_smmu_add_device(struct device *dev)
1438 struct arm_smmu_device *smmu;
1439 struct arm_smmu_master_cfg *cfg;
1440 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1443 if (using_legacy_binding) {
1444 ret = arm_smmu_register_legacy_master(dev, &smmu);
1445 fwspec = dev->iommu_fwspec;
1448 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1449 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1455 for (i = 0; i < fwspec->num_ids; i++) {
1456 u16 sid = fwspec->ids[i];
1457 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1459 if (sid & ~smmu->streamid_mask) {
1460 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1461 sid, smmu->streamid_mask);
1464 if (mask & ~smmu->smr_mask_mask) {
1465 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1466 sid, smmu->smr_mask_mask);
1472 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1478 fwspec->iommu_priv = cfg;
1480 cfg->smendx[i] = INVALID_SMENDX;
1482 ret = arm_smmu_master_alloc_smes(dev);
1490 kfree(fwspec->iommu_priv);
1491 iommu_fwspec_free(dev);
1495 static void arm_smmu_remove_device(struct device *dev)
1497 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1499 if (!fwspec || fwspec->ops != &arm_smmu_ops)
1502 arm_smmu_master_free_smes(fwspec);
1503 iommu_group_remove_device(dev);
1504 kfree(fwspec->iommu_priv);
1505 iommu_fwspec_free(dev);
1508 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1510 struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1511 struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1512 struct iommu_group *group = NULL;
1515 for_each_cfg_sme(fwspec, i, idx) {
1516 if (group && smmu->s2crs[idx].group &&
1517 group != smmu->s2crs[idx].group)
1518 return ERR_PTR(-EINVAL);
1520 group = smmu->s2crs[idx].group;
1524 return iommu_group_ref_get(group);
1526 if (dev_is_pci(dev))
1527 group = pci_device_group(dev);
1529 group = generic_device_group(dev);
1534 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1535 enum iommu_attr attr, void *data)
1537 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1540 case DOMAIN_ATTR_NESTING:
1541 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1548 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1549 enum iommu_attr attr, void *data)
1552 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1554 mutex_lock(&smmu_domain->init_mutex);
1557 case DOMAIN_ATTR_NESTING:
1558 if (smmu_domain->smmu) {
1564 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1566 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1574 mutex_unlock(&smmu_domain->init_mutex);
1578 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1582 if (args->args_count > 0)
1583 fwid |= (u16)args->args[0];
1585 if (args->args_count > 1)
1586 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1588 return iommu_fwspec_add_ids(dev, &fwid, 1);
1591 static struct iommu_ops arm_smmu_ops = {
1592 .capable = arm_smmu_capable,
1593 .domain_alloc = arm_smmu_domain_alloc,
1594 .domain_free = arm_smmu_domain_free,
1595 .attach_dev = arm_smmu_attach_dev,
1596 .map = arm_smmu_map,
1597 .unmap = arm_smmu_unmap,
1598 .map_sg = default_iommu_map_sg,
1599 .iova_to_phys = arm_smmu_iova_to_phys,
1600 .add_device = arm_smmu_add_device,
1601 .remove_device = arm_smmu_remove_device,
1602 .device_group = arm_smmu_device_group,
1603 .domain_get_attr = arm_smmu_domain_get_attr,
1604 .domain_set_attr = arm_smmu_domain_set_attr,
1605 .of_xlate = arm_smmu_of_xlate,
1606 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1609 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1611 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1612 void __iomem *cb_base;
1616 /* clear global FSR */
1617 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1618 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1621 * Reset stream mapping groups: Initial values mark all SMRn as
1622 * invalid and all S2CRn as bypass unless overridden.
1624 for (i = 0; i < smmu->num_mapping_groups; ++i)
1625 arm_smmu_write_sme(smmu, i);
1627 if (smmu->model == ARM_MMU500) {
1629 * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1630 * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1631 * bit is only present in MMU-500r2 onwards.
1633 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1634 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1635 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1637 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1639 * Allow unmatched Stream IDs to allocate bypass
1640 * TLB entries for reduced latency.
1642 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1643 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1646 /* Make sure all context banks are disabled and clear CB_FSR */
1647 for (i = 0; i < smmu->num_context_banks; ++i) {
1648 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1649 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1650 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1652 * Disable MMU-500's not-particularly-beneficial next-page
1653 * prefetcher for the sake of errata #841119 and #826419.
1655 if (smmu->model == ARM_MMU500) {
1656 reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1657 reg &= ~ARM_MMU500_ACTLR_CPRE;
1658 writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1662 /* Invalidate the TLB, just in case */
1663 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1664 writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1666 reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1668 /* Enable fault reporting */
1669 reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1671 /* Disable TLB broadcasting. */
1672 reg |= (sCR0_VMIDPNE | sCR0_PTM);
1674 /* Enable client access, handling unmatched streams as appropriate */
1675 reg &= ~sCR0_CLIENTPD;
1679 reg &= ~sCR0_USFCFG;
1681 /* Disable forced broadcasting */
1684 /* Don't upgrade barriers */
1685 reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1687 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1688 reg |= sCR0_VMID16EN;
1690 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1691 reg |= sCR0_EXIDENABLE;
1693 /* Push the button */
1694 __arm_smmu_tlb_sync(smmu);
1695 writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1698 static int arm_smmu_id_size_to_bits(int size)
1717 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1720 void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1722 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1725 dev_notice(smmu->dev, "probing hardware configuration...\n");
1726 dev_notice(smmu->dev, "SMMUv%d with:\n",
1727 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1730 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1732 /* Restrict available stages based on module parameter */
1733 if (force_stage == 1)
1734 id &= ~(ID0_S2TS | ID0_NTS);
1735 else if (force_stage == 2)
1736 id &= ~(ID0_S1TS | ID0_NTS);
1738 if (id & ID0_S1TS) {
1739 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1740 dev_notice(smmu->dev, "\tstage 1 translation\n");
1743 if (id & ID0_S2TS) {
1744 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1745 dev_notice(smmu->dev, "\tstage 2 translation\n");
1749 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1750 dev_notice(smmu->dev, "\tnested translation\n");
1753 if (!(smmu->features &
1754 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1755 dev_err(smmu->dev, "\tno translation support!\n");
1759 if ((id & ID0_S1TS) &&
1760 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1761 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1762 dev_notice(smmu->dev, "\taddress translation ops\n");
1766 * In order for DMA API calls to work properly, we must defer to what
1767 * the FW says about coherency, regardless of what the hardware claims.
1768 * Fortunately, this also opens up a workaround for systems where the
1769 * ID register value has ended up configured incorrectly.
1771 cttw_reg = !!(id & ID0_CTTW);
1772 if (cttw_fw || cttw_reg)
1773 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1774 cttw_fw ? "" : "non-");
1775 if (cttw_fw != cttw_reg)
1776 dev_notice(smmu->dev,
1777 "\t(IDR0.CTTW overridden by FW configuration)\n");
1779 /* Max. number of entries we have for stream matching/indexing */
1780 if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1781 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1784 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1786 smmu->streamid_mask = size - 1;
1788 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1789 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1792 "stream-matching supported, but no SMRs present!\n");
1796 /* Zero-initialised to mark as invalid */
1797 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1802 dev_notice(smmu->dev,
1803 "\tstream matching with %lu register groups", size);
1805 /* s2cr->type == 0 means translation, so initialise explicitly */
1806 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1810 for (i = 0; i < size; i++)
1811 smmu->s2crs[i] = s2cr_init_val;
1813 smmu->num_mapping_groups = size;
1814 mutex_init(&smmu->stream_map_mutex);
1816 if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1817 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1818 if (!(id & ID0_PTFS_NO_AARCH32S))
1819 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1823 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1824 smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1826 /* Check for size mismatch of SMMU address space from mapped region */
1827 size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1828 size *= 2 << smmu->pgshift;
1829 if (smmu->size != size)
1831 "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1834 smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1835 smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1836 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1837 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1840 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1841 smmu->num_context_banks, smmu->num_s2_context_banks);
1843 * Cavium CN88xx erratum #27704.
1844 * Ensure ASID and VMID allocation is unique across all SMMUs in
1847 if (smmu->model == CAVIUM_SMMUV2) {
1848 smmu->cavium_id_base =
1849 atomic_add_return(smmu->num_context_banks,
1850 &cavium_smmu_context_count);
1851 smmu->cavium_id_base -= smmu->num_context_banks;
1855 id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1856 size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1857 smmu->ipa_size = size;
1859 /* The output mask is also applied for bypass */
1860 size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1861 smmu->pa_size = size;
1863 if (id & ID2_VMID16)
1864 smmu->features |= ARM_SMMU_FEAT_VMID16;
1867 * What the page table walker can address actually depends on which
1868 * descriptor format is in use, but since a) we don't know that yet,
1869 * and b) it can vary per context bank, this will have to do...
1871 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1873 "failed to set DMA mask for table walker\n");
1875 if (smmu->version < ARM_SMMU_V2) {
1876 smmu->va_size = smmu->ipa_size;
1877 if (smmu->version == ARM_SMMU_V1_64K)
1878 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1880 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1881 smmu->va_size = arm_smmu_id_size_to_bits(size);
1882 if (id & ID2_PTFS_4K)
1883 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1884 if (id & ID2_PTFS_16K)
1885 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1886 if (id & ID2_PTFS_64K)
1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1890 /* Now we've corralled the various formats, what'll it do? */
1891 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1892 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1893 if (smmu->features &
1894 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1895 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1896 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1897 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1898 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1899 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1901 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1902 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1904 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1905 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1906 smmu->pgsize_bitmap);
1909 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1910 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1911 smmu->va_size, smmu->ipa_size);
1913 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1914 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1915 smmu->ipa_size, smmu->pa_size);
1920 struct arm_smmu_match_data {
1921 enum arm_smmu_arch_version version;
1922 enum arm_smmu_implementation model;
1925 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1926 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1928 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1929 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1930 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1931 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1932 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1934 static const struct of_device_id arm_smmu_of_match[] = {
1935 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1936 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1937 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1938 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1939 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1940 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1943 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1946 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1951 case ACPI_IORT_SMMU_V1:
1952 case ACPI_IORT_SMMU_CORELINK_MMU400:
1953 smmu->version = ARM_SMMU_V1;
1954 smmu->model = GENERIC_SMMU;
1956 case ACPI_IORT_SMMU_V2:
1957 smmu->version = ARM_SMMU_V2;
1958 smmu->model = GENERIC_SMMU;
1960 case ACPI_IORT_SMMU_CORELINK_MMU500:
1961 smmu->version = ARM_SMMU_V2;
1962 smmu->model = ARM_MMU500;
1971 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1972 struct arm_smmu_device *smmu)
1974 struct device *dev = smmu->dev;
1975 struct acpi_iort_node *node =
1976 *(struct acpi_iort_node **)dev_get_platdata(dev);
1977 struct acpi_iort_smmu *iort_smmu;
1980 /* Retrieve SMMU1/2 specific data */
1981 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1983 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1987 /* Ignore the configuration access interrupt */
1988 smmu->num_global_irqs = 1;
1990 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1991 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1996 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1997 struct arm_smmu_device *smmu)
2003 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2004 struct arm_smmu_device *smmu)
2006 const struct arm_smmu_match_data *data;
2007 struct device *dev = &pdev->dev;
2008 bool legacy_binding;
2010 if (of_property_read_u32(dev->of_node, "#global-interrupts",
2011 &smmu->num_global_irqs)) {
2012 dev_err(dev, "missing #global-interrupts property\n");
2016 data = of_device_get_match_data(dev);
2017 smmu->version = data->version;
2018 smmu->model = data->model;
2020 parse_driver_options(smmu);
2022 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2023 if (legacy_binding && !using_generic_binding) {
2024 if (!using_legacy_binding)
2025 pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2026 using_legacy_binding = true;
2027 } else if (!legacy_binding && !using_legacy_binding) {
2028 using_generic_binding = true;
2030 dev_err(dev, "not probing due to mismatched DT properties\n");
2034 if (of_dma_is_coherent(dev->of_node))
2035 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2040 static int arm_smmu_device_probe(struct platform_device *pdev)
2042 struct resource *res;
2043 struct arm_smmu_device *smmu;
2044 struct device *dev = &pdev->dev;
2045 int num_irqs, i, err;
2047 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2049 dev_err(dev, "failed to allocate arm_smmu_device\n");
2055 err = arm_smmu_device_dt_probe(pdev, smmu);
2057 err = arm_smmu_device_acpi_probe(pdev, smmu);
2062 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2063 smmu->base = devm_ioremap_resource(dev, res);
2064 if (IS_ERR(smmu->base))
2065 return PTR_ERR(smmu->base);
2066 smmu->size = resource_size(res);
2069 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2071 if (num_irqs > smmu->num_global_irqs)
2072 smmu->num_context_irqs++;
2075 if (!smmu->num_context_irqs) {
2076 dev_err(dev, "found %d interrupts but expected at least %d\n",
2077 num_irqs, smmu->num_global_irqs + 1);
2081 smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2084 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2088 for (i = 0; i < num_irqs; ++i) {
2089 int irq = platform_get_irq(pdev, i);
2092 dev_err(dev, "failed to get irq index %d\n", i);
2095 smmu->irqs[i] = irq;
2098 err = arm_smmu_device_cfg_probe(smmu);
2102 if (smmu->version == ARM_SMMU_V2 &&
2103 smmu->num_context_banks != smmu->num_context_irqs) {
2105 "found only %d context interrupt(s) but %d required\n",
2106 smmu->num_context_irqs, smmu->num_context_banks);
2110 for (i = 0; i < smmu->num_global_irqs; ++i) {
2111 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2112 arm_smmu_global_fault,
2114 "arm-smmu global fault",
2117 dev_err(dev, "failed to request global IRQ %d (%u)\n",
2123 iommu_register_instance(dev->fwnode, &arm_smmu_ops);
2124 platform_set_drvdata(pdev, smmu);
2125 arm_smmu_device_reset(smmu);
2126 arm_smmu_test_smr_masks(smmu);
2128 /* Oh, for a proper bus abstraction */
2129 if (!iommu_present(&platform_bus_type))
2130 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2131 #ifdef CONFIG_ARM_AMBA
2132 if (!iommu_present(&amba_bustype))
2133 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2136 if (!iommu_present(&pci_bus_type)) {
2138 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2144 static int arm_smmu_device_remove(struct platform_device *pdev)
2146 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2151 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2152 dev_err(&pdev->dev, "removing device with active domains!\n");
2154 /* Turn the thing off */
2155 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2159 static struct platform_driver arm_smmu_driver = {
2162 .of_match_table = of_match_ptr(arm_smmu_of_match),
2164 .probe = arm_smmu_device_probe,
2165 .remove = arm_smmu_device_remove,
2168 static int __init arm_smmu_init(void)
2170 static bool registered;
2174 ret = platform_driver_register(&arm_smmu_driver);
2180 static void __exit arm_smmu_exit(void)
2182 return platform_driver_unregister(&arm_smmu_driver);
2185 subsys_initcall(arm_smmu_init);
2186 module_exit(arm_smmu_exit);
2188 static int __init arm_smmu_of_init(struct device_node *np)
2190 int ret = arm_smmu_init();
2195 if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2200 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2201 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2202 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2203 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2204 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2205 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2208 static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2210 if (iort_node_match(ACPI_IORT_NODE_SMMU))
2211 return arm_smmu_init();
2215 IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2218 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2219 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2220 MODULE_LICENSE("GPL v2");