2 * IOMMU API for ARM architected SMMUv3 implementations.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 * Copyright (C) 2015 ARM Limited
18 * Author: Will Deacon <will.deacon@arm.com>
20 * This driver is powered by bad coffee and bombay mix.
23 #include <linux/delay.h>
24 #include <linux/err.h>
25 #include <linux/interrupt.h>
26 #include <linux/iommu.h>
27 #include <linux/iopoll.h>
28 #include <linux/module.h>
30 #include <linux/of_address.h>
31 #include <linux/pci.h>
32 #include <linux/platform_device.h>
34 #include "io-pgtable.h"
37 #define ARM_SMMU_IDR0 0x0
38 #define IDR0_ST_LVL_SHIFT 27
39 #define IDR0_ST_LVL_MASK 0x3
40 #define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT)
41 #define IDR0_STALL_MODEL (3 << 24)
42 #define IDR0_TTENDIAN_SHIFT 21
43 #define IDR0_TTENDIAN_MASK 0x3
44 #define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT)
45 #define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT)
46 #define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT)
47 #define IDR0_CD2L (1 << 19)
48 #define IDR0_VMID16 (1 << 18)
49 #define IDR0_PRI (1 << 16)
50 #define IDR0_SEV (1 << 14)
51 #define IDR0_MSI (1 << 13)
52 #define IDR0_ASID16 (1 << 12)
53 #define IDR0_ATS (1 << 10)
54 #define IDR0_HYP (1 << 9)
55 #define IDR0_COHACC (1 << 4)
56 #define IDR0_TTF_SHIFT 2
57 #define IDR0_TTF_MASK 0x3
58 #define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT)
59 #define IDR0_TTF_AARCH32_64 (3 << IDR0_TTF_SHIFT)
60 #define IDR0_S1P (1 << 1)
61 #define IDR0_S2P (1 << 0)
63 #define ARM_SMMU_IDR1 0x4
64 #define IDR1_TABLES_PRESET (1 << 30)
65 #define IDR1_QUEUES_PRESET (1 << 29)
66 #define IDR1_REL (1 << 28)
67 #define IDR1_CMDQ_SHIFT 21
68 #define IDR1_CMDQ_MASK 0x1f
69 #define IDR1_EVTQ_SHIFT 16
70 #define IDR1_EVTQ_MASK 0x1f
71 #define IDR1_PRIQ_SHIFT 11
72 #define IDR1_PRIQ_MASK 0x1f
73 #define IDR1_SSID_SHIFT 6
74 #define IDR1_SSID_MASK 0x1f
75 #define IDR1_SID_SHIFT 0
76 #define IDR1_SID_MASK 0x3f
78 #define ARM_SMMU_IDR5 0x14
79 #define IDR5_STALL_MAX_SHIFT 16
80 #define IDR5_STALL_MAX_MASK 0xffff
81 #define IDR5_GRAN64K (1 << 6)
82 #define IDR5_GRAN16K (1 << 5)
83 #define IDR5_GRAN4K (1 << 4)
84 #define IDR5_OAS_SHIFT 0
85 #define IDR5_OAS_MASK 0x7
86 #define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT)
87 #define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT)
88 #define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT)
89 #define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT)
90 #define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT)
91 #define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT)
93 #define ARM_SMMU_CR0 0x20
94 #define CR0_CMDQEN (1 << 3)
95 #define CR0_EVTQEN (1 << 2)
96 #define CR0_PRIQEN (1 << 1)
97 #define CR0_SMMUEN (1 << 0)
99 #define ARM_SMMU_CR0ACK 0x24
101 #define ARM_SMMU_CR1 0x28
105 #define CR1_CACHE_NC 0
106 #define CR1_CACHE_WB 1
107 #define CR1_CACHE_WT 2
108 #define CR1_TABLE_SH_SHIFT 10
109 #define CR1_TABLE_OC_SHIFT 8
110 #define CR1_TABLE_IC_SHIFT 6
111 #define CR1_QUEUE_SH_SHIFT 4
112 #define CR1_QUEUE_OC_SHIFT 2
113 #define CR1_QUEUE_IC_SHIFT 0
115 #define ARM_SMMU_CR2 0x2c
116 #define CR2_PTM (1 << 2)
117 #define CR2_RECINVSID (1 << 1)
118 #define CR2_E2H (1 << 0)
120 #define ARM_SMMU_IRQ_CTRL 0x50
121 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
122 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
123 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
125 #define ARM_SMMU_IRQ_CTRLACK 0x54
127 #define ARM_SMMU_GERROR 0x60
128 #define GERROR_SFM_ERR (1 << 8)
129 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
130 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
131 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
132 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
133 #define GERROR_PRIQ_ABT_ERR (1 << 3)
134 #define GERROR_EVTQ_ABT_ERR (1 << 2)
135 #define GERROR_CMDQ_ERR (1 << 0)
136 #define GERROR_ERR_MASK 0xfd
138 #define ARM_SMMU_GERRORN 0x64
140 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
141 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
142 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
144 #define ARM_SMMU_STRTAB_BASE 0x80
145 #define STRTAB_BASE_RA (1UL << 62)
146 #define STRTAB_BASE_ADDR_SHIFT 6
147 #define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL
149 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
150 #define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0
151 #define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f
152 #define STRTAB_BASE_CFG_SPLIT_SHIFT 6
153 #define STRTAB_BASE_CFG_SPLIT_MASK 0x1f
154 #define STRTAB_BASE_CFG_FMT_SHIFT 16
155 #define STRTAB_BASE_CFG_FMT_MASK 0x3
156 #define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT)
157 #define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT)
159 #define ARM_SMMU_CMDQ_BASE 0x90
160 #define ARM_SMMU_CMDQ_PROD 0x98
161 #define ARM_SMMU_CMDQ_CONS 0x9c
163 #define ARM_SMMU_EVTQ_BASE 0xa0
164 #define ARM_SMMU_EVTQ_PROD 0x100a8
165 #define ARM_SMMU_EVTQ_CONS 0x100ac
166 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
167 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
168 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
170 #define ARM_SMMU_PRIQ_BASE 0xc0
171 #define ARM_SMMU_PRIQ_PROD 0x100c8
172 #define ARM_SMMU_PRIQ_CONS 0x100cc
173 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
174 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
175 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
177 /* Common MSI config fields */
178 #define MSI_CFG0_ADDR_SHIFT 2
179 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL
180 #define MSI_CFG2_SH_SHIFT 4
181 #define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT)
182 #define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT)
183 #define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT)
184 #define MSI_CFG2_MEMATTR_SHIFT 0
185 #define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT)
187 #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1))
188 #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift))
189 #define Q_OVERFLOW_FLAG (1 << 31)
190 #define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG)
191 #define Q_ENT(q, p) ((q)->base + \
192 Q_IDX(q, p) * (q)->ent_dwords)
194 #define Q_BASE_RWA (1UL << 62)
195 #define Q_BASE_ADDR_SHIFT 5
196 #define Q_BASE_ADDR_MASK 0xfffffffffffUL
197 #define Q_BASE_LOG2SIZE_SHIFT 0
198 #define Q_BASE_LOG2SIZE_MASK 0x1fUL
203 * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
204 * 2lvl: 128k L1 entries,
205 * 256 lazy entries per table (each table covers a PCI bus)
207 #define STRTAB_L1_SZ_SHIFT 20
208 #define STRTAB_SPLIT 8
210 #define STRTAB_L1_DESC_DWORDS 1
211 #define STRTAB_L1_DESC_SPAN_SHIFT 0
212 #define STRTAB_L1_DESC_SPAN_MASK 0x1fUL
213 #define STRTAB_L1_DESC_L2PTR_SHIFT 6
214 #define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL
216 #define STRTAB_STE_DWORDS 8
217 #define STRTAB_STE_0_V (1UL << 0)
218 #define STRTAB_STE_0_CFG_SHIFT 1
219 #define STRTAB_STE_0_CFG_MASK 0x7UL
220 #define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT)
221 #define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT)
222 #define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT)
223 #define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT)
225 #define STRTAB_STE_0_S1FMT_SHIFT 4
226 #define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT)
227 #define STRTAB_STE_0_S1CTXPTR_SHIFT 6
228 #define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL
229 #define STRTAB_STE_0_S1CDMAX_SHIFT 59
230 #define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL
232 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
233 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
234 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
235 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
236 #define STRTAB_STE_1_S1C_SH_NSH 0UL
237 #define STRTAB_STE_1_S1C_SH_OSH 2UL
238 #define STRTAB_STE_1_S1C_SH_ISH 3UL
239 #define STRTAB_STE_1_S1CIR_SHIFT 2
240 #define STRTAB_STE_1_S1COR_SHIFT 4
241 #define STRTAB_STE_1_S1CSH_SHIFT 6
243 #define STRTAB_STE_1_S1STALLD (1UL << 27)
245 #define STRTAB_STE_1_EATS_ABT 0UL
246 #define STRTAB_STE_1_EATS_TRANS 1UL
247 #define STRTAB_STE_1_EATS_S1CHK 2UL
248 #define STRTAB_STE_1_EATS_SHIFT 28
250 #define STRTAB_STE_1_STRW_NSEL1 0UL
251 #define STRTAB_STE_1_STRW_EL2 2UL
252 #define STRTAB_STE_1_STRW_SHIFT 30
254 #define STRTAB_STE_2_S2VMID_SHIFT 0
255 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL
256 #define STRTAB_STE_2_VTCR_SHIFT 32
257 #define STRTAB_STE_2_VTCR_MASK 0x7ffffUL
258 #define STRTAB_STE_2_S2AA64 (1UL << 51)
259 #define STRTAB_STE_2_S2ENDI (1UL << 52)
260 #define STRTAB_STE_2_S2PTW (1UL << 54)
261 #define STRTAB_STE_2_S2R (1UL << 58)
263 #define STRTAB_STE_3_S2TTB_SHIFT 4
264 #define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL
266 /* Context descriptor (stage-1 only) */
267 #define CTXDESC_CD_DWORDS 8
268 #define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0
269 #define ARM64_TCR_T0SZ_SHIFT 0
270 #define ARM64_TCR_T0SZ_MASK 0x1fUL
271 #define CTXDESC_CD_0_TCR_TG0_SHIFT 6
272 #define ARM64_TCR_TG0_SHIFT 14
273 #define ARM64_TCR_TG0_MASK 0x3UL
274 #define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8
275 #define ARM64_TCR_IRGN0_SHIFT 8
276 #define ARM64_TCR_IRGN0_MASK 0x3UL
277 #define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10
278 #define ARM64_TCR_ORGN0_SHIFT 10
279 #define ARM64_TCR_ORGN0_MASK 0x3UL
280 #define CTXDESC_CD_0_TCR_SH0_SHIFT 12
281 #define ARM64_TCR_SH0_SHIFT 12
282 #define ARM64_TCR_SH0_MASK 0x3UL
283 #define CTXDESC_CD_0_TCR_EPD0_SHIFT 14
284 #define ARM64_TCR_EPD0_SHIFT 7
285 #define ARM64_TCR_EPD0_MASK 0x1UL
286 #define CTXDESC_CD_0_TCR_EPD1_SHIFT 30
287 #define ARM64_TCR_EPD1_SHIFT 23
288 #define ARM64_TCR_EPD1_MASK 0x1UL
290 #define CTXDESC_CD_0_ENDI (1UL << 15)
291 #define CTXDESC_CD_0_V (1UL << 31)
293 #define CTXDESC_CD_0_TCR_IPS_SHIFT 32
294 #define ARM64_TCR_IPS_SHIFT 32
295 #define ARM64_TCR_IPS_MASK 0x7UL
296 #define CTXDESC_CD_0_TCR_TBI0_SHIFT 38
297 #define ARM64_TCR_TBI0_SHIFT 37
298 #define ARM64_TCR_TBI0_MASK 0x1UL
300 #define CTXDESC_CD_0_AA64 (1UL << 41)
301 #define CTXDESC_CD_0_R (1UL << 45)
302 #define CTXDESC_CD_0_A (1UL << 46)
303 #define CTXDESC_CD_0_ASET_SHIFT 47
304 #define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT)
305 #define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT)
306 #define CTXDESC_CD_0_ASID_SHIFT 48
307 #define CTXDESC_CD_0_ASID_MASK 0xffffUL
309 #define CTXDESC_CD_1_TTB0_SHIFT 4
310 #define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL
312 #define CTXDESC_CD_3_MAIR_SHIFT 0
314 /* Convert between AArch64 (CPU) TCR format and SMMU CD format */
315 #define ARM_SMMU_TCR2CD(tcr, fld) \
316 (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \
317 << CTXDESC_CD_0_TCR_##fld##_SHIFT)
320 #define CMDQ_ENT_DWORDS 2
321 #define CMDQ_MAX_SZ_SHIFT 8
323 #define CMDQ_ERR_SHIFT 24
324 #define CMDQ_ERR_MASK 0x7f
325 #define CMDQ_ERR_CERROR_NONE_IDX 0
326 #define CMDQ_ERR_CERROR_ILL_IDX 1
327 #define CMDQ_ERR_CERROR_ABT_IDX 2
329 #define CMDQ_0_OP_SHIFT 0
330 #define CMDQ_0_OP_MASK 0xffUL
331 #define CMDQ_0_SSV (1UL << 11)
333 #define CMDQ_PREFETCH_0_SID_SHIFT 32
334 #define CMDQ_PREFETCH_1_SIZE_SHIFT 0
335 #define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL
337 #define CMDQ_CFGI_0_SID_SHIFT 32
338 #define CMDQ_CFGI_0_SID_MASK 0xffffffffUL
339 #define CMDQ_CFGI_1_LEAF (1UL << 0)
340 #define CMDQ_CFGI_1_RANGE_SHIFT 0
341 #define CMDQ_CFGI_1_RANGE_MASK 0x1fUL
343 #define CMDQ_TLBI_0_VMID_SHIFT 32
344 #define CMDQ_TLBI_0_ASID_SHIFT 48
345 #define CMDQ_TLBI_1_LEAF (1UL << 0)
346 #define CMDQ_TLBI_1_VA_MASK ~0xfffUL
347 #define CMDQ_TLBI_1_IPA_MASK 0xfffffffff000UL
349 #define CMDQ_PRI_0_SSID_SHIFT 12
350 #define CMDQ_PRI_0_SSID_MASK 0xfffffUL
351 #define CMDQ_PRI_0_SID_SHIFT 32
352 #define CMDQ_PRI_0_SID_MASK 0xffffffffUL
353 #define CMDQ_PRI_1_GRPID_SHIFT 0
354 #define CMDQ_PRI_1_GRPID_MASK 0x1ffUL
355 #define CMDQ_PRI_1_RESP_SHIFT 12
356 #define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT)
357 #define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
358 #define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
360 #define CMDQ_SYNC_0_CS_SHIFT 12
361 #define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
362 #define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT)
365 #define EVTQ_ENT_DWORDS 4
366 #define EVTQ_MAX_SZ_SHIFT 7
368 #define EVTQ_0_ID_SHIFT 0
369 #define EVTQ_0_ID_MASK 0xffUL
372 #define PRIQ_ENT_DWORDS 2
373 #define PRIQ_MAX_SZ_SHIFT 8
375 #define PRIQ_0_SID_SHIFT 0
376 #define PRIQ_0_SID_MASK 0xffffffffUL
377 #define PRIQ_0_SSID_SHIFT 32
378 #define PRIQ_0_SSID_MASK 0xfffffUL
379 #define PRIQ_0_OF (1UL << 57)
380 #define PRIQ_0_PERM_PRIV (1UL << 58)
381 #define PRIQ_0_PERM_EXEC (1UL << 59)
382 #define PRIQ_0_PERM_READ (1UL << 60)
383 #define PRIQ_0_PERM_WRITE (1UL << 61)
384 #define PRIQ_0_PRG_LAST (1UL << 62)
385 #define PRIQ_0_SSID_V (1UL << 63)
387 #define PRIQ_1_PRG_IDX_SHIFT 0
388 #define PRIQ_1_PRG_IDX_MASK 0x1ffUL
389 #define PRIQ_1_ADDR_SHIFT 12
390 #define PRIQ_1_ADDR_MASK 0xfffffffffffffUL
392 /* High-level queue structures */
393 #define ARM_SMMU_POLL_TIMEOUT_US 100
395 static bool disable_bypass;
396 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
397 MODULE_PARM_DESC(disable_bypass,
398 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
406 struct arm_smmu_cmdq_ent {
409 bool substream_valid;
411 /* Command-specific fields */
413 #define CMDQ_OP_PREFETCH_CFG 0x1
420 #define CMDQ_OP_CFGI_STE 0x3
421 #define CMDQ_OP_CFGI_ALL 0x4
430 #define CMDQ_OP_TLBI_NH_ASID 0x11
431 #define CMDQ_OP_TLBI_NH_VA 0x12
432 #define CMDQ_OP_TLBI_EL2_ALL 0x20
433 #define CMDQ_OP_TLBI_S12_VMALL 0x28
434 #define CMDQ_OP_TLBI_S2_IPA 0x2a
435 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
443 #define CMDQ_OP_PRI_RESP 0x41
451 #define CMDQ_OP_CMD_SYNC 0x46
455 struct arm_smmu_queue {
456 int irq; /* Wired interrupt */
467 u32 __iomem *prod_reg;
468 u32 __iomem *cons_reg;
471 struct arm_smmu_cmdq {
472 struct arm_smmu_queue q;
476 struct arm_smmu_evtq {
477 struct arm_smmu_queue q;
481 struct arm_smmu_priq {
482 struct arm_smmu_queue q;
485 /* High-level stream table and context descriptor structures */
486 struct arm_smmu_strtab_l1_desc {
490 dma_addr_t l2ptr_dma;
493 struct arm_smmu_s1_cfg {
495 dma_addr_t cdptr_dma;
497 struct arm_smmu_ctx_desc {
505 struct arm_smmu_s2_cfg {
511 struct arm_smmu_strtab_ent {
514 bool bypass; /* Overrides s1/s2 config */
515 struct arm_smmu_s1_cfg *s1_cfg;
516 struct arm_smmu_s2_cfg *s2_cfg;
519 struct arm_smmu_strtab_cfg {
521 dma_addr_t strtab_dma;
522 struct arm_smmu_strtab_l1_desc *l1_desc;
523 unsigned int num_l1_ents;
529 /* An SMMUv3 instance */
530 struct arm_smmu_device {
534 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
535 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
536 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
537 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
538 #define ARM_SMMU_FEAT_PRI (1 << 4)
539 #define ARM_SMMU_FEAT_ATS (1 << 5)
540 #define ARM_SMMU_FEAT_SEV (1 << 6)
541 #define ARM_SMMU_FEAT_MSI (1 << 7)
542 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
543 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
544 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
545 #define ARM_SMMU_FEAT_STALLS (1 << 11)
546 #define ARM_SMMU_FEAT_HYP (1 << 12)
549 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
552 struct arm_smmu_cmdq cmdq;
553 struct arm_smmu_evtq evtq;
554 struct arm_smmu_priq priq;
558 unsigned long ias; /* IPA */
559 unsigned long oas; /* PA */
561 #define ARM_SMMU_MAX_ASIDS (1 << 16)
562 unsigned int asid_bits;
563 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
565 #define ARM_SMMU_MAX_VMIDS (1 << 16)
566 unsigned int vmid_bits;
567 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
569 unsigned int ssid_bits;
570 unsigned int sid_bits;
572 struct arm_smmu_strtab_cfg strtab_cfg;
573 struct list_head list;
576 /* SMMU private data for an IOMMU group */
577 struct arm_smmu_group {
578 struct arm_smmu_device *smmu;
579 struct arm_smmu_domain *domain;
582 struct arm_smmu_strtab_ent ste;
585 /* SMMU private data for an IOMMU domain */
586 enum arm_smmu_domain_stage {
587 ARM_SMMU_DOMAIN_S1 = 0,
589 ARM_SMMU_DOMAIN_NESTED,
592 struct arm_smmu_domain {
593 struct arm_smmu_device *smmu;
594 struct mutex init_mutex; /* Protects smmu pointer */
596 struct io_pgtable_ops *pgtbl_ops;
597 spinlock_t pgtbl_lock;
599 enum arm_smmu_domain_stage stage;
601 struct arm_smmu_s1_cfg s1_cfg;
602 struct arm_smmu_s2_cfg s2_cfg;
605 struct iommu_domain domain;
608 /* Our list of SMMU instances */
609 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
610 static LIST_HEAD(arm_smmu_devices);
612 struct arm_smmu_option_prop {
617 static struct arm_smmu_option_prop arm_smmu_options[] = {
618 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
622 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
624 return container_of(dom, struct arm_smmu_domain, domain);
627 static void parse_driver_options(struct arm_smmu_device *smmu)
632 if (of_property_read_bool(smmu->dev->of_node,
633 arm_smmu_options[i].prop)) {
634 smmu->options |= arm_smmu_options[i].opt;
635 dev_notice(smmu->dev, "option %s\n",
636 arm_smmu_options[i].prop);
638 } while (arm_smmu_options[++i].opt);
641 /* Low-level queue manipulation functions */
642 static bool queue_full(struct arm_smmu_queue *q)
644 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
645 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
648 static bool queue_empty(struct arm_smmu_queue *q)
650 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
651 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
654 static void queue_sync_cons(struct arm_smmu_queue *q)
656 q->cons = readl_relaxed(q->cons_reg);
659 static void queue_inc_cons(struct arm_smmu_queue *q)
661 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
663 q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
664 writel(q->cons, q->cons_reg);
667 static int queue_sync_prod(struct arm_smmu_queue *q)
670 u32 prod = readl_relaxed(q->prod_reg);
672 if (Q_OVF(q, prod) != Q_OVF(q, q->prod))
679 static void queue_inc_prod(struct arm_smmu_queue *q)
681 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;
683 q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
684 writel(q->prod, q->prod_reg);
687 static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)
689 if (Q_WRP(q, q->cons) == Q_WRP(q, until))
690 return Q_IDX(q, q->cons) < Q_IDX(q, until);
692 return Q_IDX(q, q->cons) >= Q_IDX(q, until);
695 static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)
697 ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
699 while (queue_sync_cons(q), __queue_cons_before(q, until)) {
700 if (ktime_compare(ktime_get(), timeout) > 0)
714 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
718 for (i = 0; i < n_dwords; ++i)
719 *dst++ = cpu_to_le64(*src++);
722 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)
727 queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);
732 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
736 for (i = 0; i < n_dwords; ++i)
737 *dst++ = le64_to_cpu(*src++);
740 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
745 queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);
750 /* High-level queue accessors */
751 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
753 memset(cmd, 0, CMDQ_ENT_DWORDS << 3);
754 cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT;
756 switch (ent->opcode) {
757 case CMDQ_OP_TLBI_EL2_ALL:
758 case CMDQ_OP_TLBI_NSNH_ALL:
760 case CMDQ_OP_PREFETCH_CFG:
761 cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT;
762 cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT;
763 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
765 case CMDQ_OP_CFGI_STE:
766 cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT;
767 cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0;
769 case CMDQ_OP_CFGI_ALL:
770 /* Cover the entire SID range */
771 cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT;
773 case CMDQ_OP_TLBI_NH_VA:
774 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
775 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
776 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
778 case CMDQ_OP_TLBI_S2_IPA:
779 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
780 cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0;
781 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
783 case CMDQ_OP_TLBI_NH_ASID:
784 cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT;
786 case CMDQ_OP_TLBI_S12_VMALL:
787 cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT;
789 case CMDQ_OP_PRI_RESP:
790 cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0;
791 cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT;
792 cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT;
793 cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT;
794 switch (ent->pri.resp) {
796 cmd[1] |= CMDQ_PRI_1_RESP_DENY;
799 cmd[1] |= CMDQ_PRI_1_RESP_FAIL;
802 cmd[1] |= CMDQ_PRI_1_RESP_SUCC;
808 case CMDQ_OP_CMD_SYNC:
809 cmd[0] |= CMDQ_SYNC_0_CS_SEV;
818 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
820 static const char *cerror_str[] = {
821 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
822 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
823 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
827 u64 cmd[CMDQ_ENT_DWORDS];
828 struct arm_smmu_queue *q = &smmu->cmdq.q;
829 u32 cons = readl_relaxed(q->cons_reg);
830 u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK;
831 struct arm_smmu_cmdq_ent cmd_sync = {
832 .opcode = CMDQ_OP_CMD_SYNC,
835 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
839 case CMDQ_ERR_CERROR_ILL_IDX:
841 case CMDQ_ERR_CERROR_ABT_IDX:
842 dev_err(smmu->dev, "retrying command fetch\n");
843 case CMDQ_ERR_CERROR_NONE_IDX:
848 * We may have concurrent producers, so we need to be careful
849 * not to touch any of the shadow cmdq state.
851 queue_read(cmd, Q_ENT(q, idx), q->ent_dwords);
852 dev_err(smmu->dev, "skipping command in error state:\n");
853 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
854 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
856 /* Convert the erroneous command into a CMD_SYNC */
857 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
858 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
862 queue_write(cmd, Q_ENT(q, idx), q->ent_dwords);
865 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
866 struct arm_smmu_cmdq_ent *ent)
869 u64 cmd[CMDQ_ENT_DWORDS];
870 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
871 struct arm_smmu_queue *q = &smmu->cmdq.q;
873 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
874 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
879 spin_lock(&smmu->cmdq.lock);
880 while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {
882 * Keep the queue locked, otherwise the producer could wrap
883 * twice and we could see a future consumer pointer that looks
884 * like it's behind us.
886 if (queue_poll_cons(q, until, wfe))
887 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
890 if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))
891 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
892 spin_unlock(&smmu->cmdq.lock);
895 /* Context descriptor manipulation functions */
896 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
900 /* Repack the TCR. Just care about TTBR0 for now */
901 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
902 val |= ARM_SMMU_TCR2CD(tcr, TG0);
903 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
904 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
905 val |= ARM_SMMU_TCR2CD(tcr, SH0);
906 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
907 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
908 val |= ARM_SMMU_TCR2CD(tcr, IPS);
909 val |= ARM_SMMU_TCR2CD(tcr, TBI0);
914 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
915 struct arm_smmu_s1_cfg *cfg)
920 * We don't need to issue any invalidation here, as we'll invalidate
921 * the STE when installing the new entry anyway.
923 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
927 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE |
928 CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT |
930 cfg->cdptr[0] = cpu_to_le64(val);
932 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT;
933 cfg->cdptr[1] = cpu_to_le64(val);
935 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT);
938 /* Stream table manipulation functions */
940 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
944 val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK)
945 << STRTAB_L1_DESC_SPAN_SHIFT;
946 val |= desc->l2ptr_dma &
947 STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT;
949 *dst = cpu_to_le64(val);
952 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
954 struct arm_smmu_cmdq_ent cmd = {
955 .opcode = CMDQ_OP_CFGI_STE,
962 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
963 cmd.opcode = CMDQ_OP_CMD_SYNC;
964 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
967 static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
968 __le64 *dst, struct arm_smmu_strtab_ent *ste)
971 * This is hideously complicated, but we only really care about
972 * three cases at the moment:
974 * 1. Invalid (all zero) -> bypass (init)
975 * 2. Bypass -> translation (attach)
976 * 3. Translation -> bypass (detach)
978 * Given that we can't update the STE atomically and the SMMU
979 * doesn't read the thing in a defined order, that leaves us
980 * with the following maintenance requirements:
982 * 1. Update Config, return (init time STEs aren't live)
983 * 2. Write everything apart from dword 0, sync, write dword 0, sync
984 * 3. Update Config, sync
986 u64 val = le64_to_cpu(dst[0]);
987 bool ste_live = false;
988 struct arm_smmu_cmdq_ent prefetch_cmd = {
989 .opcode = CMDQ_OP_PREFETCH_CFG,
995 if (val & STRTAB_STE_0_V) {
998 cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT;
1000 case STRTAB_STE_0_CFG_BYPASS:
1002 case STRTAB_STE_0_CFG_S1_TRANS:
1003 case STRTAB_STE_0_CFG_S2_TRANS:
1007 BUG(); /* STE corruption */
1011 /* Nuke the existing Config, as we're going to rewrite it */
1012 val &= ~(STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT);
1015 val |= STRTAB_STE_0_V;
1017 val &= ~STRTAB_STE_0_V;
1020 val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT
1021 : STRTAB_STE_0_CFG_BYPASS;
1022 dst[0] = cpu_to_le64(val);
1023 dst[2] = 0; /* Nuke the VMID */
1025 arm_smmu_sync_ste_for_sid(smmu, sid);
1031 dst[1] = cpu_to_le64(
1032 STRTAB_STE_1_S1C_CACHE_WBRA
1033 << STRTAB_STE_1_S1CIR_SHIFT |
1034 STRTAB_STE_1_S1C_CACHE_WBRA
1035 << STRTAB_STE_1_S1COR_SHIFT |
1036 STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT |
1037 STRTAB_STE_1_S1STALLD |
1038 #ifdef CONFIG_PCI_ATS
1039 STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT |
1041 STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT);
1043 val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK
1044 << STRTAB_STE_0_S1CTXPTR_SHIFT) |
1045 STRTAB_STE_0_CFG_S1_TRANS;
1051 dst[2] = cpu_to_le64(
1052 ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT |
1053 (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK)
1054 << STRTAB_STE_2_VTCR_SHIFT |
1056 STRTAB_STE_2_S2ENDI |
1058 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1061 dst[3] = cpu_to_le64(ste->s2_cfg->vttbr &
1062 STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT);
1064 val |= STRTAB_STE_0_CFG_S2_TRANS;
1067 arm_smmu_sync_ste_for_sid(smmu, sid);
1068 dst[0] = cpu_to_le64(val);
1069 arm_smmu_sync_ste_for_sid(smmu, sid);
1071 /* It's likely that we'll want to use the new STE soon */
1072 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1073 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1076 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1079 struct arm_smmu_strtab_ent ste = {
1084 for (i = 0; i < nent; ++i) {
1085 arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste);
1086 strtab += STRTAB_STE_DWORDS;
1090 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1094 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1095 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1100 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1101 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1103 desc->span = STRTAB_SPLIT + 1;
1104 desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1108 "failed to allocate l2 stream table for SID %u\n",
1113 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1114 arm_smmu_write_strtab_l1_desc(strtab, desc);
1118 /* IRQ and event handlers */
1119 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1122 struct arm_smmu_device *smmu = dev;
1123 struct arm_smmu_queue *q = &smmu->evtq.q;
1124 u64 evt[EVTQ_ENT_DWORDS];
1126 while (!queue_remove_raw(q, evt)) {
1127 u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
1129 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1130 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1131 dev_info(smmu->dev, "\t0x%016llx\n",
1132 (unsigned long long)evt[i]);
1135 /* Sync our overflow flag, as we believe we're up to speed */
1136 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1140 static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev)
1142 irqreturn_t ret = IRQ_WAKE_THREAD;
1143 struct arm_smmu_device *smmu = dev;
1144 struct arm_smmu_queue *q = &smmu->evtq.q;
1147 * Not much we can do on overflow, so scream and pretend we're
1150 if (queue_sync_prod(q) == -EOVERFLOW)
1151 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1152 else if (queue_empty(q))
1158 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1160 struct arm_smmu_device *smmu = dev;
1161 struct arm_smmu_queue *q = &smmu->priq.q;
1162 u64 evt[PRIQ_ENT_DWORDS];
1164 while (!queue_remove_raw(q, evt)) {
1169 sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK;
1170 ssv = evt[0] & PRIQ_0_SSID_V;
1171 ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0;
1172 last = evt[0] & PRIQ_0_PRG_LAST;
1173 grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK;
1175 dev_info(smmu->dev, "unexpected PRI request received:\n");
1177 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1178 sid, ssid, grpid, last ? "L" : "",
1179 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1180 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1181 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1182 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1183 evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT);
1186 struct arm_smmu_cmdq_ent cmd = {
1187 .opcode = CMDQ_OP_PRI_RESP,
1188 .substream_valid = ssv,
1193 .resp = PRI_RESP_DENY,
1197 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1201 /* Sync our overflow flag, as we believe we're up to speed */
1202 q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);
1206 static irqreturn_t arm_smmu_priq_handler(int irq, void *dev)
1208 irqreturn_t ret = IRQ_WAKE_THREAD;
1209 struct arm_smmu_device *smmu = dev;
1210 struct arm_smmu_queue *q = &smmu->priq.q;
1212 /* PRIQ overflow indicates a programming error */
1213 if (queue_sync_prod(q) == -EOVERFLOW)
1214 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1215 else if (queue_empty(q))
1221 static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev)
1223 /* We don't actually use CMD_SYNC interrupts for anything */
1227 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1229 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1231 u32 gerror, gerrorn;
1232 struct arm_smmu_device *smmu = dev;
1234 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1235 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1238 if (!(gerror & GERROR_ERR_MASK))
1239 return IRQ_NONE; /* No errors pending */
1242 "unexpected global error reported (0x%08x), this could be serious\n",
1245 if (gerror & GERROR_SFM_ERR) {
1246 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1247 arm_smmu_device_disable(smmu);
1250 if (gerror & GERROR_MSI_GERROR_ABT_ERR)
1251 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1253 if (gerror & GERROR_MSI_PRIQ_ABT_ERR) {
1254 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1255 arm_smmu_priq_handler(irq, smmu->dev);
1258 if (gerror & GERROR_MSI_EVTQ_ABT_ERR) {
1259 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1260 arm_smmu_evtq_handler(irq, smmu->dev);
1263 if (gerror & GERROR_MSI_CMDQ_ABT_ERR) {
1264 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1265 arm_smmu_cmdq_sync_handler(irq, smmu->dev);
1268 if (gerror & GERROR_PRIQ_ABT_ERR)
1269 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1271 if (gerror & GERROR_EVTQ_ABT_ERR)
1272 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1274 if (gerror & GERROR_CMDQ_ERR)
1275 arm_smmu_cmdq_skip_err(smmu);
1277 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1281 /* IO_PGTABLE API */
1282 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
1284 struct arm_smmu_cmdq_ent cmd;
1286 cmd.opcode = CMDQ_OP_CMD_SYNC;
1287 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1290 static void arm_smmu_tlb_sync(void *cookie)
1292 struct arm_smmu_domain *smmu_domain = cookie;
1293 __arm_smmu_tlb_sync(smmu_domain->smmu);
1296 static void arm_smmu_tlb_inv_context(void *cookie)
1298 struct arm_smmu_domain *smmu_domain = cookie;
1299 struct arm_smmu_device *smmu = smmu_domain->smmu;
1300 struct arm_smmu_cmdq_ent cmd;
1302 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1303 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1304 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1307 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1308 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1311 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1312 __arm_smmu_tlb_sync(smmu);
1315 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
1316 bool leaf, void *cookie)
1318 struct arm_smmu_domain *smmu_domain = cookie;
1319 struct arm_smmu_device *smmu = smmu_domain->smmu;
1320 struct arm_smmu_cmdq_ent cmd = {
1327 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1328 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
1329 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1331 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
1332 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1335 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1338 static struct iommu_gather_ops arm_smmu_gather_ops = {
1339 .tlb_flush_all = arm_smmu_tlb_inv_context,
1340 .tlb_add_flush = arm_smmu_tlb_inv_range_nosync,
1341 .tlb_sync = arm_smmu_tlb_sync,
1345 static bool arm_smmu_capable(enum iommu_cap cap)
1348 case IOMMU_CAP_CACHE_COHERENCY:
1350 case IOMMU_CAP_INTR_REMAP:
1351 return true; /* MSIs are just memory writes */
1352 case IOMMU_CAP_NOEXEC:
1359 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1361 struct arm_smmu_domain *smmu_domain;
1363 if (type != IOMMU_DOMAIN_UNMANAGED)
1367 * Allocate the domain and initialise some of its data structures.
1368 * We can't really do anything meaningful until we've added a
1371 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1375 mutex_init(&smmu_domain->init_mutex);
1376 spin_lock_init(&smmu_domain->pgtbl_lock);
1377 return &smmu_domain->domain;
1380 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
1382 int idx, size = 1 << span;
1385 idx = find_first_zero_bit(map, size);
1388 } while (test_and_set_bit(idx, map));
1393 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
1395 clear_bit(idx, map);
1398 static void arm_smmu_domain_free(struct iommu_domain *domain)
1400 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1401 struct arm_smmu_device *smmu = smmu_domain->smmu;
1403 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1405 /* Free the CD and ASID, if we allocated them */
1406 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1407 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1410 dma_free_coherent(smmu_domain->smmu->dev,
1411 CTXDESC_CD_DWORDS << 3,
1415 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
1418 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1420 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
1426 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
1427 struct io_pgtable_cfg *pgtbl_cfg)
1431 struct arm_smmu_device *smmu = smmu_domain->smmu;
1432 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1434 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
1435 if (IS_ERR_VALUE(asid))
1438 cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
1439 &cfg->cdptr_dma, GFP_KERNEL);
1441 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1445 cfg->cd.asid = asid;
1446 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
1447 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1448 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
1452 arm_smmu_bitmap_free(smmu->asid_map, asid);
1456 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
1457 struct io_pgtable_cfg *pgtbl_cfg)
1460 struct arm_smmu_device *smmu = smmu_domain->smmu;
1461 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
1463 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
1464 if (IS_ERR_VALUE(vmid))
1468 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
1469 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1473 static struct iommu_ops arm_smmu_ops;
1475 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
1478 unsigned long ias, oas;
1479 enum io_pgtable_fmt fmt;
1480 struct io_pgtable_cfg pgtbl_cfg;
1481 struct io_pgtable_ops *pgtbl_ops;
1482 int (*finalise_stage_fn)(struct arm_smmu_domain *,
1483 struct io_pgtable_cfg *);
1484 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1485 struct arm_smmu_device *smmu = smmu_domain->smmu;
1487 /* Restrict the stage to what we can actually support */
1488 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
1489 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
1490 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
1491 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1493 switch (smmu_domain->stage) {
1494 case ARM_SMMU_DOMAIN_S1:
1497 fmt = ARM_64_LPAE_S1;
1498 finalise_stage_fn = arm_smmu_domain_finalise_s1;
1500 case ARM_SMMU_DOMAIN_NESTED:
1501 case ARM_SMMU_DOMAIN_S2:
1504 fmt = ARM_64_LPAE_S2;
1505 finalise_stage_fn = arm_smmu_domain_finalise_s2;
1511 pgtbl_cfg = (struct io_pgtable_cfg) {
1512 .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap,
1515 .tlb = &arm_smmu_gather_ops,
1516 .iommu_dev = smmu->dev,
1519 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1523 arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1524 smmu_domain->pgtbl_ops = pgtbl_ops;
1526 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
1527 if (IS_ERR_VALUE(ret))
1528 free_io_pgtable_ops(pgtbl_ops);
1533 static struct arm_smmu_group *arm_smmu_group_get(struct device *dev)
1535 struct iommu_group *group;
1536 struct arm_smmu_group *smmu_group;
1538 group = iommu_group_get(dev);
1542 smmu_group = iommu_group_get_iommudata(group);
1543 iommu_group_put(group);
1547 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
1550 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1552 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1553 struct arm_smmu_strtab_l1_desc *l1_desc;
1556 /* Two-level walk */
1557 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
1558 l1_desc = &cfg->l1_desc[idx];
1559 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
1560 step = &l1_desc->l2ptr[idx];
1562 /* Simple linear lookup */
1563 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
1569 static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group)
1572 struct arm_smmu_domain *smmu_domain = smmu_group->domain;
1573 struct arm_smmu_strtab_ent *ste = &smmu_group->ste;
1574 struct arm_smmu_device *smmu = smmu_group->smmu;
1576 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1577 ste->s1_cfg = &smmu_domain->s1_cfg;
1579 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg);
1582 ste->s2_cfg = &smmu_domain->s2_cfg;
1585 for (i = 0; i < smmu_group->num_sids; ++i) {
1586 u32 sid = smmu_group->sids[i];
1587 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
1589 arm_smmu_write_strtab_ent(smmu, sid, step, ste);
1595 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1598 struct arm_smmu_device *smmu;
1599 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1600 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1605 /* Already attached to a different domain? */
1606 if (smmu_group->domain && smmu_group->domain != smmu_domain)
1609 smmu = smmu_group->smmu;
1610 mutex_lock(&smmu_domain->init_mutex);
1612 if (!smmu_domain->smmu) {
1613 smmu_domain->smmu = smmu;
1614 ret = arm_smmu_domain_finalise(domain);
1616 smmu_domain->smmu = NULL;
1619 } else if (smmu_domain->smmu != smmu) {
1621 "cannot attach to SMMU %s (upstream of %s)\n",
1622 dev_name(smmu_domain->smmu->dev),
1623 dev_name(smmu->dev));
1628 /* Group already attached to this domain? */
1629 if (smmu_group->domain)
1632 smmu_group->domain = smmu_domain;
1633 smmu_group->ste.bypass = false;
1635 ret = arm_smmu_install_ste_for_group(smmu_group);
1636 if (IS_ERR_VALUE(ret))
1637 smmu_group->domain = NULL;
1640 mutex_unlock(&smmu_domain->init_mutex);
1644 static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
1646 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1647 struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev);
1649 BUG_ON(!smmu_domain);
1650 BUG_ON(!smmu_group);
1652 mutex_lock(&smmu_domain->init_mutex);
1653 BUG_ON(smmu_group->domain != smmu_domain);
1655 smmu_group->ste.bypass = true;
1656 if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group)))
1657 dev_warn(dev, "failed to install bypass STE\n");
1659 smmu_group->domain = NULL;
1660 mutex_unlock(&smmu_domain->init_mutex);
1663 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1664 phys_addr_t paddr, size_t size, int prot)
1667 unsigned long flags;
1668 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1669 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1674 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1675 ret = ops->map(ops, iova, paddr, size, prot);
1676 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1681 arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
1684 unsigned long flags;
1685 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1686 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1691 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1692 ret = ops->unmap(ops, iova, size);
1693 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1698 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1701 unsigned long flags;
1702 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1703 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1708 spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1709 ret = ops->iova_to_phys(ops, iova);
1710 spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1715 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp)
1717 *(u32 *)sidp = alias;
1718 return 0; /* Continue walking */
1721 static void __arm_smmu_release_pci_iommudata(void *data)
1726 static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev)
1728 struct device_node *of_node;
1729 struct arm_smmu_device *curr, *smmu = NULL;
1730 struct pci_bus *bus = pdev->bus;
1732 /* Walk up to the root bus */
1733 while (!pci_is_root_bus(bus))
1736 /* Follow the "iommus" phandle from the host controller */
1737 of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0);
1741 /* See if we can find an SMMU corresponding to the phandle */
1742 spin_lock(&arm_smmu_devices_lock);
1743 list_for_each_entry(curr, &arm_smmu_devices, list) {
1744 if (curr->dev->of_node == of_node) {
1749 spin_unlock(&arm_smmu_devices_lock);
1750 of_node_put(of_node);
1754 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
1756 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
1758 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
1759 limit *= 1UL << STRTAB_SPLIT;
1764 static int arm_smmu_add_device(struct device *dev)
1768 struct pci_dev *pdev;
1769 struct iommu_group *group;
1770 struct arm_smmu_group *smmu_group;
1771 struct arm_smmu_device *smmu;
1773 /* We only support PCI, for now */
1774 if (!dev_is_pci(dev))
1777 pdev = to_pci_dev(dev);
1778 group = iommu_group_get_for_dev(dev);
1780 return PTR_ERR(group);
1782 smmu_group = iommu_group_get_iommudata(group);
1784 smmu = arm_smmu_get_for_pci_dev(pdev);
1790 smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL);
1796 smmu_group->ste.valid = true;
1797 smmu_group->smmu = smmu;
1798 iommu_group_set_iommudata(group, smmu_group,
1799 __arm_smmu_release_pci_iommudata);
1801 smmu = smmu_group->smmu;
1804 /* Assume SID == RID until firmware tells us otherwise */
1805 pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid);
1806 for (i = 0; i < smmu_group->num_sids; ++i) {
1807 /* If we already know about this SID, then we're done */
1808 if (smmu_group->sids[i] == sid)
1812 /* Check the SID is in range of the SMMU and our stream table */
1813 if (!arm_smmu_sid_in_range(smmu, sid)) {
1818 /* Ensure l2 strtab is initialised */
1819 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
1820 ret = arm_smmu_init_l2_strtab(smmu, sid);
1825 /* Resize the SID array for the group */
1826 smmu_group->num_sids++;
1827 sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids),
1830 smmu_group->num_sids--;
1835 /* Add the new SID */
1836 sids[smmu_group->num_sids - 1] = sid;
1837 smmu_group->sids = sids;
1841 iommu_group_put(group);
1845 static void arm_smmu_remove_device(struct device *dev)
1847 iommu_group_remove_device(dev);
1850 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1851 enum iommu_attr attr, void *data)
1853 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1856 case DOMAIN_ATTR_NESTING:
1857 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1864 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1865 enum iommu_attr attr, void *data)
1868 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1870 mutex_lock(&smmu_domain->init_mutex);
1873 case DOMAIN_ATTR_NESTING:
1874 if (smmu_domain->smmu) {
1880 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1882 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1890 mutex_unlock(&smmu_domain->init_mutex);
1894 static struct iommu_ops arm_smmu_ops = {
1895 .capable = arm_smmu_capable,
1896 .domain_alloc = arm_smmu_domain_alloc,
1897 .domain_free = arm_smmu_domain_free,
1898 .attach_dev = arm_smmu_attach_dev,
1899 .detach_dev = arm_smmu_detach_dev,
1900 .map = arm_smmu_map,
1901 .unmap = arm_smmu_unmap,
1902 .iova_to_phys = arm_smmu_iova_to_phys,
1903 .add_device = arm_smmu_add_device,
1904 .remove_device = arm_smmu_remove_device,
1905 .domain_get_attr = arm_smmu_domain_get_attr,
1906 .domain_set_attr = arm_smmu_domain_set_attr,
1907 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1910 /* Probing and initialisation functions */
1911 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
1912 struct arm_smmu_queue *q,
1913 unsigned long prod_off,
1914 unsigned long cons_off,
1917 size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;
1919 q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);
1921 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n",
1926 q->prod_reg = smmu->base + prod_off;
1927 q->cons_reg = smmu->base + cons_off;
1928 q->ent_dwords = dwords;
1930 q->q_base = Q_BASE_RWA;
1931 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;
1932 q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)
1933 << Q_BASE_LOG2SIZE_SHIFT;
1935 q->prod = q->cons = 0;
1939 static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu,
1940 struct arm_smmu_queue *q)
1942 size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;
1944 dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);
1947 static void arm_smmu_free_queues(struct arm_smmu_device *smmu)
1949 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
1950 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
1952 if (smmu->features & ARM_SMMU_FEAT_PRI)
1953 arm_smmu_free_one_queue(smmu, &smmu->priq.q);
1956 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
1961 spin_lock_init(&smmu->cmdq.lock);
1962 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
1963 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS);
1968 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
1969 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS);
1974 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
1977 ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
1978 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS);
1985 arm_smmu_free_one_queue(smmu, &smmu->evtq.q);
1987 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);
1992 static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu)
1996 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1998 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1999 for (i = 0; i < cfg->num_l1_ents; ++i) {
2000 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i];
2005 dma_free_coherent(smmu->dev, size, desc->l2ptr,
2010 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2013 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2014 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2015 void *strtab = smmu->strtab_cfg.strtab;
2017 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2018 if (!cfg->l1_desc) {
2019 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2023 for (i = 0; i < cfg->num_l1_ents; ++i) {
2024 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2025 strtab += STRTAB_L1_DESC_DWORDS << 3;
2031 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2037 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2040 * If we can resolve everything with a single L2 table, then we
2041 * just need a single L1 descriptor. Otherwise, calculate the L1
2042 * size, capped to the SIDSIZE.
2044 if (smmu->sid_bits < STRTAB_SPLIT) {
2047 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2048 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2050 cfg->num_l1_ents = 1 << size;
2052 size += STRTAB_SPLIT;
2053 if (size < smmu->sid_bits)
2055 "2-level strtab only covers %u/%u bits of SID\n",
2056 size, smmu->sid_bits);
2058 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2059 strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2063 "failed to allocate l1 stream table (%u bytes)\n",
2067 cfg->strtab = strtab;
2069 /* Configure strtab_base_cfg for 2 levels */
2070 reg = STRTAB_BASE_CFG_FMT_2LVL;
2071 reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2072 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2073 reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK)
2074 << STRTAB_BASE_CFG_SPLIT_SHIFT;
2075 cfg->strtab_base_cfg = reg;
2077 ret = arm_smmu_init_l1_strtab(smmu);
2079 dma_free_coherent(smmu->dev,
2086 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2091 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2093 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2094 strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2098 "failed to allocate linear stream table (%u bytes)\n",
2102 cfg->strtab = strtab;
2103 cfg->num_l1_ents = 1 << smmu->sid_bits;
2105 /* Configure strtab_base_cfg for a linear table covering all SIDs */
2106 reg = STRTAB_BASE_CFG_FMT_LINEAR;
2107 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK)
2108 << STRTAB_BASE_CFG_LOG2SIZE_SHIFT;
2109 cfg->strtab_base_cfg = reg;
2111 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2115 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2120 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2121 ret = arm_smmu_init_strtab_2lvl(smmu);
2123 ret = arm_smmu_init_strtab_linear(smmu);
2128 /* Set the strtab base address */
2129 reg = smmu->strtab_cfg.strtab_dma &
2130 STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT;
2131 reg |= STRTAB_BASE_RA;
2132 smmu->strtab_cfg.strtab_base = reg;
2134 /* Allocate the first VMID for stage-2 bypass STEs */
2135 set_bit(0, smmu->vmid_map);
2139 static void arm_smmu_free_strtab(struct arm_smmu_device *smmu)
2141 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2142 u32 size = cfg->num_l1_ents;
2144 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2145 arm_smmu_free_l2_strtab(smmu);
2146 size *= STRTAB_L1_DESC_DWORDS << 3;
2148 size *= STRTAB_STE_DWORDS * 3;
2151 dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma);
2154 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2158 ret = arm_smmu_init_queues(smmu);
2162 ret = arm_smmu_init_strtab(smmu);
2164 goto out_free_queues;
2169 arm_smmu_free_queues(smmu);
2173 static void arm_smmu_free_structures(struct arm_smmu_device *smmu)
2175 arm_smmu_free_strtab(smmu);
2176 arm_smmu_free_queues(smmu);
2179 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2180 unsigned int reg_off, unsigned int ack_off)
2184 writel_relaxed(val, smmu->base + reg_off);
2185 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2186 1, ARM_SMMU_POLL_TIMEOUT_US);
2189 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
2192 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
2194 /* Disable IRQs first */
2195 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
2196 ARM_SMMU_IRQ_CTRLACK);
2198 dev_err(smmu->dev, "failed to disable irqs\n");
2202 /* Clear the MSI address regs */
2203 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
2204 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
2206 /* Request wired interrupt lines */
2207 irq = smmu->evtq.q.irq;
2209 ret = devm_request_threaded_irq(smmu->dev, irq,
2210 arm_smmu_evtq_handler,
2211 arm_smmu_evtq_thread,
2212 0, "arm-smmu-v3-evtq", smmu);
2213 if (IS_ERR_VALUE(ret))
2214 dev_warn(smmu->dev, "failed to enable evtq irq\n");
2217 irq = smmu->cmdq.q.irq;
2219 ret = devm_request_irq(smmu->dev, irq,
2220 arm_smmu_cmdq_sync_handler, 0,
2221 "arm-smmu-v3-cmdq-sync", smmu);
2222 if (IS_ERR_VALUE(ret))
2223 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n");
2226 irq = smmu->gerr_irq;
2228 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
2229 0, "arm-smmu-v3-gerror", smmu);
2230 if (IS_ERR_VALUE(ret))
2231 dev_warn(smmu->dev, "failed to enable gerror irq\n");
2234 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2235 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
2237 irq = smmu->priq.q.irq;
2239 ret = devm_request_threaded_irq(smmu->dev, irq,
2240 arm_smmu_priq_handler,
2241 arm_smmu_priq_thread,
2242 0, "arm-smmu-v3-priq",
2244 if (IS_ERR_VALUE(ret))
2246 "failed to enable priq irq\n");
2248 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
2252 /* Enable interrupt generation on the SMMU */
2253 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
2254 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
2256 dev_warn(smmu->dev, "failed to enable irqs\n");
2261 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
2265 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
2267 dev_err(smmu->dev, "failed to clear cr0\n");
2272 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
2276 struct arm_smmu_cmdq_ent cmd;
2278 /* Clear CR0 and sync (disables SMMU and queue processing) */
2279 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
2280 if (reg & CR0_SMMUEN)
2281 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
2283 ret = arm_smmu_device_disable(smmu);
2287 /* CR1 (table and queue memory attributes) */
2288 reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) |
2289 (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) |
2290 (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) |
2291 (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) |
2292 (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) |
2293 (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT);
2294 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
2296 /* CR2 (random crap) */
2297 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
2298 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
2301 writeq_relaxed(smmu->strtab_cfg.strtab_base,
2302 smmu->base + ARM_SMMU_STRTAB_BASE);
2303 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
2304 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
2307 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
2308 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
2309 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
2311 enables = CR0_CMDQEN;
2312 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2315 dev_err(smmu->dev, "failed to enable command queue\n");
2319 /* Invalidate any cached configuration */
2320 cmd.opcode = CMDQ_OP_CFGI_ALL;
2321 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2322 cmd.opcode = CMDQ_OP_CMD_SYNC;
2323 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2325 /* Invalidate any stale TLB entries */
2326 if (smmu->features & ARM_SMMU_FEAT_HYP) {
2327 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
2328 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2331 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
2332 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2333 cmd.opcode = CMDQ_OP_CMD_SYNC;
2334 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2337 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
2338 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);
2339 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);
2341 enables |= CR0_EVTQEN;
2342 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2345 dev_err(smmu->dev, "failed to enable event queue\n");
2350 if (smmu->features & ARM_SMMU_FEAT_PRI) {
2351 writeq_relaxed(smmu->priq.q.q_base,
2352 smmu->base + ARM_SMMU_PRIQ_BASE);
2353 writel_relaxed(smmu->priq.q.prod,
2354 smmu->base + ARM_SMMU_PRIQ_PROD);
2355 writel_relaxed(smmu->priq.q.cons,
2356 smmu->base + ARM_SMMU_PRIQ_CONS);
2358 enables |= CR0_PRIQEN;
2359 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2362 dev_err(smmu->dev, "failed to enable PRI queue\n");
2367 ret = arm_smmu_setup_irqs(smmu);
2369 dev_err(smmu->dev, "failed to setup irqs\n");
2373 /* Enable the SMMU interface */
2374 enables |= CR0_SMMUEN;
2375 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
2378 dev_err(smmu->dev, "failed to enable SMMU interface\n");
2385 static int arm_smmu_device_probe(struct arm_smmu_device *smmu)
2389 unsigned long pgsize_bitmap = 0;
2392 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
2394 /* 2-level structures */
2395 if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL)
2396 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
2398 if (reg & IDR0_CD2L)
2399 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
2402 * Translation table endianness.
2403 * We currently require the same endianness as the CPU, but this
2404 * could be changed later by adding a new IO_PGTABLE_QUIRK.
2406 switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) {
2407 case IDR0_TTENDIAN_MIXED:
2408 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
2411 case IDR0_TTENDIAN_BE:
2412 smmu->features |= ARM_SMMU_FEAT_TT_BE;
2415 case IDR0_TTENDIAN_LE:
2416 smmu->features |= ARM_SMMU_FEAT_TT_LE;
2420 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
2424 /* Boolean feature flags */
2425 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
2426 smmu->features |= ARM_SMMU_FEAT_PRI;
2428 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
2429 smmu->features |= ARM_SMMU_FEAT_ATS;
2432 smmu->features |= ARM_SMMU_FEAT_SEV;
2435 smmu->features |= ARM_SMMU_FEAT_MSI;
2438 smmu->features |= ARM_SMMU_FEAT_HYP;
2441 * The dma-coherent property is used in preference to the ID
2442 * register, but warn on mismatch.
2444 coherent = of_dma_is_coherent(smmu->dev->of_node);
2446 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
2448 if (!!(reg & IDR0_COHACC) != coherent)
2449 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n",
2450 coherent ? "true" : "false");
2452 if (reg & IDR0_STALL_MODEL)
2453 smmu->features |= ARM_SMMU_FEAT_STALLS;
2456 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
2459 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
2461 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
2462 dev_err(smmu->dev, "no translation support!\n");
2466 /* We only support the AArch64 table format at present */
2467 switch (reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) {
2468 case IDR0_TTF_AARCH32_64:
2471 case IDR0_TTF_AARCH64:
2474 dev_err(smmu->dev, "AArch64 table format not supported!\n");
2478 /* ASID/VMID sizes */
2479 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
2480 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
2483 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
2484 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
2485 dev_err(smmu->dev, "embedded implementation not supported\n");
2489 /* Queue sizes, capped at 4k */
2490 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,
2491 reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK);
2492 if (!smmu->cmdq.q.max_n_shift) {
2493 /* Odd alignment restrictions on the base, so ignore for now */
2494 dev_err(smmu->dev, "unit-length command queue not supported\n");
2498 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,
2499 reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK);
2500 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,
2501 reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK);
2503 /* SID/SSID sizes */
2504 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK;
2505 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK;
2508 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
2510 /* Maximum number of outstanding stalls */
2511 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT
2512 & IDR5_STALL_MAX_MASK;
2515 if (reg & IDR5_GRAN64K)
2516 pgsize_bitmap |= SZ_64K | SZ_512M;
2517 if (reg & IDR5_GRAN16K)
2518 pgsize_bitmap |= SZ_16K | SZ_32M;
2519 if (reg & IDR5_GRAN4K)
2520 pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2522 arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap;
2524 /* Output address size */
2525 switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) {
2526 case IDR5_OAS_32_BIT:
2529 case IDR5_OAS_36_BIT:
2532 case IDR5_OAS_40_BIT:
2535 case IDR5_OAS_42_BIT:
2538 case IDR5_OAS_44_BIT:
2543 "unknown output address size. Truncating to 48-bit\n");
2545 case IDR5_OAS_48_BIT:
2549 /* Set the DMA mask for our table walker */
2550 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
2552 "failed to set DMA mask for table walker\n");
2554 smmu->ias = max(smmu->ias, smmu->oas);
2556 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
2557 smmu->ias, smmu->oas, smmu->features);
2561 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
2564 struct resource *res;
2565 struct arm_smmu_device *smmu;
2566 struct device *dev = &pdev->dev;
2568 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2570 dev_err(dev, "failed to allocate arm_smmu_device\n");
2576 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2577 if (resource_size(res) + 1 < SZ_128K) {
2578 dev_err(dev, "MMIO region too small (%pr)\n", res);
2582 smmu->base = devm_ioremap_resource(dev, res);
2583 if (IS_ERR(smmu->base))
2584 return PTR_ERR(smmu->base);
2586 /* Interrupt lines */
2587 irq = platform_get_irq_byname(pdev, "eventq");
2589 smmu->evtq.q.irq = irq;
2591 irq = platform_get_irq_byname(pdev, "priq");
2593 smmu->priq.q.irq = irq;
2595 irq = platform_get_irq_byname(pdev, "cmdq-sync");
2597 smmu->cmdq.q.irq = irq;
2599 irq = platform_get_irq_byname(pdev, "gerror");
2601 smmu->gerr_irq = irq;
2603 parse_driver_options(smmu);
2606 ret = arm_smmu_device_probe(smmu);
2610 /* Initialise in-memory data structures */
2611 ret = arm_smmu_init_structures(smmu);
2615 /* Reset the device */
2616 ret = arm_smmu_device_reset(smmu);
2618 goto out_free_structures;
2620 /* Record our private device structure */
2621 INIT_LIST_HEAD(&smmu->list);
2622 spin_lock(&arm_smmu_devices_lock);
2623 list_add(&smmu->list, &arm_smmu_devices);
2624 spin_unlock(&arm_smmu_devices_lock);
2627 out_free_structures:
2628 arm_smmu_free_structures(smmu);
2632 static int arm_smmu_device_remove(struct platform_device *pdev)
2634 struct arm_smmu_device *curr, *smmu = NULL;
2635 struct device *dev = &pdev->dev;
2637 spin_lock(&arm_smmu_devices_lock);
2638 list_for_each_entry(curr, &arm_smmu_devices, list) {
2639 if (curr->dev == dev) {
2641 list_del(&smmu->list);
2645 spin_unlock(&arm_smmu_devices_lock);
2650 arm_smmu_device_disable(smmu);
2651 arm_smmu_free_structures(smmu);
2655 static struct of_device_id arm_smmu_of_match[] = {
2656 { .compatible = "arm,smmu-v3", },
2659 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2661 static struct platform_driver arm_smmu_driver = {
2663 .name = "arm-smmu-v3",
2664 .of_match_table = of_match_ptr(arm_smmu_of_match),
2666 .probe = arm_smmu_device_dt_probe,
2667 .remove = arm_smmu_device_remove,
2670 static int __init arm_smmu_init(void)
2672 struct device_node *np;
2675 np = of_find_matching_node(NULL, arm_smmu_of_match);
2681 ret = platform_driver_register(&arm_smmu_driver);
2685 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2688 static void __exit arm_smmu_exit(void)
2690 return platform_driver_unregister(&arm_smmu_driver);
2693 subsys_initcall(arm_smmu_init);
2694 module_exit(arm_smmu_exit);
2696 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
2697 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2698 MODULE_LICENSE("GPL v2");