drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/iommu.h>
  43 #include <linux/iopoll.h>
  44 #include <linux/module.h>
  45 #include <linux/of.h>
  46 #include <linux/of_address.h>
  47 #include <linux/of_device.h>
  48 #include <linux/of_iommu.h>
  49 #include <linux/pci.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52 #include <linux/spinlock.h>
  53
  54 #include <linux/amba/bus.h>
  55
  56 #include "io-pgtable.h"
  57
  58 /* Maximum number of context banks per SMMU */
  59 #define ARM_SMMU_MAX_CBS                128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_EXIDENABLE                 (1 << 3)
  92 #define sCR0_GCFGFRE                    (1 << 4)
  93 #define sCR0_GCFGFIE                    (1 << 5)
  94 #define sCR0_USFCFG                     (1 << 10)
  95 #define sCR0_VMIDPNE                    (1 << 11)
  96 #define sCR0_PTM                        (1 << 12)
  97 #define sCR0_FB                         (1 << 13)
  98 #define sCR0_VMID16EN                   (1 << 31)
  99 #define sCR0_BSU_SHIFT                  14
 100 #define sCR0_BSU_MASK                   0x3
 101
 102 /* Auxiliary Configuration register */
 103 #define ARM_SMMU_GR0_sACR               0x10
 104
 105 /* Identification registers */
 106 #define ARM_SMMU_GR0_ID0                0x20
 107 #define ARM_SMMU_GR0_ID1                0x24
 108 #define ARM_SMMU_GR0_ID2                0x28
 109 #define ARM_SMMU_GR0_ID3                0x2c
 110 #define ARM_SMMU_GR0_ID4                0x30
 111 #define ARM_SMMU_GR0_ID5                0x34
 112 #define ARM_SMMU_GR0_ID6                0x38
 113 #define ARM_SMMU_GR0_ID7                0x3c
 114 #define ARM_SMMU_GR0_sGFSR              0x48
 115 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 116 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 117 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 118
 119 #define ID0_S1TS                        (1 << 30)
 120 #define ID0_S2TS                        (1 << 29)
 121 #define ID0_NTS                         (1 << 28)
 122 #define ID0_SMS                         (1 << 27)
 123 #define ID0_ATOSNS                      (1 << 26)
 124 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 125 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 126 #define ID0_CTTW                        (1 << 14)
 127 #define ID0_NUMIRPT_SHIFT               16
 128 #define ID0_NUMIRPT_MASK                0xff
 129 #define ID0_NUMSIDB_SHIFT               9
 130 #define ID0_NUMSIDB_MASK                0xf
 131 #define ID0_EXIDS                       (1 << 8)
 132 #define ID0_NUMSMRG_SHIFT               0
 133 #define ID0_NUMSMRG_MASK                0xff
 134
 135 #define ID1_PAGESIZE                    (1 << 31)
 136 #define ID1_NUMPAGENDXB_SHIFT           28
 137 #define ID1_NUMPAGENDXB_MASK            7
 138 #define ID1_NUMS2CB_SHIFT               16
 139 #define ID1_NUMS2CB_MASK                0xff
 140 #define ID1_NUMCB_SHIFT                 0
 141 #define ID1_NUMCB_MASK                  0xff
 142
 143 #define ID2_OAS_SHIFT                   4
 144 #define ID2_OAS_MASK                    0xf
 145 #define ID2_IAS_SHIFT                   0
 146 #define ID2_IAS_MASK                    0xf
 147 #define ID2_UBS_SHIFT                   8
 148 #define ID2_UBS_MASK                    0xf
 149 #define ID2_PTFS_4K                     (1 << 12)
 150 #define ID2_PTFS_16K                    (1 << 13)
 151 #define ID2_PTFS_64K                    (1 << 14)
 152 #define ID2_VMID16                      (1 << 15)
 153
 154 #define ID7_MAJOR_SHIFT                 4
 155 #define ID7_MAJOR_MASK                  0xf
 156
 157 /* Global TLB invalidation */
 158 #define ARM_SMMU_GR0_TLBIVMID           0x64
 159 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 160 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 161 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 162 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 163 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 164 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 165
 166 /* Stream mapping registers */
 167 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 168 #define SMR_VALID                       (1 << 31)
 169 #define SMR_MASK_SHIFT                  16
 170 #define SMR_ID_SHIFT                    0
 171
 172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 173 #define S2CR_CBNDX_SHIFT                0
 174 #define S2CR_CBNDX_MASK                 0xff
 175 #define S2CR_EXIDVALID                  (1 << 10)
 176 #define S2CR_TYPE_SHIFT                 16
 177 #define S2CR_TYPE_MASK                  0x3
 178 enum arm_smmu_s2cr_type {
 179         S2CR_TYPE_TRANS,
 180         S2CR_TYPE_BYPASS,
 181         S2CR_TYPE_FAULT,
 182 };
 183
 184 #define S2CR_PRIVCFG_SHIFT              24
 185 #define S2CR_PRIVCFG_MASK               0x3
 186 enum arm_smmu_s2cr_privcfg {
 187         S2CR_PRIVCFG_DEFAULT,
 188         S2CR_PRIVCFG_DIPAN,
 189         S2CR_PRIVCFG_UNPRIV,
 190         S2CR_PRIVCFG_PRIV,
 191 };
 192
 193 /* Context bank attribute registers */
 194 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 195 #define CBAR_VMID_SHIFT                 0
 196 #define CBAR_VMID_MASK                  0xff
 197 #define CBAR_S1_BPSHCFG_SHIFT           8
 198 #define CBAR_S1_BPSHCFG_MASK            3
 199 #define CBAR_S1_BPSHCFG_NSH             3
 200 #define CBAR_S1_MEMATTR_SHIFT           12
 201 #define CBAR_S1_MEMATTR_MASK            0xf
 202 #define CBAR_S1_MEMATTR_WB              0xf
 203 #define CBAR_TYPE_SHIFT                 16
 204 #define CBAR_TYPE_MASK                  0x3
 205 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 206 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 207 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 208 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 209 #define CBAR_IRPTNDX_SHIFT              24
 210 #define CBAR_IRPTNDX_MASK               0xff
 211
 212 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 213 #define CBA2R_RW64_32BIT                (0 << 0)
 214 #define CBA2R_RW64_64BIT                (1 << 0)
 215 #define CBA2R_VMID_SHIFT                16
 216 #define CBA2R_VMID_MASK                 0xffff
 217
 218 /* Translation context bank */
 219 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 220 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 221
 222 #define ARM_SMMU_CB_SCTLR               0x0
 223 #define ARM_SMMU_CB_ACTLR               0x4
 224 #define ARM_SMMU_CB_RESUME              0x8
 225 #define ARM_SMMU_CB_TTBCR2              0x10
 226 #define ARM_SMMU_CB_TTBR0               0x20
 227 #define ARM_SMMU_CB_TTBR1               0x28
 228 #define ARM_SMMU_CB_TTBCR               0x30
 229 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 230 #define ARM_SMMU_CB_S1_MAIR0            0x38
 231 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 232 #define ARM_SMMU_CB_PAR                 0x50
 233 #define ARM_SMMU_CB_FSR                 0x58
 234 #define ARM_SMMU_CB_FAR                 0x60
 235 #define ARM_SMMU_CB_FSYNR0              0x68
 236 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 237 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 238 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 239 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 240 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 241 #define ARM_SMMU_CB_ATS1PR              0x800
 242 #define ARM_SMMU_CB_ATSR                0x8f0
 243
 244 #define SCTLR_S1_ASIDPNE                (1 << 12)
 245 #define SCTLR_CFCFG                     (1 << 7)
 246 #define SCTLR_CFIE                      (1 << 6)
 247 #define SCTLR_CFRE                      (1 << 5)
 248 #define SCTLR_E                         (1 << 4)
 249 #define SCTLR_AFE                       (1 << 2)
 250 #define SCTLR_TRE                       (1 << 1)
 251 #define SCTLR_M                         (1 << 0)
 252
 253 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 254
 255 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 256 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 257
 258 #define CB_PAR_F                        (1 << 0)
 259
 260 #define ATSR_ACTIVE                     (1 << 0)
 261
 262 #define RESUME_RETRY                    (0 << 0)
 263 #define RESUME_TERMINATE                (1 << 0)
 264
 265 #define TTBCR2_SEP_SHIFT                15
 266 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 267 #define TTBCR2_AS                       (1 << 4)
 268
 269 #define TTBRn_ASID_SHIFT                48
 270
 271 #define FSR_MULTI                       (1 << 31)
 272 #define FSR_SS                          (1 << 30)
 273 #define FSR_UUT                         (1 << 8)
 274 #define FSR_ASF                         (1 << 7)
 275 #define FSR_TLBLKF                      (1 << 6)
 276 #define FSR_TLBMCF                      (1 << 5)
 277 #define FSR_EF                          (1 << 4)
 278 #define FSR_PF                          (1 << 3)
 279 #define FSR_AFF                         (1 << 2)
 280 #define FSR_TF                          (1 << 1)
 281
 282 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 283                                          FSR_TLBMCF | FSR_TLBLKF)
 284 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 285                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 286
 287 #define FSYNR0_WNR                      (1 << 4)
 288
 289 #define MSI_IOVA_BASE                   0x8000000
 290 #define MSI_IOVA_LENGTH                 0x100000
 291
 292 static int force_stage;
 293 module_param(force_stage, int, S_IRUGO);
 294 MODULE_PARM_DESC(force_stage,
 295         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 296 static bool disable_bypass;
 297 module_param(disable_bypass, bool, S_IRUGO);
 298 MODULE_PARM_DESC(disable_bypass,
 299         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 300
 301 enum arm_smmu_arch_version {
 302         ARM_SMMU_V1,
 303         ARM_SMMU_V1_64K,
 304         ARM_SMMU_V2,
 305 };
 306
 307 enum arm_smmu_implementation {
 308         GENERIC_SMMU,
 309         ARM_MMU500,
 310         CAVIUM_SMMUV2,
 311 };
 312
 313 struct arm_smmu_s2cr {
 314         struct iommu_group              *group;
 315         int                             count;
 316         enum arm_smmu_s2cr_type         type;
 317         enum arm_smmu_s2cr_privcfg      privcfg;
 318         u8                              cbndx;
 319 };
 320
 321 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 322         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 323 }
 324
 325 struct arm_smmu_smr {
 326         u16                             mask;
 327         u16                             id;
 328         bool                            valid;
 329 };
 330
 331 struct arm_smmu_master_cfg {
 332         struct arm_smmu_device          *smmu;
 333         s16                             smendx[];
 334 };
 335 #define INVALID_SMENDX                  -1
 336 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 337 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 338 #define fwspec_smendx(fw, i) \
 339         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 340 #define for_each_cfg_sme(fw, i, idx) \
 341         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 342
 343 struct arm_smmu_device {
 344         struct device                   *dev;
 345
 346         void __iomem                    *base;
 347         unsigned long                   size;
 348         unsigned long                   pgshift;
 349
 350 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 351 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 352 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 353 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 354 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 355 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 356 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 357 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 358 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 359 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 360 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 361 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 362 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 363         u32                             features;
 364
 365 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 366         u32                             options;
 367         enum arm_smmu_arch_version      version;
 368         enum arm_smmu_implementation    model;
 369
 370         u32                             num_context_banks;
 371         u32                             num_s2_context_banks;
 372         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 373         atomic_t                        irptndx;
 374
 375         u32                             num_mapping_groups;
 376         u16                             streamid_mask;
 377         u16                             smr_mask_mask;
 378         struct arm_smmu_smr             *smrs;
 379         struct arm_smmu_s2cr            *s2crs;
 380         struct mutex                    stream_map_mutex;
 381
 382         unsigned long                   va_size;
 383         unsigned long                   ipa_size;
 384         unsigned long                   pa_size;
 385         unsigned long                   pgsize_bitmap;
 386
 387         u32                             num_global_irqs;
 388         u32                             num_context_irqs;
 389         unsigned int                    *irqs;
 390
 391         u32                             cavium_id_base; /* Specific to Cavium */
 392
 393         /* IOMMU core code handle */
 394         struct iommu_device             iommu;
 395 };
 396
 397 enum arm_smmu_context_fmt {
 398         ARM_SMMU_CTX_FMT_NONE,
 399         ARM_SMMU_CTX_FMT_AARCH64,
 400         ARM_SMMU_CTX_FMT_AARCH32_L,
 401         ARM_SMMU_CTX_FMT_AARCH32_S,
 402 };
 403
 404 struct arm_smmu_cfg {
 405         u8                              cbndx;
 406         u8                              irptndx;
 407         u32                             cbar;
 408         enum arm_smmu_context_fmt       fmt;
 409 };
 410 #define INVALID_IRPTNDX                 0xff
 411
 412 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 413 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 414
 415 enum arm_smmu_domain_stage {
 416         ARM_SMMU_DOMAIN_S1 = 0,
 417         ARM_SMMU_DOMAIN_S2,
 418         ARM_SMMU_DOMAIN_NESTED,
 419 };
 420
 421 struct arm_smmu_domain {
 422         struct arm_smmu_device          *smmu;
 423         struct io_pgtable_ops           *pgtbl_ops;
 424         spinlock_t                      pgtbl_lock;
 425         struct arm_smmu_cfg             cfg;
 426         enum arm_smmu_domain_stage      stage;
 427         struct mutex                    init_mutex; /* Protects smmu pointer */
 428         struct iommu_domain             domain;
 429 };
 430
 431 struct arm_smmu_option_prop {
 432         u32 opt;
 433         const char *prop;
 434 };
 435
 436 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 437
 438 static bool using_legacy_binding, using_generic_binding;
 439
 440 static struct arm_smmu_option_prop arm_smmu_options[] = {
 441         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 442         { 0, NULL},
 443 };
 444
 445 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 446 {
 447         return container_of(dom, struct arm_smmu_domain, domain);
 448 }
 449
 450 static void parse_driver_options(struct arm_smmu_device *smmu)
 451 {
 452         int i = 0;
 453
 454         do {
 455                 if (of_property_read_bool(smmu->dev->of_node,
 456                                                 arm_smmu_options[i].prop)) {
 457                         smmu->options |= arm_smmu_options[i].opt;
 458                         dev_notice(smmu->dev, "option %s\n",
 459                                 arm_smmu_options[i].prop);
 460                 }
 461         } while (arm_smmu_options[++i].opt);
 462 }
 463
 464 static struct device_node *dev_get_dev_node(struct device *dev)
 465 {
 466         if (dev_is_pci(dev)) {
 467                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 468
 469                 while (!pci_is_root_bus(bus))
 470                         bus = bus->parent;
 471                 return of_node_get(bus->bridge->parent->of_node);
 472         }
 473
 474         return of_node_get(dev->of_node);
 475 }
 476
 477 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 478 {
 479         *((__be32 *)data) = cpu_to_be32(alias);
 480         return 0; /* Continue walking */
 481 }
 482
 483 static int __find_legacy_master_phandle(struct device *dev, void *data)
 484 {
 485         struct of_phandle_iterator *it = *(void **)data;
 486         struct device_node *np = it->node;
 487         int err;
 488
 489         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 490                             "#stream-id-cells", 0)
 491                 if (it->node == np) {
 492                         *(void **)data = dev;
 493                         return 1;
 494                 }
 495         it->node = np;
 496         return err == -ENOENT ? 0 : err;
 497 }
 498
 499 static struct platform_driver arm_smmu_driver;
 500 static struct iommu_ops arm_smmu_ops;
 501
 502 static int arm_smmu_register_legacy_master(struct device *dev,
 503                                            struct arm_smmu_device **smmu)
 504 {
 505         struct device *smmu_dev;
 506         struct device_node *np;
 507         struct of_phandle_iterator it;
 508         void *data = &it;
 509         u32 *sids;
 510         __be32 pci_sid;
 511         int err;
 512
 513         np = dev_get_dev_node(dev);
 514         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 515                 of_node_put(np);
 516                 return -ENODEV;
 517         }
 518
 519         it.node = np;
 520         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 521                                      __find_legacy_master_phandle);
 522         smmu_dev = data;
 523         of_node_put(np);
 524         if (err == 0)
 525                 return -ENODEV;
 526         if (err < 0)
 527                 return err;
 528
 529         if (dev_is_pci(dev)) {
 530                 /* "mmu-masters" assumes Stream ID == Requester ID */
 531                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 532                                        &pci_sid);
 533                 it.cur = &pci_sid;
 534                 it.cur_count = 1;
 535         }
 536
 537         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 538                                 &arm_smmu_ops);
 539         if (err)
 540                 return err;
 541
 542         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 543         if (!sids)
 544                 return -ENOMEM;
 545
 546         *smmu = dev_get_drvdata(smmu_dev);
 547         of_phandle_iterator_args(&it, sids, it.cur_count);
 548         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 549         kfree(sids);
 550         return err;
 551 }
 552
 553 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 554 {
 555         int idx;
 556
 557         do {
 558                 idx = find_next_zero_bit(map, end, start);
 559                 if (idx == end)
 560                         return -ENOSPC;
 561         } while (test_and_set_bit(idx, map));
 562
 563         return idx;
 564 }
 565
 566 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 567 {
 568         clear_bit(idx, map);
 569 }
 570
 571 /* Wait for any pending TLB invalidations to complete */
 572 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 573 {
 574         int count = 0;
 575         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 576
 577         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 578         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 579                & sTLBGSTATUS_GSACTIVE) {
 580                 cpu_relax();
 581                 if (++count == TLB_LOOP_TIMEOUT) {
 582                         dev_err_ratelimited(smmu->dev,
 583                         "TLB sync timed out -- SMMU may be deadlocked\n");
 584                         return;
 585                 }
 586                 udelay(1);
 587         }
 588 }
 589
 590 static void arm_smmu_tlb_sync(void *cookie)
 591 {
 592         struct arm_smmu_domain *smmu_domain = cookie;
 593         __arm_smmu_tlb_sync(smmu_domain->smmu);
 594 }
 595
 596 static void arm_smmu_tlb_inv_context(void *cookie)
 597 {
 598         struct arm_smmu_domain *smmu_domain = cookie;
 599         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 600         struct arm_smmu_device *smmu = smmu_domain->smmu;
 601         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 602         void __iomem *base;
 603
 604         if (stage1) {
 605                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 606                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 607                                base + ARM_SMMU_CB_S1_TLBIASID);
 608         } else {
 609                 base = ARM_SMMU_GR0(smmu);
 610                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 611                                base + ARM_SMMU_GR0_TLBIVMID);
 612         }
 613
 614         __arm_smmu_tlb_sync(smmu);
 615 }
 616
 617 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 618                                           size_t granule, bool leaf, void *cookie)
 619 {
 620         struct arm_smmu_domain *smmu_domain = cookie;
 621         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 622         struct arm_smmu_device *smmu = smmu_domain->smmu;
 623         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 624         void __iomem *reg;
 625
 626         if (stage1) {
 627                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 628                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 629
 630                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 631                         iova &= ~12UL;
 632                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 633                         do {
 634                                 writel_relaxed(iova, reg);
 635                                 iova += granule;
 636                         } while (size -= granule);
 637                 } else {
 638                         iova >>= 12;
 639                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 640                         do {
 641                                 writeq_relaxed(iova, reg);
 642                                 iova += granule >> 12;
 643                         } while (size -= granule);
 644                 }
 645         } else if (smmu->version == ARM_SMMU_V2) {
 646                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 647                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 648                               ARM_SMMU_CB_S2_TLBIIPAS2;
 649                 iova >>= 12;
 650                 do {
 651                         smmu_write_atomic_lq(iova, reg);
 652                         iova += granule >> 12;
 653                 } while (size -= granule);
 654         } else {
 655                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 656                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 657         }
 658 }
 659
 660 static const struct iommu_gather_ops arm_smmu_gather_ops = {
 661         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 662         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 663         .tlb_sync       = arm_smmu_tlb_sync,
 664 };
 665
 666 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 667 {
 668         u32 fsr, fsynr;
 669         unsigned long iova;
 670         struct iommu_domain *domain = dev;
 671         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 672         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 673         struct arm_smmu_device *smmu = smmu_domain->smmu;
 674         void __iomem *cb_base;
 675
 676         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 677         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 678
 679         if (!(fsr & FSR_FAULT))
 680                 return IRQ_NONE;
 681
 682         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 683         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 684
 685         dev_err_ratelimited(smmu->dev,
 686         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 687                             fsr, iova, fsynr, cfg->cbndx);
 688
 689         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 690         return IRQ_HANDLED;
 691 }
 692
 693 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 694 {
 695         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 696         struct arm_smmu_device *smmu = dev;
 697         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 698
 699         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 700         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 701         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 702         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 703
 704         if (!gfsr)
 705                 return IRQ_NONE;
 706
 707         dev_err_ratelimited(smmu->dev,
 708                 "Unexpected global fault, this could be serious\n");
 709         dev_err_ratelimited(smmu->dev,
 710                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 711                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 712
 713         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 714         return IRQ_HANDLED;
 715 }
 716
 717 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 718                                        struct io_pgtable_cfg *pgtbl_cfg)
 719 {
 720         u32 reg, reg2;
 721         u64 reg64;
 722         bool stage1;
 723         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 724         struct arm_smmu_device *smmu = smmu_domain->smmu;
 725         void __iomem *cb_base, *gr1_base;
 726
 727         gr1_base = ARM_SMMU_GR1(smmu);
 728         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 729         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 730
 731         if (smmu->version > ARM_SMMU_V1) {
 732                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 733                         reg = CBA2R_RW64_64BIT;
 734                 else
 735                         reg = CBA2R_RW64_32BIT;
 736                 /* 16-bit VMIDs live in CBA2R */
 737                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 738                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 739
 740                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 741         }
 742
 743         /* CBAR */
 744         reg = cfg->cbar;
 745         if (smmu->version < ARM_SMMU_V2)
 746                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 747
 748         /*
 749          * Use the weakest shareability/memory types, so they are
 750          * overridden by the ttbcr/pte.
 751          */
 752         if (stage1) {
 753                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 754                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 755         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 756                 /* 8-bit VMIDs live in CBAR */
 757                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 758         }
 759         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 760
 761         /* TTBRs */
 762         if (stage1) {
 763                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 764
 765                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 766                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 767                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 768                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 769                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 770                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 771                 } else {
 772                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 773                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 774                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 775                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 776                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 777                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 778                 }
 779         } else {
 780                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 781                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 782         }
 783
 784         /* TTBCR */
 785         if (stage1) {
 786                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 787                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 788                         reg2 = 0;
 789                 } else {
 790                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 791                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 792                         reg2 |= TTBCR2_SEP_UPSTREAM;
 793                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 794                                 reg2 |= TTBCR2_AS;
 795                 }
 796                 if (smmu->version > ARM_SMMU_V1)
 797                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 798         } else {
 799                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 800         }
 801         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 802
 803         /* MAIRs (stage-1 only) */
 804         if (stage1) {
 805                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 806                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 807                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 808                 } else {
 809                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 810                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 811                 }
 812                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 813                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 814         }
 815
 816         /* SCTLR */
 817         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 818         if (stage1)
 819                 reg |= SCTLR_S1_ASIDPNE;
 820 #ifdef __BIG_ENDIAN
 821         reg |= SCTLR_E;
 822 #endif
 823         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 824 }
 825
 826 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 827                                         struct arm_smmu_device *smmu)
 828 {
 829         int irq, start, ret = 0;
 830         unsigned long ias, oas;
 831         struct io_pgtable_ops *pgtbl_ops;
 832         struct io_pgtable_cfg pgtbl_cfg;
 833         enum io_pgtable_fmt fmt;
 834         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 835         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 836
 837         mutex_lock(&smmu_domain->init_mutex);
 838         if (smmu_domain->smmu)
 839                 goto out_unlock;
 840
 841         /*
 842          * Mapping the requested stage onto what we support is surprisingly
 843          * complicated, mainly because the spec allows S1+S2 SMMUs without
 844          * support for nested translation. That means we end up with the
 845          * following table:
 846          *
 847          * Requested        Supported        Actual
 848          *     S1               N              S1
 849          *     S1             S1+S2            S1
 850          *     S1               S2             S2
 851          *     S1               S1             S1
 852          *     N                N              N
 853          *     N              S1+S2            S2
 854          *     N                S2             S2
 855          *     N                S1             S1
 856          *
 857          * Note that you can't actually request stage-2 mappings.
 858          */
 859         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 860                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 861         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 862                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 863
 864         /*
 865          * Choosing a suitable context format is even more fiddly. Until we
 866          * grow some way for the caller to express a preference, and/or move
 867          * the decision into the io-pgtable code where it arguably belongs,
 868          * just aim for the closest thing to the rest of the system, and hope
 869          * that the hardware isn't esoteric enough that we can't assume AArch64
 870          * support to be a superset of AArch32 support...
 871          */
 872         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 873                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 874         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 875             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 876             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 877             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 878                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 879         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 880             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 881                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 882                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 883                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 884
 885         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 886                 ret = -EINVAL;
 887                 goto out_unlock;
 888         }
 889
 890         switch (smmu_domain->stage) {
 891         case ARM_SMMU_DOMAIN_S1:
 892                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 893                 start = smmu->num_s2_context_banks;
 894                 ias = smmu->va_size;
 895                 oas = smmu->ipa_size;
 896                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 897                         fmt = ARM_64_LPAE_S1;
 898                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 899                         fmt = ARM_32_LPAE_S1;
 900                         ias = min(ias, 32UL);
 901                         oas = min(oas, 40UL);
 902                 } else {
 903                         fmt = ARM_V7S;
 904                         ias = min(ias, 32UL);
 905                         oas = min(oas, 32UL);
 906                 }
 907                 break;
 908         case ARM_SMMU_DOMAIN_NESTED:
 909                 /*
 910                  * We will likely want to change this if/when KVM gets
 911                  * involved.
 912                  */
 913         case ARM_SMMU_DOMAIN_S2:
 914                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 915                 start = 0;
 916                 ias = smmu->ipa_size;
 917                 oas = smmu->pa_size;
 918                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 919                         fmt = ARM_64_LPAE_S2;
 920                 } else {
 921                         fmt = ARM_32_LPAE_S2;
 922                         ias = min(ias, 40UL);
 923                         oas = min(oas, 40UL);
 924                 }
 925                 break;
 926         default:
 927                 ret = -EINVAL;
 928                 goto out_unlock;
 929         }
 930
 931         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 932                                       smmu->num_context_banks);
 933         if (ret < 0)
 934                 goto out_unlock;
 935
 936         cfg->cbndx = ret;
 937         if (smmu->version < ARM_SMMU_V2) {
 938                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 939                 cfg->irptndx %= smmu->num_context_irqs;
 940         } else {
 941                 cfg->irptndx = cfg->cbndx;
 942         }
 943
 944         pgtbl_cfg = (struct io_pgtable_cfg) {
 945                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 946                 .ias            = ias,
 947                 .oas            = oas,
 948                 .tlb            = &arm_smmu_gather_ops,
 949                 .iommu_dev      = smmu->dev,
 950         };
 951
 952         smmu_domain->smmu = smmu;
 953         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 954         if (!pgtbl_ops) {
 955                 ret = -ENOMEM;
 956                 goto out_clear_smmu;
 957         }
 958
 959         /* Update the domain's page sizes to reflect the page table format */
 960         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 961         domain->geometry.aperture_end = (1UL << ias) - 1;
 962         domain->geometry.force_aperture = true;
 963
 964         /* Initialise the context bank with our page table cfg */
 965         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 966
 967         /*
 968          * Request context fault interrupt. Do this last to avoid the
 969          * handler seeing a half-initialised domain state.
 970          */
 971         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 972         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 973                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 974         if (ret < 0) {
 975                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 976                         cfg->irptndx, irq);
 977                 cfg->irptndx = INVALID_IRPTNDX;
 978         }
 979
 980         mutex_unlock(&smmu_domain->init_mutex);
 981
 982         /* Publish page table ops for map/unmap */
 983         smmu_domain->pgtbl_ops = pgtbl_ops;
 984         return 0;
 985
 986 out_clear_smmu:
 987         smmu_domain->smmu = NULL;
 988 out_unlock:
 989         mutex_unlock(&smmu_domain->init_mutex);
 990         return ret;
 991 }
 992
 993 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 994 {
 995         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 996         struct arm_smmu_device *smmu = smmu_domain->smmu;
 997         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 998         void __iomem *cb_base;
 999         int irq;
1000
1001         if (!smmu)
1002                 return;
1003
1004         /*
1005          * Disable the context bank and free the page tables before freeing
1006          * it.
1007          */
1008         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1009         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1010
1011         if (cfg->irptndx != INVALID_IRPTNDX) {
1012                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1013                 devm_free_irq(smmu->dev, irq, domain);
1014         }
1015
1016         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1017         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1018 }
1019
1020 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1021 {
1022         struct arm_smmu_domain *smmu_domain;
1023
1024         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1025                 return NULL;
1026         /*
1027          * Allocate the domain and initialise some of its data structures.
1028          * We can't really do anything meaningful until we've added a
1029          * master.
1030          */
1031         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1032         if (!smmu_domain)
1033                 return NULL;
1034
1035         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1036             iommu_get_dma_cookie(&smmu_domain->domain))) {
1037                 kfree(smmu_domain);
1038                 return NULL;
1039         }
1040
1041         mutex_init(&smmu_domain->init_mutex);
1042         spin_lock_init(&smmu_domain->pgtbl_lock);
1043
1044         return &smmu_domain->domain;
1045 }
1046
1047 static void arm_smmu_domain_free(struct iommu_domain *domain)
1048 {
1049         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1050
1051         /*
1052          * Free the domain resources. We assume that all devices have
1053          * already been detached.
1054          */
1055         iommu_put_dma_cookie(domain);
1056         arm_smmu_destroy_domain_context(domain);
1057         kfree(smmu_domain);
1058 }
1059
1060 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1061 {
1062         struct arm_smmu_smr *smr = smmu->smrs + idx;
1063         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1064
1065         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1066                 reg |= SMR_VALID;
1067         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1068 }
1069
1070 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1071 {
1072         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1073         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1074                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1075                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1076
1077         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1078             smmu->smrs[idx].valid)
1079                 reg |= S2CR_EXIDVALID;
1080         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1081 }
1082
1083 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1084 {
1085         arm_smmu_write_s2cr(smmu, idx);
1086         if (smmu->smrs)
1087                 arm_smmu_write_smr(smmu, idx);
1088 }
1089
1090 /*
1091  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1092  * should be called after sCR0 is written.
1093  */
1094 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1095 {
1096         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1097         u32 smr;
1098
1099         if (!smmu->smrs)
1100                 return;
1101
1102         /*
1103          * SMR.ID bits may not be preserved if the corresponding MASK
1104          * bits are set, so check each one separately. We can reject
1105          * masters later if they try to claim IDs outside these masks.
1106          */
1107         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1108         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1109         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1110         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1111
1112         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1113         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1114         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1115         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1116 }
1117
1118 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1119 {
1120         struct arm_smmu_smr *smrs = smmu->smrs;
1121         int i, free_idx = -ENOSPC;
1122
1123         /* Stream indexing is blissfully easy */
1124         if (!smrs)
1125                 return id;
1126
1127         /* Validating SMRs is... less so */
1128         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1129                 if (!smrs[i].valid) {
1130                         /*
1131                          * Note the first free entry we come across, which
1132                          * we'll claim in the end if nothing else matches.
1133                          */
1134                         if (free_idx < 0)
1135                                 free_idx = i;
1136                         continue;
1137                 }
1138                 /*
1139                  * If the new entry is _entirely_ matched by an existing entry,
1140                  * then reuse that, with the guarantee that there also cannot
1141                  * be any subsequent conflicting entries. In normal use we'd
1142                  * expect simply identical entries for this case, but there's
1143                  * no harm in accommodating the generalisation.
1144                  */
1145                 if ((mask & smrs[i].mask) == mask &&
1146                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1147                         return i;
1148                 /*
1149                  * If the new entry has any other overlap with an existing one,
1150                  * though, then there always exists at least one stream ID
1151                  * which would cause a conflict, and we can't allow that risk.
1152                  */
1153                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1154                         return -EINVAL;
1155         }
1156
1157         return free_idx;
1158 }
1159
1160 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1161 {
1162         if (--smmu->s2crs[idx].count)
1163                 return false;
1164
1165         smmu->s2crs[idx] = s2cr_init_val;
1166         if (smmu->smrs)
1167                 smmu->smrs[idx].valid = false;
1168
1169         return true;
1170 }
1171
1172 static int arm_smmu_master_alloc_smes(struct device *dev)
1173 {
1174         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1175         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1176         struct arm_smmu_device *smmu = cfg->smmu;
1177         struct arm_smmu_smr *smrs = smmu->smrs;
1178         struct iommu_group *group;
1179         int i, idx, ret;
1180
1181         mutex_lock(&smmu->stream_map_mutex);
1182         /* Figure out a viable stream map entry allocation */
1183         for_each_cfg_sme(fwspec, i, idx) {
1184                 u16 sid = fwspec->ids[i];
1185                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1186
1187                 if (idx != INVALID_SMENDX) {
1188                         ret = -EEXIST;
1189                         goto out_err;
1190                 }
1191
1192                 ret = arm_smmu_find_sme(smmu, sid, mask);
1193                 if (ret < 0)
1194                         goto out_err;
1195
1196                 idx = ret;
1197                 if (smrs && smmu->s2crs[idx].count == 0) {
1198                         smrs[idx].id = sid;
1199                         smrs[idx].mask = mask;
1200                         smrs[idx].valid = true;
1201                 }
1202                 smmu->s2crs[idx].count++;
1203                 cfg->smendx[i] = (s16)idx;
1204         }
1205
1206         group = iommu_group_get_for_dev(dev);
1207         if (!group)
1208                 group = ERR_PTR(-ENOMEM);
1209         if (IS_ERR(group)) {
1210                 ret = PTR_ERR(group);
1211                 goto out_err;
1212         }
1213         iommu_group_put(group);
1214
1215         /* It worked! Now, poke the actual hardware */
1216         for_each_cfg_sme(fwspec, i, idx) {
1217                 arm_smmu_write_sme(smmu, idx);
1218                 smmu->s2crs[idx].group = group;
1219         }
1220
1221         mutex_unlock(&smmu->stream_map_mutex);
1222         return 0;
1223
1224 out_err:
1225         while (i--) {
1226                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1227                 cfg->smendx[i] = INVALID_SMENDX;
1228         }
1229         mutex_unlock(&smmu->stream_map_mutex);
1230         return ret;
1231 }
1232
1233 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1234 {
1235         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1236         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1237         int i, idx;
1238
1239         mutex_lock(&smmu->stream_map_mutex);
1240         for_each_cfg_sme(fwspec, i, idx) {
1241                 if (arm_smmu_free_sme(smmu, idx))
1242                         arm_smmu_write_sme(smmu, idx);
1243                 cfg->smendx[i] = INVALID_SMENDX;
1244         }
1245         mutex_unlock(&smmu->stream_map_mutex);
1246 }
1247
1248 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1249                                       struct iommu_fwspec *fwspec)
1250 {
1251         struct arm_smmu_device *smmu = smmu_domain->smmu;
1252         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1253         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1254         u8 cbndx = smmu_domain->cfg.cbndx;
1255         int i, idx;
1256
1257         for_each_cfg_sme(fwspec, i, idx) {
1258                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1259                         continue;
1260
1261                 s2cr[idx].type = type;
1262                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1263                 s2cr[idx].cbndx = cbndx;
1264                 arm_smmu_write_s2cr(smmu, idx);
1265         }
1266         return 0;
1267 }
1268
1269 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1270 {
1271         int ret;
1272         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1273         struct arm_smmu_device *smmu;
1274         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1275
1276         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1277                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1278                 return -ENXIO;
1279         }
1280
1281         /*
1282          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1283          * domains between of_xlate() and add_device() - we have no way to cope
1284          * with that, so until ARM gets converted to rely on groups and default
1285          * domains, just say no (but more politely than by dereferencing NULL).
1286          * This should be at least a WARN_ON once that's sorted.
1287          */
1288         if (!fwspec->iommu_priv)
1289                 return -ENODEV;
1290
1291         smmu = fwspec_smmu(fwspec);
1292         /* Ensure that the domain is finalised */
1293         ret = arm_smmu_init_domain_context(domain, smmu);
1294         if (ret < 0)
1295                 return ret;
1296
1297         /*
1298          * Sanity check the domain. We don't support domains across
1299          * different SMMUs.
1300          */
1301         if (smmu_domain->smmu != smmu) {
1302                 dev_err(dev,
1303                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1304                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1305                 return -EINVAL;
1306         }
1307
1308         /* Looks ok, so add the device to the domain */
1309         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1310 }
1311
1312 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1313                         phys_addr_t paddr, size_t size, int prot)
1314 {
1315         int ret;
1316         unsigned long flags;
1317         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1318         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1319
1320         if (!ops)
1321                 return -ENODEV;
1322
1323         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1324         ret = ops->map(ops, iova, paddr, size, prot);
1325         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1326         return ret;
1327 }
1328
1329 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1330                              size_t size)
1331 {
1332         size_t ret;
1333         unsigned long flags;
1334         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1335         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1336
1337         if (!ops)
1338                 return 0;
1339
1340         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1341         ret = ops->unmap(ops, iova, size);
1342         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1343         return ret;
1344 }
1345
1346 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1347                                               dma_addr_t iova)
1348 {
1349         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1350         struct arm_smmu_device *smmu = smmu_domain->smmu;
1351         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1352         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1353         struct device *dev = smmu->dev;
1354         void __iomem *cb_base;
1355         u32 tmp;
1356         u64 phys;
1357         unsigned long va;
1358
1359         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1360
1361         /* ATS1 registers can only be written atomically */
1362         va = iova & ~0xfffUL;
1363         if (smmu->version == ARM_SMMU_V2)
1364                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1365         else /* Register is only 32-bit in v1 */
1366                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1367
1368         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1369                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1370                 dev_err(dev,
1371                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1372                         &iova);
1373                 return ops->iova_to_phys(ops, iova);
1374         }
1375
1376         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1377         if (phys & CB_PAR_F) {
1378                 dev_err(dev, "translation fault!\n");
1379                 dev_err(dev, "PAR = 0x%llx\n", phys);
1380                 return 0;
1381         }
1382
1383         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1384 }
1385
1386 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1387                                         dma_addr_t iova)
1388 {
1389         phys_addr_t ret;
1390         unsigned long flags;
1391         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1392         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1393
1394         if (!ops)
1395                 return 0;
1396
1397         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1398         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1399                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1400                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1401         } else {
1402                 ret = ops->iova_to_phys(ops, iova);
1403         }
1404
1405         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1406
1407         return ret;
1408 }
1409
1410 static bool arm_smmu_capable(enum iommu_cap cap)
1411 {
1412         switch (cap) {
1413         case IOMMU_CAP_CACHE_COHERENCY:
1414                 /*
1415                  * Return true here as the SMMU can always send out coherent
1416                  * requests.
1417                  */
1418                 return true;
1419         case IOMMU_CAP_NOEXEC:
1420                 return true;
1421         default:
1422                 return false;
1423         }
1424 }
1425
1426 static int arm_smmu_match_node(struct device *dev, void *data)
1427 {
1428         return dev->fwnode == data;
1429 }
1430
1431 static
1432 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1433 {
1434         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1435                                                 fwnode, arm_smmu_match_node);
1436         put_device(dev);
1437         return dev ? dev_get_drvdata(dev) : NULL;
1438 }
1439
1440 static int arm_smmu_add_device(struct device *dev)
1441 {
1442         struct arm_smmu_device *smmu;
1443         struct arm_smmu_master_cfg *cfg;
1444         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1445         int i, ret;
1446
1447         if (using_legacy_binding) {
1448                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1449                 fwspec = dev->iommu_fwspec;
1450                 if (ret)
1451                         goto out_free;
1452         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1453                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1454         } else {
1455                 return -ENODEV;
1456         }
1457
1458         ret = -EINVAL;
1459         for (i = 0; i < fwspec->num_ids; i++) {
1460                 u16 sid = fwspec->ids[i];
1461                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1462
1463                 if (sid & ~smmu->streamid_mask) {
1464                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1465                                 sid, smmu->streamid_mask);
1466                         goto out_free;
1467                 }
1468                 if (mask & ~smmu->smr_mask_mask) {
1469                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1470                                 sid, smmu->smr_mask_mask);
1471                         goto out_free;
1472                 }
1473         }
1474
1475         ret = -ENOMEM;
1476         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1477                       GFP_KERNEL);
1478         if (!cfg)
1479                 goto out_free;
1480
1481         cfg->smmu = smmu;
1482         fwspec->iommu_priv = cfg;
1483         while (i--)
1484                 cfg->smendx[i] = INVALID_SMENDX;
1485
1486         ret = arm_smmu_master_alloc_smes(dev);
1487         if (ret)
1488                 goto out_free;
1489
1490         iommu_device_link(&smmu->iommu, dev);
1491
1492         return 0;
1493
1494 out_free:
1495         if (fwspec)
1496                 kfree(fwspec->iommu_priv);
1497         iommu_fwspec_free(dev);
1498         return ret;
1499 }
1500
1501 static void arm_smmu_remove_device(struct device *dev)
1502 {
1503         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1504         struct arm_smmu_master_cfg *cfg;
1505         struct arm_smmu_device *smmu;
1506
1507
1508         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1509                 return;
1510
1511         cfg  = fwspec->iommu_priv;
1512         smmu = cfg->smmu;
1513
1514         iommu_device_unlink(&smmu->iommu, dev);
1515         arm_smmu_master_free_smes(fwspec);
1516         iommu_group_remove_device(dev);
1517         kfree(fwspec->iommu_priv);
1518         iommu_fwspec_free(dev);
1519 }
1520
1521 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1522 {
1523         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1524         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1525         struct iommu_group *group = NULL;
1526         int i, idx;
1527
1528         for_each_cfg_sme(fwspec, i, idx) {
1529                 if (group && smmu->s2crs[idx].group &&
1530                     group != smmu->s2crs[idx].group)
1531                         return ERR_PTR(-EINVAL);
1532
1533                 group = smmu->s2crs[idx].group;
1534         }
1535
1536         if (group)
1537                 return iommu_group_ref_get(group);
1538
1539         if (dev_is_pci(dev))
1540                 group = pci_device_group(dev);
1541         else
1542                 group = generic_device_group(dev);
1543
1544         return group;
1545 }
1546
1547 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1548                                     enum iommu_attr attr, void *data)
1549 {
1550         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1551
1552         switch (attr) {
1553         case DOMAIN_ATTR_NESTING:
1554                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1555                 return 0;
1556         default:
1557                 return -ENODEV;
1558         }
1559 }
1560
1561 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1562                                     enum iommu_attr attr, void *data)
1563 {
1564         int ret = 0;
1565         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1566
1567         mutex_lock(&smmu_domain->init_mutex);
1568
1569         switch (attr) {
1570         case DOMAIN_ATTR_NESTING:
1571                 if (smmu_domain->smmu) {
1572                         ret = -EPERM;
1573                         goto out_unlock;
1574                 }
1575
1576                 if (*(int *)data)
1577                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1578                 else
1579                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1580
1581                 break;
1582         default:
1583                 ret = -ENODEV;
1584         }
1585
1586 out_unlock:
1587         mutex_unlock(&smmu_domain->init_mutex);
1588         return ret;
1589 }
1590
1591 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1592 {
1593         u32 fwid = 0;
1594
1595         if (args->args_count > 0)
1596                 fwid |= (u16)args->args[0];
1597
1598         if (args->args_count > 1)
1599                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1600
1601         return iommu_fwspec_add_ids(dev, &fwid, 1);
1602 }
1603
1604 static void arm_smmu_get_resv_regions(struct device *dev,
1605                                       struct list_head *head)
1606 {
1607         struct iommu_resv_region *region;
1608         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1609
1610         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1611                                          prot, IOMMU_RESV_SW_MSI);
1612         if (!region)
1613                 return;
1614
1615         list_add_tail(&region->list, head);
1616 }
1617
1618 static void arm_smmu_put_resv_regions(struct device *dev,
1619                                       struct list_head *head)
1620 {
1621         struct iommu_resv_region *entry, *next;
1622
1623         list_for_each_entry_safe(entry, next, head, list)
1624                 kfree(entry);
1625 }
1626
1627 static struct iommu_ops arm_smmu_ops = {
1628         .capable                = arm_smmu_capable,
1629         .domain_alloc           = arm_smmu_domain_alloc,
1630         .domain_free            = arm_smmu_domain_free,
1631         .attach_dev             = arm_smmu_attach_dev,
1632         .map                    = arm_smmu_map,
1633         .unmap                  = arm_smmu_unmap,
1634         .map_sg                 = default_iommu_map_sg,
1635         .iova_to_phys           = arm_smmu_iova_to_phys,
1636         .add_device             = arm_smmu_add_device,
1637         .remove_device          = arm_smmu_remove_device,
1638         .device_group           = arm_smmu_device_group,
1639         .domain_get_attr        = arm_smmu_domain_get_attr,
1640         .domain_set_attr        = arm_smmu_domain_set_attr,
1641         .of_xlate               = arm_smmu_of_xlate,
1642         .get_resv_regions       = arm_smmu_get_resv_regions,
1643         .put_resv_regions       = arm_smmu_put_resv_regions,
1644         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1645 };
1646
1647 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1648 {
1649         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1650         void __iomem *cb_base;
1651         int i;
1652         u32 reg, major;
1653
1654         /* clear global FSR */
1655         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1656         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1657
1658         /*
1659          * Reset stream mapping groups: Initial values mark all SMRn as
1660          * invalid and all S2CRn as bypass unless overridden.
1661          */
1662         for (i = 0; i < smmu->num_mapping_groups; ++i)
1663                 arm_smmu_write_sme(smmu, i);
1664
1665         if (smmu->model == ARM_MMU500) {
1666                 /*
1667                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1668                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1669                  * bit is only present in MMU-500r2 onwards.
1670                  */
1671                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1672                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1673                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1674                 if (major >= 2)
1675                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1676                 /*
1677                  * Allow unmatched Stream IDs to allocate bypass
1678                  * TLB entries for reduced latency.
1679                  */
1680                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1681                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1682         }
1683
1684         /* Make sure all context banks are disabled and clear CB_FSR  */
1685         for (i = 0; i < smmu->num_context_banks; ++i) {
1686                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1687                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1688                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1689                 /*
1690                  * Disable MMU-500's not-particularly-beneficial next-page
1691                  * prefetcher for the sake of errata #841119 and #826419.
1692                  */
1693                 if (smmu->model == ARM_MMU500) {
1694                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1695                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1696                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1697                 }
1698         }
1699
1700         /* Invalidate the TLB, just in case */
1701         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1702         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1703
1704         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1705
1706         /* Enable fault reporting */
1707         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1708
1709         /* Disable TLB broadcasting. */
1710         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1711
1712         /* Enable client access, handling unmatched streams as appropriate */
1713         reg &= ~sCR0_CLIENTPD;
1714         if (disable_bypass)
1715                 reg |= sCR0_USFCFG;
1716         else
1717                 reg &= ~sCR0_USFCFG;
1718
1719         /* Disable forced broadcasting */
1720         reg &= ~sCR0_FB;
1721
1722         /* Don't upgrade barriers */
1723         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1724
1725         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1726                 reg |= sCR0_VMID16EN;
1727
1728         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1729                 reg |= sCR0_EXIDENABLE;
1730
1731         /* Push the button */
1732         __arm_smmu_tlb_sync(smmu);
1733         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1734 }
1735
1736 static int arm_smmu_id_size_to_bits(int size)
1737 {
1738         switch (size) {
1739         case 0:
1740                 return 32;
1741         case 1:
1742                 return 36;
1743         case 2:
1744                 return 40;
1745         case 3:
1746                 return 42;
1747         case 4:
1748                 return 44;
1749         case 5:
1750         default:
1751                 return 48;
1752         }
1753 }
1754
1755 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1756 {
1757         unsigned long size;
1758         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1759         u32 id;
1760         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1761         int i;
1762
1763         dev_notice(smmu->dev, "probing hardware configuration...\n");
1764         dev_notice(smmu->dev, "SMMUv%d with:\n",
1765                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1766
1767         /* ID0 */
1768         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1769
1770         /* Restrict available stages based on module parameter */
1771         if (force_stage == 1)
1772                 id &= ~(ID0_S2TS | ID0_NTS);
1773         else if (force_stage == 2)
1774                 id &= ~(ID0_S1TS | ID0_NTS);
1775
1776         if (id & ID0_S1TS) {
1777                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1778                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1779         }
1780
1781         if (id & ID0_S2TS) {
1782                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1783                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1784         }
1785
1786         if (id & ID0_NTS) {
1787                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1788                 dev_notice(smmu->dev, "\tnested translation\n");
1789         }
1790
1791         if (!(smmu->features &
1792                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1793                 dev_err(smmu->dev, "\tno translation support!\n");
1794                 return -ENODEV;
1795         }
1796
1797         if ((id & ID0_S1TS) &&
1798                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1799                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1800                 dev_notice(smmu->dev, "\taddress translation ops\n");
1801         }
1802
1803         /*
1804          * In order for DMA API calls to work properly, we must defer to what
1805          * the FW says about coherency, regardless of what the hardware claims.
1806          * Fortunately, this also opens up a workaround for systems where the
1807          * ID register value has ended up configured incorrectly.
1808          */
1809         cttw_reg = !!(id & ID0_CTTW);
1810         if (cttw_fw || cttw_reg)
1811                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1812                            cttw_fw ? "" : "non-");
1813         if (cttw_fw != cttw_reg)
1814                 dev_notice(smmu->dev,
1815                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1816
1817         /* Max. number of entries we have for stream matching/indexing */
1818         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1819                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1820                 size = 1 << 16;
1821         } else {
1822                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1823         }
1824         smmu->streamid_mask = size - 1;
1825         if (id & ID0_SMS) {
1826                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1827                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1828                 if (size == 0) {
1829                         dev_err(smmu->dev,
1830                                 "stream-matching supported, but no SMRs present!\n");
1831                         return -ENODEV;
1832                 }
1833
1834                 /* Zero-initialised to mark as invalid */
1835                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1836                                           GFP_KERNEL);
1837                 if (!smmu->smrs)
1838                         return -ENOMEM;
1839
1840                 dev_notice(smmu->dev,
1841                            "\tstream matching with %lu register groups", size);
1842         }
1843         /* s2cr->type == 0 means translation, so initialise explicitly */
1844         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1845                                          GFP_KERNEL);
1846         if (!smmu->s2crs)
1847                 return -ENOMEM;
1848         for (i = 0; i < size; i++)
1849                 smmu->s2crs[i] = s2cr_init_val;
1850
1851         smmu->num_mapping_groups = size;
1852         mutex_init(&smmu->stream_map_mutex);
1853
1854         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1855                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1856                 if (!(id & ID0_PTFS_NO_AARCH32S))
1857                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1858         }
1859
1860         /* ID1 */
1861         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1862         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1863
1864         /* Check for size mismatch of SMMU address space from mapped region */
1865         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1866         size *= 2 << smmu->pgshift;
1867         if (smmu->size != size)
1868                 dev_warn(smmu->dev,
1869                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1870                         size, smmu->size);
1871
1872         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1873         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1874         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1875                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1876                 return -ENODEV;
1877         }
1878         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1879                    smmu->num_context_banks, smmu->num_s2_context_banks);
1880         /*
1881          * Cavium CN88xx erratum #27704.
1882          * Ensure ASID and VMID allocation is unique across all SMMUs in
1883          * the system.
1884          */
1885         if (smmu->model == CAVIUM_SMMUV2) {
1886                 smmu->cavium_id_base =
1887                         atomic_add_return(smmu->num_context_banks,
1888                                           &cavium_smmu_context_count);
1889                 smmu->cavium_id_base -= smmu->num_context_banks;
1890         }
1891
1892         /* ID2 */
1893         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1894         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1895         smmu->ipa_size = size;
1896
1897         /* The output mask is also applied for bypass */
1898         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1899         smmu->pa_size = size;
1900
1901         if (id & ID2_VMID16)
1902                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1903
1904         /*
1905          * What the page table walker can address actually depends on which
1906          * descriptor format is in use, but since a) we don't know that yet,
1907          * and b) it can vary per context bank, this will have to do...
1908          */
1909         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1910                 dev_warn(smmu->dev,
1911                          "failed to set DMA mask for table walker\n");
1912
1913         if (smmu->version < ARM_SMMU_V2) {
1914                 smmu->va_size = smmu->ipa_size;
1915                 if (smmu->version == ARM_SMMU_V1_64K)
1916                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1917         } else {
1918                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1919                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1920                 if (id & ID2_PTFS_4K)
1921                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1922                 if (id & ID2_PTFS_16K)
1923                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1924                 if (id & ID2_PTFS_64K)
1925                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1926         }
1927
1928         /* Now we've corralled the various formats, what'll it do? */
1929         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1930                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1931         if (smmu->features &
1932             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1933                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1934         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1935                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1936         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1937                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1938
1939         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1940                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1941         else
1942                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1943         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1944                    smmu->pgsize_bitmap);
1945
1946
1947         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1948                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1949                            smmu->va_size, smmu->ipa_size);
1950
1951         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1952                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1953                            smmu->ipa_size, smmu->pa_size);
1954
1955         return 0;
1956 }
1957
1958 struct arm_smmu_match_data {
1959         enum arm_smmu_arch_version version;
1960         enum arm_smmu_implementation model;
1961 };
1962
1963 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1964 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1965
1966 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1967 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1968 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1969 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1970 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1971
1972 static const struct of_device_id arm_smmu_of_match[] = {
1973         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1974         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1975         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1976         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1977         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1978         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1979         { },
1980 };
1981 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1982
1983 #ifdef CONFIG_ACPI
1984 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1985 {
1986         int ret = 0;
1987
1988         switch (model) {
1989         case ACPI_IORT_SMMU_V1:
1990         case ACPI_IORT_SMMU_CORELINK_MMU400:
1991                 smmu->version = ARM_SMMU_V1;
1992                 smmu->model = GENERIC_SMMU;
1993                 break;
1994         case ACPI_IORT_SMMU_V2:
1995                 smmu->version = ARM_SMMU_V2;
1996                 smmu->model = GENERIC_SMMU;
1997                 break;
1998         case ACPI_IORT_SMMU_CORELINK_MMU500:
1999                 smmu->version = ARM_SMMU_V2;
2000                 smmu->model = ARM_MMU500;
2001                 break;
2002         default:
2003                 ret = -ENODEV;
2004         }
2005
2006         return ret;
2007 }
2008
2009 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2010                                       struct arm_smmu_device *smmu)
2011 {
2012         struct device *dev = smmu->dev;
2013         struct acpi_iort_node *node =
2014                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2015         struct acpi_iort_smmu *iort_smmu;
2016         int ret;
2017
2018         /* Retrieve SMMU1/2 specific data */
2019         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2020
2021         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2022         if (ret < 0)
2023                 return ret;
2024
2025         /* Ignore the configuration access interrupt */
2026         smmu->num_global_irqs = 1;
2027
2028         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2029                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2030
2031         return 0;
2032 }
2033 #else
2034 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2035                                              struct arm_smmu_device *smmu)
2036 {
2037         return -ENODEV;
2038 }
2039 #endif
2040
2041 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2042                                     struct arm_smmu_device *smmu)
2043 {
2044         const struct arm_smmu_match_data *data;
2045         struct device *dev = &pdev->dev;
2046         bool legacy_binding;
2047
2048         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2049                                  &smmu->num_global_irqs)) {
2050                 dev_err(dev, "missing #global-interrupts property\n");
2051                 return -ENODEV;
2052         }
2053
2054         data = of_device_get_match_data(dev);
2055         smmu->version = data->version;
2056         smmu->model = data->model;
2057
2058         parse_driver_options(smmu);
2059
2060         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2061         if (legacy_binding && !using_generic_binding) {
2062                 if (!using_legacy_binding)
2063                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2064                 using_legacy_binding = true;
2065         } else if (!legacy_binding && !using_legacy_binding) {
2066                 using_generic_binding = true;
2067         } else {
2068                 dev_err(dev, "not probing due to mismatched DT properties\n");
2069                 return -ENODEV;
2070         }
2071
2072         if (of_dma_is_coherent(dev->of_node))
2073                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2074
2075         return 0;
2076 }
2077
2078 static int arm_smmu_device_probe(struct platform_device *pdev)
2079 {
2080         struct resource *res;
2081         resource_size_t ioaddr;
2082         struct arm_smmu_device *smmu;
2083         struct device *dev = &pdev->dev;
2084         int num_irqs, i, err;
2085
2086         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2087         if (!smmu) {
2088                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2089                 return -ENOMEM;
2090         }
2091         smmu->dev = dev;
2092
2093         if (dev->of_node)
2094                 err = arm_smmu_device_dt_probe(pdev, smmu);
2095         else
2096                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2097
2098         if (err)
2099                 return err;
2100
2101         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2102         ioaddr = res->start;
2103         smmu->base = devm_ioremap_resource(dev, res);
2104         if (IS_ERR(smmu->base))
2105                 return PTR_ERR(smmu->base);
2106         smmu->size = resource_size(res);
2107
2108         num_irqs = 0;
2109         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2110                 num_irqs++;
2111                 if (num_irqs > smmu->num_global_irqs)
2112                         smmu->num_context_irqs++;
2113         }
2114
2115         if (!smmu->num_context_irqs) {
2116                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2117                         num_irqs, smmu->num_global_irqs + 1);
2118                 return -ENODEV;
2119         }
2120
2121         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2122                                   GFP_KERNEL);
2123         if (!smmu->irqs) {
2124                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2125                 return -ENOMEM;
2126         }
2127
2128         for (i = 0; i < num_irqs; ++i) {
2129                 int irq = platform_get_irq(pdev, i);
2130
2131                 if (irq < 0) {
2132                         dev_err(dev, "failed to get irq index %d\n", i);
2133                         return -ENODEV;
2134                 }
2135                 smmu->irqs[i] = irq;
2136         }
2137
2138         err = arm_smmu_device_cfg_probe(smmu);
2139         if (err)
2140                 return err;
2141
2142         if (smmu->version == ARM_SMMU_V2 &&
2143             smmu->num_context_banks != smmu->num_context_irqs) {
2144                 dev_err(dev,
2145                         "found only %d context interrupt(s) but %d required\n",
2146                         smmu->num_context_irqs, smmu->num_context_banks);
2147                 return -ENODEV;
2148         }
2149
2150         for (i = 0; i < smmu->num_global_irqs; ++i) {
2151                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2152                                        arm_smmu_global_fault,
2153                                        IRQF_SHARED,
2154                                        "arm-smmu global fault",
2155                                        smmu);
2156                 if (err) {
2157                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2158                                 i, smmu->irqs[i]);
2159                         return err;
2160                 }
2161         }
2162
2163         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2164                                      "smmu.%pa", &ioaddr);
2165         if (err) {
2166                 dev_err(dev, "Failed to register iommu in sysfs\n");
2167                 return err;
2168         }
2169
2170         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2171         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2172
2173         err = iommu_device_register(&smmu->iommu);
2174         if (err) {
2175                 dev_err(dev, "Failed to register iommu\n");
2176                 return err;
2177         }
2178
2179         platform_set_drvdata(pdev, smmu);
2180         arm_smmu_device_reset(smmu);
2181         arm_smmu_test_smr_masks(smmu);
2182
2183         /* Oh, for a proper bus abstraction */
2184         if (!iommu_present(&platform_bus_type))
2185                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2186 #ifdef CONFIG_ARM_AMBA
2187         if (!iommu_present(&amba_bustype))
2188                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2189 #endif
2190 #ifdef CONFIG_PCI
2191         if (!iommu_present(&pci_bus_type)) {
2192                 pci_request_acs();
2193                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2194         }
2195 #endif
2196         return 0;
2197 }
2198
2199 static int arm_smmu_device_remove(struct platform_device *pdev)
2200 {
2201         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2202
2203         if (!smmu)
2204                 return -ENODEV;
2205
2206         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2207                 dev_err(&pdev->dev, "removing device with active domains!\n");
2208
2209         /* Turn the thing off */
2210         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2211         return 0;
2212 }
2213
2214 static struct platform_driver arm_smmu_driver = {
2215         .driver = {
2216                 .name           = "arm-smmu",
2217                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2218         },
2219         .probe  = arm_smmu_device_probe,
2220         .remove = arm_smmu_device_remove,
2221 };
2222
2223 static int __init arm_smmu_init(void)
2224 {
2225         static bool registered;
2226         int ret = 0;
2227
2228         if (!registered) {
2229                 ret = platform_driver_register(&arm_smmu_driver);
2230                 registered = !ret;
2231         }
2232         return ret;
2233 }
2234
2235 static void __exit arm_smmu_exit(void)
2236 {
2237         return platform_driver_unregister(&arm_smmu_driver);
2238 }
2239
2240 subsys_initcall(arm_smmu_init);
2241 module_exit(arm_smmu_exit);
2242
2243 static int __init arm_smmu_of_init(struct device_node *np)
2244 {
2245         int ret = arm_smmu_init();
2246
2247         if (ret)
2248                 return ret;
2249
2250         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2251                 return -ENODEV;
2252
2253         return 0;
2254 }
2255 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2256 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2257 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2258 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2259 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2260 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2261
2262 #ifdef CONFIG_ACPI
2263 static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2264 {
2265         if (iort_node_match(ACPI_IORT_NODE_SMMU))
2266                 return arm_smmu_init();
2267
2268         return 0;
2269 }
2270 IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2271 #endif
2272
2273 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2274 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2275 MODULE_LICENSE("GPL v2");