drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/atomic.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-iommu.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/io.h>
  38 #include <linux/io-64-nonatomic-hi-lo.h>
  39 #include <linux/iommu.h>
  40 #include <linux/iopoll.h>
  41 #include <linux/module.h>
  42 #include <linux/of.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_device.h>
  45 #include <linux/of_iommu.h>
  46 #include <linux/pci.h>
  47 #include <linux/platform_device.h>
  48 #include <linux/slab.h>
  49 #include <linux/spinlock.h>
  50
  51 #include <linux/amba/bus.h>
  52
  53 #include "io-pgtable.h"
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* SMMU global address space */
  59 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  60 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  61
  62 /*
  63  * SMMU global address space with conditional offset to access secure
  64  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  65  * nsGFSYNR0: 0x450)
  66  */
  67 #define ARM_SMMU_GR0_NS(smmu)                                           \
  68         ((smmu)->base +                                                 \
  69                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  70                         ? 0x400 : 0))
  71
  72 /*
  73  * Some 64-bit registers only make sense to write atomically, but in such
  74  * cases all the data relevant to AArch32 formats lies within the lower word,
  75  * therefore this actually makes more sense than it might first appear.
  76  */
  77 #ifdef CONFIG_64BIT
  78 #define smmu_write_atomic_lq            writeq_relaxed
  79 #else
  80 #define smmu_write_atomic_lq            writel_relaxed
  81 #endif
  82
  83 /* Configuration registers */
  84 #define ARM_SMMU_GR0_sCR0               0x0
  85 #define sCR0_CLIENTPD                   (1 << 0)
  86 #define sCR0_GFRE                       (1 << 1)
  87 #define sCR0_GFIE                       (1 << 2)
  88 #define sCR0_GCFGFRE                    (1 << 4)
  89 #define sCR0_GCFGFIE                    (1 << 5)
  90 #define sCR0_USFCFG                     (1 << 10)
  91 #define sCR0_VMIDPNE                    (1 << 11)
  92 #define sCR0_PTM                        (1 << 12)
  93 #define sCR0_FB                         (1 << 13)
  94 #define sCR0_VMID16EN                   (1 << 31)
  95 #define sCR0_BSU_SHIFT                  14
  96 #define sCR0_BSU_MASK                   0x3
  97
  98 /* Auxiliary Configuration register */
  99 #define ARM_SMMU_GR0_sACR               0x10
 100
 101 /* Identification registers */
 102 #define ARM_SMMU_GR0_ID0                0x20
 103 #define ARM_SMMU_GR0_ID1                0x24
 104 #define ARM_SMMU_GR0_ID2                0x28
 105 #define ARM_SMMU_GR0_ID3                0x2c
 106 #define ARM_SMMU_GR0_ID4                0x30
 107 #define ARM_SMMU_GR0_ID5                0x34
 108 #define ARM_SMMU_GR0_ID6                0x38
 109 #define ARM_SMMU_GR0_ID7                0x3c
 110 #define ARM_SMMU_GR0_sGFSR              0x48
 111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 114
 115 #define ID0_S1TS                        (1 << 30)
 116 #define ID0_S2TS                        (1 << 29)
 117 #define ID0_NTS                         (1 << 28)
 118 #define ID0_SMS                         (1 << 27)
 119 #define ID0_ATOSNS                      (1 << 26)
 120 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 121 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 122 #define ID0_CTTW                        (1 << 14)
 123 #define ID0_NUMIRPT_SHIFT               16
 124 #define ID0_NUMIRPT_MASK                0xff
 125 #define ID0_NUMSIDB_SHIFT               9
 126 #define ID0_NUMSIDB_MASK                0xf
 127 #define ID0_NUMSMRG_SHIFT               0
 128 #define ID0_NUMSMRG_MASK                0xff
 129
 130 #define ID1_PAGESIZE                    (1 << 31)
 131 #define ID1_NUMPAGENDXB_SHIFT           28
 132 #define ID1_NUMPAGENDXB_MASK            7
 133 #define ID1_NUMS2CB_SHIFT               16
 134 #define ID1_NUMS2CB_MASK                0xff
 135 #define ID1_NUMCB_SHIFT                 0
 136 #define ID1_NUMCB_MASK                  0xff
 137
 138 #define ID2_OAS_SHIFT                   4
 139 #define ID2_OAS_MASK                    0xf
 140 #define ID2_IAS_SHIFT                   0
 141 #define ID2_IAS_MASK                    0xf
 142 #define ID2_UBS_SHIFT                   8
 143 #define ID2_UBS_MASK                    0xf
 144 #define ID2_PTFS_4K                     (1 << 12)
 145 #define ID2_PTFS_16K                    (1 << 13)
 146 #define ID2_PTFS_64K                    (1 << 14)
 147 #define ID2_VMID16                      (1 << 15)
 148
 149 #define ID7_MAJOR_SHIFT                 4
 150 #define ID7_MAJOR_MASK                  0xf
 151
 152 /* Global TLB invalidation */
 153 #define ARM_SMMU_GR0_TLBIVMID           0x64
 154 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 155 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 156 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 157 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 158 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 159 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 160
 161 /* Stream mapping registers */
 162 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 163 #define SMR_VALID                       (1 << 31)
 164 #define SMR_MASK_SHIFT                  16
 165 #define SMR_ID_SHIFT                    0
 166
 167 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 168 #define S2CR_CBNDX_SHIFT                0
 169 #define S2CR_CBNDX_MASK                 0xff
 170 #define S2CR_TYPE_SHIFT                 16
 171 #define S2CR_TYPE_MASK                  0x3
 172 enum arm_smmu_s2cr_type {
 173         S2CR_TYPE_TRANS,
 174         S2CR_TYPE_BYPASS,
 175         S2CR_TYPE_FAULT,
 176 };
 177
 178 #define S2CR_PRIVCFG_SHIFT              24
 179 #define S2CR_PRIVCFG_MASK               0x3
 180 enum arm_smmu_s2cr_privcfg {
 181         S2CR_PRIVCFG_DEFAULT,
 182         S2CR_PRIVCFG_DIPAN,
 183         S2CR_PRIVCFG_UNPRIV,
 184         S2CR_PRIVCFG_PRIV,
 185 };
 186
 187 /* Context bank attribute registers */
 188 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 189 #define CBAR_VMID_SHIFT                 0
 190 #define CBAR_VMID_MASK                  0xff
 191 #define CBAR_S1_BPSHCFG_SHIFT           8
 192 #define CBAR_S1_BPSHCFG_MASK            3
 193 #define CBAR_S1_BPSHCFG_NSH             3
 194 #define CBAR_S1_MEMATTR_SHIFT           12
 195 #define CBAR_S1_MEMATTR_MASK            0xf
 196 #define CBAR_S1_MEMATTR_WB              0xf
 197 #define CBAR_TYPE_SHIFT                 16
 198 #define CBAR_TYPE_MASK                  0x3
 199 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 201 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 203 #define CBAR_IRPTNDX_SHIFT              24
 204 #define CBAR_IRPTNDX_MASK               0xff
 205
 206 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 207 #define CBA2R_RW64_32BIT                (0 << 0)
 208 #define CBA2R_RW64_64BIT                (1 << 0)
 209 #define CBA2R_VMID_SHIFT                16
 210 #define CBA2R_VMID_MASK                 0xffff
 211
 212 /* Translation context bank */
 213 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 214 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 215
 216 #define ARM_SMMU_CB_SCTLR               0x0
 217 #define ARM_SMMU_CB_ACTLR               0x4
 218 #define ARM_SMMU_CB_RESUME              0x8
 219 #define ARM_SMMU_CB_TTBCR2              0x10
 220 #define ARM_SMMU_CB_TTBR0               0x20
 221 #define ARM_SMMU_CB_TTBR1               0x28
 222 #define ARM_SMMU_CB_TTBCR               0x30
 223 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 224 #define ARM_SMMU_CB_S1_MAIR0            0x38
 225 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 226 #define ARM_SMMU_CB_PAR                 0x50
 227 #define ARM_SMMU_CB_FSR                 0x58
 228 #define ARM_SMMU_CB_FAR                 0x60
 229 #define ARM_SMMU_CB_FSYNR0              0x68
 230 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 231 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 232 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 233 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 234 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 235 #define ARM_SMMU_CB_ATS1PR              0x800
 236 #define ARM_SMMU_CB_ATSR                0x8f0
 237
 238 #define SCTLR_S1_ASIDPNE                (1 << 12)
 239 #define SCTLR_CFCFG                     (1 << 7)
 240 #define SCTLR_CFIE                      (1 << 6)
 241 #define SCTLR_CFRE                      (1 << 5)
 242 #define SCTLR_E                         (1 << 4)
 243 #define SCTLR_AFE                       (1 << 2)
 244 #define SCTLR_TRE                       (1 << 1)
 245 #define SCTLR_M                         (1 << 0)
 246
 247 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 248
 249 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 250
 251 #define CB_PAR_F                        (1 << 0)
 252
 253 #define ATSR_ACTIVE                     (1 << 0)
 254
 255 #define RESUME_RETRY                    (0 << 0)
 256 #define RESUME_TERMINATE                (1 << 0)
 257
 258 #define TTBCR2_SEP_SHIFT                15
 259 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 260
 261 #define TTBRn_ASID_SHIFT                48
 262
 263 #define FSR_MULTI                       (1 << 31)
 264 #define FSR_SS                          (1 << 30)
 265 #define FSR_UUT                         (1 << 8)
 266 #define FSR_ASF                         (1 << 7)
 267 #define FSR_TLBLKF                      (1 << 6)
 268 #define FSR_TLBMCF                      (1 << 5)
 269 #define FSR_EF                          (1 << 4)
 270 #define FSR_PF                          (1 << 3)
 271 #define FSR_AFF                         (1 << 2)
 272 #define FSR_TF                          (1 << 1)
 273
 274 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 275                                          FSR_TLBMCF | FSR_TLBLKF)
 276 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 277                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 278
 279 #define FSYNR0_WNR                      (1 << 4)
 280
 281 static int force_stage;
 282 module_param(force_stage, int, S_IRUGO);
 283 MODULE_PARM_DESC(force_stage,
 284         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 285 static bool disable_bypass;
 286 module_param(disable_bypass, bool, S_IRUGO);
 287 MODULE_PARM_DESC(disable_bypass,
 288         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 289
 290 enum arm_smmu_arch_version {
 291         ARM_SMMU_V1,
 292         ARM_SMMU_V1_64K,
 293         ARM_SMMU_V2,
 294 };
 295
 296 enum arm_smmu_implementation {
 297         GENERIC_SMMU,
 298         ARM_MMU500,
 299         CAVIUM_SMMUV2,
 300 };
 301
 302 struct arm_smmu_s2cr {
 303         struct iommu_group              *group;
 304         int                             count;
 305         enum arm_smmu_s2cr_type         type;
 306         enum arm_smmu_s2cr_privcfg      privcfg;
 307         u8                              cbndx;
 308 };
 309
 310 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 311         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 312 }
 313
 314 struct arm_smmu_smr {
 315         u16                             mask;
 316         u16                             id;
 317         bool                            valid;
 318 };
 319
 320 struct arm_smmu_master_cfg {
 321         struct arm_smmu_device          *smmu;
 322         s16                             smendx[];
 323 };
 324 #define INVALID_SMENDX                  -1
 325 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 326 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 327 #define for_each_cfg_sme(fw, i, idx) \
 328         for (i = 0; idx = __fwspec_cfg(fw)->smendx[i], i < fw->num_ids; ++i)
 329
 330 struct arm_smmu_device {
 331         struct device                   *dev;
 332
 333         void __iomem                    *base;
 334         unsigned long                   size;
 335         unsigned long                   pgshift;
 336
 337 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 338 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 339 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 340 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 341 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 342 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 343 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 344 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 345 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 346 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 347 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 348 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 349         u32                             features;
 350
 351 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 352         u32                             options;
 353         enum arm_smmu_arch_version      version;
 354         enum arm_smmu_implementation    model;
 355
 356         u32                             num_context_banks;
 357         u32                             num_s2_context_banks;
 358         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 359         atomic_t                        irptndx;
 360
 361         u32                             num_mapping_groups;
 362         u16                             streamid_mask;
 363         u16                             smr_mask_mask;
 364         struct arm_smmu_smr             *smrs;
 365         struct arm_smmu_s2cr            *s2crs;
 366         struct mutex                    stream_map_mutex;
 367
 368         unsigned long                   va_size;
 369         unsigned long                   ipa_size;
 370         unsigned long                   pa_size;
 371         unsigned long                   pgsize_bitmap;
 372
 373         u32                             num_global_irqs;
 374         u32                             num_context_irqs;
 375         unsigned int                    *irqs;
 376
 377         u32                             cavium_id_base; /* Specific to Cavium */
 378 };
 379
 380 enum arm_smmu_context_fmt {
 381         ARM_SMMU_CTX_FMT_NONE,
 382         ARM_SMMU_CTX_FMT_AARCH64,
 383         ARM_SMMU_CTX_FMT_AARCH32_L,
 384         ARM_SMMU_CTX_FMT_AARCH32_S,
 385 };
 386
 387 struct arm_smmu_cfg {
 388         u8                              cbndx;
 389         u8                              irptndx;
 390         u32                             cbar;
 391         enum arm_smmu_context_fmt       fmt;
 392 };
 393 #define INVALID_IRPTNDX                 0xff
 394
 395 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 396 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 397
 398 enum arm_smmu_domain_stage {
 399         ARM_SMMU_DOMAIN_S1 = 0,
 400         ARM_SMMU_DOMAIN_S2,
 401         ARM_SMMU_DOMAIN_NESTED,
 402 };
 403
 404 struct arm_smmu_domain {
 405         struct arm_smmu_device          *smmu;
 406         struct io_pgtable_ops           *pgtbl_ops;
 407         spinlock_t                      pgtbl_lock;
 408         struct arm_smmu_cfg             cfg;
 409         enum arm_smmu_domain_stage      stage;
 410         struct mutex                    init_mutex; /* Protects smmu pointer */
 411         struct iommu_domain             domain;
 412 };
 413
 414 struct arm_smmu_option_prop {
 415         u32 opt;
 416         const char *prop;
 417 };
 418
 419 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 420
 421 static bool using_legacy_binding, using_generic_binding;
 422
 423 static struct arm_smmu_option_prop arm_smmu_options[] = {
 424         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 425         { 0, NULL},
 426 };
 427
 428 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 429 {
 430         return container_of(dom, struct arm_smmu_domain, domain);
 431 }
 432
 433 static void parse_driver_options(struct arm_smmu_device *smmu)
 434 {
 435         int i = 0;
 436
 437         do {
 438                 if (of_property_read_bool(smmu->dev->of_node,
 439                                                 arm_smmu_options[i].prop)) {
 440                         smmu->options |= arm_smmu_options[i].opt;
 441                         dev_notice(smmu->dev, "option %s\n",
 442                                 arm_smmu_options[i].prop);
 443                 }
 444         } while (arm_smmu_options[++i].opt);
 445 }
 446
 447 static struct device_node *dev_get_dev_node(struct device *dev)
 448 {
 449         if (dev_is_pci(dev)) {
 450                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 451
 452                 while (!pci_is_root_bus(bus))
 453                         bus = bus->parent;
 454                 return of_node_get(bus->bridge->parent->of_node);
 455         }
 456
 457         return of_node_get(dev->of_node);
 458 }
 459
 460 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 461 {
 462         *((__be32 *)data) = cpu_to_be32(alias);
 463         return 0; /* Continue walking */
 464 }
 465
 466 static int __find_legacy_master_phandle(struct device *dev, void *data)
 467 {
 468         struct of_phandle_iterator *it = *(void **)data;
 469         struct device_node *np = it->node;
 470         int err;
 471
 472         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 473                             "#stream-id-cells", 0)
 474                 if (it->node == np) {
 475                         *(void **)data = dev;
 476                         return 1;
 477                 }
 478         it->node = np;
 479         return err == -ENOENT ? 0 : err;
 480 }
 481
 482 static struct platform_driver arm_smmu_driver;
 483 static struct iommu_ops arm_smmu_ops;
 484
 485 static int arm_smmu_register_legacy_master(struct device *dev,
 486                                            struct arm_smmu_device **smmu)
 487 {
 488         struct device *smmu_dev;
 489         struct device_node *np;
 490         struct of_phandle_iterator it;
 491         void *data = &it;
 492         u32 *sids;
 493         __be32 pci_sid;
 494         int err;
 495
 496         np = dev_get_dev_node(dev);
 497         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 498                 of_node_put(np);
 499                 return -ENODEV;
 500         }
 501
 502         it.node = np;
 503         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 504                                      __find_legacy_master_phandle);
 505         smmu_dev = data;
 506         of_node_put(np);
 507         if (err == 0)
 508                 return -ENODEV;
 509         if (err < 0)
 510                 return err;
 511
 512         if (dev_is_pci(dev)) {
 513                 /* "mmu-masters" assumes Stream ID == Requester ID */
 514                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 515                                        &pci_sid);
 516                 it.cur = &pci_sid;
 517                 it.cur_count = 1;
 518         }
 519
 520         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 521                                 &arm_smmu_ops);
 522         if (err)
 523                 return err;
 524
 525         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 526         if (!sids)
 527                 return -ENOMEM;
 528
 529         *smmu = dev_get_drvdata(smmu_dev);
 530         of_phandle_iterator_args(&it, sids, it.cur_count);
 531         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 532         kfree(sids);
 533         return err;
 534 }
 535
 536 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 537 {
 538         int idx;
 539
 540         do {
 541                 idx = find_next_zero_bit(map, end, start);
 542                 if (idx == end)
 543                         return -ENOSPC;
 544         } while (test_and_set_bit(idx, map));
 545
 546         return idx;
 547 }
 548
 549 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 550 {
 551         clear_bit(idx, map);
 552 }
 553
 554 /* Wait for any pending TLB invalidations to complete */
 555 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 556 {
 557         int count = 0;
 558         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 559
 560         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 561         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 562                & sTLBGSTATUS_GSACTIVE) {
 563                 cpu_relax();
 564                 if (++count == TLB_LOOP_TIMEOUT) {
 565                         dev_err_ratelimited(smmu->dev,
 566                         "TLB sync timed out -- SMMU may be deadlocked\n");
 567                         return;
 568                 }
 569                 udelay(1);
 570         }
 571 }
 572
 573 static void arm_smmu_tlb_sync(void *cookie)
 574 {
 575         struct arm_smmu_domain *smmu_domain = cookie;
 576         __arm_smmu_tlb_sync(smmu_domain->smmu);
 577 }
 578
 579 static void arm_smmu_tlb_inv_context(void *cookie)
 580 {
 581         struct arm_smmu_domain *smmu_domain = cookie;
 582         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 583         struct arm_smmu_device *smmu = smmu_domain->smmu;
 584         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 585         void __iomem *base;
 586
 587         if (stage1) {
 588                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 589                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 590                                base + ARM_SMMU_CB_S1_TLBIASID);
 591         } else {
 592                 base = ARM_SMMU_GR0(smmu);
 593                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 594                                base + ARM_SMMU_GR0_TLBIVMID);
 595         }
 596
 597         __arm_smmu_tlb_sync(smmu);
 598 }
 599
 600 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 601                                           size_t granule, bool leaf, void *cookie)
 602 {
 603         struct arm_smmu_domain *smmu_domain = cookie;
 604         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 605         struct arm_smmu_device *smmu = smmu_domain->smmu;
 606         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 607         void __iomem *reg;
 608
 609         if (stage1) {
 610                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 611                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 612
 613                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 614                         iova &= ~12UL;
 615                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 616                         do {
 617                                 writel_relaxed(iova, reg);
 618                                 iova += granule;
 619                         } while (size -= granule);
 620                 } else {
 621                         iova >>= 12;
 622                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 623                         do {
 624                                 writeq_relaxed(iova, reg);
 625                                 iova += granule >> 12;
 626                         } while (size -= granule);
 627                 }
 628         } else if (smmu->version == ARM_SMMU_V2) {
 629                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 630                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 631                               ARM_SMMU_CB_S2_TLBIIPAS2;
 632                 iova >>= 12;
 633                 do {
 634                         smmu_write_atomic_lq(iova, reg);
 635                         iova += granule >> 12;
 636                 } while (size -= granule);
 637         } else {
 638                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 639                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 640         }
 641 }
 642
 643 static struct iommu_gather_ops arm_smmu_gather_ops = {
 644         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 645         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 646         .tlb_sync       = arm_smmu_tlb_sync,
 647 };
 648
 649 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 650 {
 651         u32 fsr, fsynr;
 652         unsigned long iova;
 653         struct iommu_domain *domain = dev;
 654         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 655         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 656         struct arm_smmu_device *smmu = smmu_domain->smmu;
 657         void __iomem *cb_base;
 658
 659         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 660         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 661
 662         if (!(fsr & FSR_FAULT))
 663                 return IRQ_NONE;
 664
 665         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 666         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 667
 668         dev_err_ratelimited(smmu->dev,
 669         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 670                             fsr, iova, fsynr, cfg->cbndx);
 671
 672         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 673         return IRQ_HANDLED;
 674 }
 675
 676 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 677 {
 678         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 679         struct arm_smmu_device *smmu = dev;
 680         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 681
 682         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 683         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 684         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 685         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 686
 687         if (!gfsr)
 688                 return IRQ_NONE;
 689
 690         dev_err_ratelimited(smmu->dev,
 691                 "Unexpected global fault, this could be serious\n");
 692         dev_err_ratelimited(smmu->dev,
 693                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 694                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 695
 696         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 697         return IRQ_HANDLED;
 698 }
 699
 700 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 701                                        struct io_pgtable_cfg *pgtbl_cfg)
 702 {
 703         u32 reg, reg2;
 704         u64 reg64;
 705         bool stage1;
 706         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 707         struct arm_smmu_device *smmu = smmu_domain->smmu;
 708         void __iomem *cb_base, *gr1_base;
 709
 710         gr1_base = ARM_SMMU_GR1(smmu);
 711         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 712         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 713
 714         if (smmu->version > ARM_SMMU_V1) {
 715                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 716                         reg = CBA2R_RW64_64BIT;
 717                 else
 718                         reg = CBA2R_RW64_32BIT;
 719                 /* 16-bit VMIDs live in CBA2R */
 720                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 721                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 722
 723                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 724         }
 725
 726         /* CBAR */
 727         reg = cfg->cbar;
 728         if (smmu->version < ARM_SMMU_V2)
 729                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 730
 731         /*
 732          * Use the weakest shareability/memory types, so they are
 733          * overridden by the ttbcr/pte.
 734          */
 735         if (stage1) {
 736                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 737                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 738         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 739                 /* 8-bit VMIDs live in CBAR */
 740                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 741         }
 742         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 743
 744         /* TTBRs */
 745         if (stage1) {
 746                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 747
 748                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 749                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 750                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 751                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 752                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 753                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 754                 } else {
 755                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 756                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 757                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 758                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 759                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 760                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 761                 }
 762         } else {
 763                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 764                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 765         }
 766
 767         /* TTBCR */
 768         if (stage1) {
 769                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 770                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 771                         reg2 = 0;
 772                 } else {
 773                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 774                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 775                         reg2 |= TTBCR2_SEP_UPSTREAM;
 776                 }
 777                 if (smmu->version > ARM_SMMU_V1)
 778                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 779         } else {
 780                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 781         }
 782         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 783
 784         /* MAIRs (stage-1 only) */
 785         if (stage1) {
 786                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 787                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 788                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 789                 } else {
 790                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 791                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 792                 }
 793                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 794                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 795         }
 796
 797         /* SCTLR */
 798         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 799         if (stage1)
 800                 reg |= SCTLR_S1_ASIDPNE;
 801 #ifdef __BIG_ENDIAN
 802         reg |= SCTLR_E;
 803 #endif
 804         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 805 }
 806
 807 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 808                                         struct arm_smmu_device *smmu)
 809 {
 810         int irq, start, ret = 0;
 811         unsigned long ias, oas;
 812         struct io_pgtable_ops *pgtbl_ops;
 813         struct io_pgtable_cfg pgtbl_cfg;
 814         enum io_pgtable_fmt fmt;
 815         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 816         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 817
 818         mutex_lock(&smmu_domain->init_mutex);
 819         if (smmu_domain->smmu)
 820                 goto out_unlock;
 821
 822         /*
 823          * Mapping the requested stage onto what we support is surprisingly
 824          * complicated, mainly because the spec allows S1+S2 SMMUs without
 825          * support for nested translation. That means we end up with the
 826          * following table:
 827          *
 828          * Requested        Supported        Actual
 829          *     S1               N              S1
 830          *     S1             S1+S2            S1
 831          *     S1               S2             S2
 832          *     S1               S1             S1
 833          *     N                N              N
 834          *     N              S1+S2            S2
 835          *     N                S2             S2
 836          *     N                S1             S1
 837          *
 838          * Note that you can't actually request stage-2 mappings.
 839          */
 840         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 841                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 842         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 843                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 844
 845         /*
 846          * Choosing a suitable context format is even more fiddly. Until we
 847          * grow some way for the caller to express a preference, and/or move
 848          * the decision into the io-pgtable code where it arguably belongs,
 849          * just aim for the closest thing to the rest of the system, and hope
 850          * that the hardware isn't esoteric enough that we can't assume AArch64
 851          * support to be a superset of AArch32 support...
 852          */
 853         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 854                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 855         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 856             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 857             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 858             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 859                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 860         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 861             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 862                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 863                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 864                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 865
 866         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 867                 ret = -EINVAL;
 868                 goto out_unlock;
 869         }
 870
 871         switch (smmu_domain->stage) {
 872         case ARM_SMMU_DOMAIN_S1:
 873                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 874                 start = smmu->num_s2_context_banks;
 875                 ias = smmu->va_size;
 876                 oas = smmu->ipa_size;
 877                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 878                         fmt = ARM_64_LPAE_S1;
 879                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 880                         fmt = ARM_32_LPAE_S1;
 881                         ias = min(ias, 32UL);
 882                         oas = min(oas, 40UL);
 883                 } else {
 884                         fmt = ARM_V7S;
 885                         ias = min(ias, 32UL);
 886                         oas = min(oas, 32UL);
 887                 }
 888                 break;
 889         case ARM_SMMU_DOMAIN_NESTED:
 890                 /*
 891                  * We will likely want to change this if/when KVM gets
 892                  * involved.
 893                  */
 894         case ARM_SMMU_DOMAIN_S2:
 895                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 896                 start = 0;
 897                 ias = smmu->ipa_size;
 898                 oas = smmu->pa_size;
 899                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 900                         fmt = ARM_64_LPAE_S2;
 901                 } else {
 902                         fmt = ARM_32_LPAE_S2;
 903                         ias = min(ias, 40UL);
 904                         oas = min(oas, 40UL);
 905                 }
 906                 break;
 907         default:
 908                 ret = -EINVAL;
 909                 goto out_unlock;
 910         }
 911
 912         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 913                                       smmu->num_context_banks);
 914         if (ret < 0)
 915                 goto out_unlock;
 916
 917         cfg->cbndx = ret;
 918         if (smmu->version < ARM_SMMU_V2) {
 919                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 920                 cfg->irptndx %= smmu->num_context_irqs;
 921         } else {
 922                 cfg->irptndx = cfg->cbndx;
 923         }
 924
 925         pgtbl_cfg = (struct io_pgtable_cfg) {
 926                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 927                 .ias            = ias,
 928                 .oas            = oas,
 929                 .tlb            = &arm_smmu_gather_ops,
 930                 .iommu_dev      = smmu->dev,
 931         };
 932
 933         smmu_domain->smmu = smmu;
 934         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 935         if (!pgtbl_ops) {
 936                 ret = -ENOMEM;
 937                 goto out_clear_smmu;
 938         }
 939
 940         /* Update the domain's page sizes to reflect the page table format */
 941         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 942         domain->geometry.aperture_end = (1UL << ias) - 1;
 943         domain->geometry.force_aperture = true;
 944
 945         /* Initialise the context bank with our page table cfg */
 946         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 947
 948         /*
 949          * Request context fault interrupt. Do this last to avoid the
 950          * handler seeing a half-initialised domain state.
 951          */
 952         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 953         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 954                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 955         if (ret < 0) {
 956                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 957                         cfg->irptndx, irq);
 958                 cfg->irptndx = INVALID_IRPTNDX;
 959         }
 960
 961         mutex_unlock(&smmu_domain->init_mutex);
 962
 963         /* Publish page table ops for map/unmap */
 964         smmu_domain->pgtbl_ops = pgtbl_ops;
 965         return 0;
 966
 967 out_clear_smmu:
 968         smmu_domain->smmu = NULL;
 969 out_unlock:
 970         mutex_unlock(&smmu_domain->init_mutex);
 971         return ret;
 972 }
 973
 974 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 975 {
 976         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 977         struct arm_smmu_device *smmu = smmu_domain->smmu;
 978         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 979         void __iomem *cb_base;
 980         int irq;
 981
 982         if (!smmu)
 983                 return;
 984
 985         /*
 986          * Disable the context bank and free the page tables before freeing
 987          * it.
 988          */
 989         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 990         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 991
 992         if (cfg->irptndx != INVALID_IRPTNDX) {
 993                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 994                 devm_free_irq(smmu->dev, irq, domain);
 995         }
 996
 997         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 998         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 999 }
1000
1001 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1002 {
1003         struct arm_smmu_domain *smmu_domain;
1004
1005         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1006                 return NULL;
1007         /*
1008          * Allocate the domain and initialise some of its data structures.
1009          * We can't really do anything meaningful until we've added a
1010          * master.
1011          */
1012         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1013         if (!smmu_domain)
1014                 return NULL;
1015
1016         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1017             iommu_get_dma_cookie(&smmu_domain->domain))) {
1018                 kfree(smmu_domain);
1019                 return NULL;
1020         }
1021
1022         mutex_init(&smmu_domain->init_mutex);
1023         spin_lock_init(&smmu_domain->pgtbl_lock);
1024
1025         return &smmu_domain->domain;
1026 }
1027
1028 static void arm_smmu_domain_free(struct iommu_domain *domain)
1029 {
1030         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1031
1032         /*
1033          * Free the domain resources. We assume that all devices have
1034          * already been detached.
1035          */
1036         iommu_put_dma_cookie(domain);
1037         arm_smmu_destroy_domain_context(domain);
1038         kfree(smmu_domain);
1039 }
1040
1041 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1042 {
1043         struct arm_smmu_smr *smr = smmu->smrs + idx;
1044         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1045
1046         if (smr->valid)
1047                 reg |= SMR_VALID;
1048         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1049 }
1050
1051 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1052 {
1053         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1054         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1055                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1056                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1057
1058         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1059 }
1060
1061 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1062 {
1063         arm_smmu_write_s2cr(smmu, idx);
1064         if (smmu->smrs)
1065                 arm_smmu_write_smr(smmu, idx);
1066 }
1067
1068 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1069 {
1070         struct arm_smmu_smr *smrs = smmu->smrs;
1071         int i, free_idx = -ENOSPC;
1072
1073         /* Stream indexing is blissfully easy */
1074         if (!smrs)
1075                 return id;
1076
1077         /* Validating SMRs is... less so */
1078         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1079                 if (!smrs[i].valid) {
1080                         /*
1081                          * Note the first free entry we come across, which
1082                          * we'll claim in the end if nothing else matches.
1083                          */
1084                         if (free_idx < 0)
1085                                 free_idx = i;
1086                         continue;
1087                 }
1088                 /*
1089                  * If the new entry is _entirely_ matched by an existing entry,
1090                  * then reuse that, with the guarantee that there also cannot
1091                  * be any subsequent conflicting entries. In normal use we'd
1092                  * expect simply identical entries for this case, but there's
1093                  * no harm in accommodating the generalisation.
1094                  */
1095                 if ((mask & smrs[i].mask) == mask &&
1096                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1097                         return i;
1098                 /*
1099                  * If the new entry has any other overlap with an existing one,
1100                  * though, then there always exists at least one stream ID
1101                  * which would cause a conflict, and we can't allow that risk.
1102                  */
1103                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1104                         return -EINVAL;
1105         }
1106
1107         return free_idx;
1108 }
1109
1110 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1111 {
1112         if (--smmu->s2crs[idx].count)
1113                 return false;
1114
1115         smmu->s2crs[idx] = s2cr_init_val;
1116         if (smmu->smrs)
1117                 smmu->smrs[idx].valid = false;
1118
1119         return true;
1120 }
1121
1122 static int arm_smmu_master_alloc_smes(struct device *dev)
1123 {
1124         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1125         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1126         struct arm_smmu_device *smmu = cfg->smmu;
1127         struct arm_smmu_smr *smrs = smmu->smrs;
1128         struct iommu_group *group;
1129         int i, idx, ret;
1130
1131         mutex_lock(&smmu->stream_map_mutex);
1132         /* Figure out a viable stream map entry allocation */
1133         for_each_cfg_sme(fwspec, i, idx) {
1134                 u16 sid = fwspec->ids[i];
1135                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1136
1137                 if (idx != INVALID_SMENDX) {
1138                         ret = -EEXIST;
1139                         goto out_err;
1140                 }
1141
1142                 ret = arm_smmu_find_sme(smmu, sid, mask);
1143                 if (ret < 0)
1144                         goto out_err;
1145
1146                 idx = ret;
1147                 if (smrs && smmu->s2crs[idx].count == 0) {
1148                         smrs[idx].id = sid;
1149                         smrs[idx].mask = mask;
1150                         smrs[idx].valid = true;
1151                 }
1152                 smmu->s2crs[idx].count++;
1153                 cfg->smendx[i] = (s16)idx;
1154         }
1155
1156         group = iommu_group_get_for_dev(dev);
1157         if (!group)
1158                 group = ERR_PTR(-ENOMEM);
1159         if (IS_ERR(group)) {
1160                 ret = PTR_ERR(group);
1161                 goto out_err;
1162         }
1163         iommu_group_put(group);
1164
1165         /* It worked! Now, poke the actual hardware */
1166         for_each_cfg_sme(fwspec, i, idx) {
1167                 arm_smmu_write_sme(smmu, idx);
1168                 smmu->s2crs[idx].group = group;
1169         }
1170
1171         mutex_unlock(&smmu->stream_map_mutex);
1172         return 0;
1173
1174 out_err:
1175         while (i--) {
1176                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1177                 cfg->smendx[i] = INVALID_SMENDX;
1178         }
1179         mutex_unlock(&smmu->stream_map_mutex);
1180         return ret;
1181 }
1182
1183 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1184 {
1185         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1186         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1187         int i, idx;
1188
1189         mutex_lock(&smmu->stream_map_mutex);
1190         for_each_cfg_sme(fwspec, i, idx) {
1191                 if (arm_smmu_free_sme(smmu, idx))
1192                         arm_smmu_write_sme(smmu, idx);
1193                 cfg->smendx[i] = INVALID_SMENDX;
1194         }
1195         mutex_unlock(&smmu->stream_map_mutex);
1196 }
1197
1198 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1199                                       struct iommu_fwspec *fwspec)
1200 {
1201         struct arm_smmu_device *smmu = smmu_domain->smmu;
1202         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1203         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1204         u8 cbndx = smmu_domain->cfg.cbndx;
1205         int i, idx;
1206
1207         for_each_cfg_sme(fwspec, i, idx) {
1208                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1209                         continue;
1210
1211                 s2cr[idx].type = type;
1212                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1213                 s2cr[idx].cbndx = cbndx;
1214                 arm_smmu_write_s2cr(smmu, idx);
1215         }
1216         return 0;
1217 }
1218
1219 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1220 {
1221         int ret;
1222         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1223         struct arm_smmu_device *smmu;
1224         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1225
1226         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1227                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1228                 return -ENXIO;
1229         }
1230
1231         /*
1232          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1233          * domains between of_xlate() and add_device() - we have no way to cope
1234          * with that, so until ARM gets converted to rely on groups and default
1235          * domains, just say no (but more politely than by dereferencing NULL).
1236          * This should be at least a WARN_ON once that's sorted.
1237          */
1238         if (!fwspec->iommu_priv)
1239                 return -ENODEV;
1240
1241         smmu = fwspec_smmu(fwspec);
1242         /* Ensure that the domain is finalised */
1243         ret = arm_smmu_init_domain_context(domain, smmu);
1244         if (ret < 0)
1245                 return ret;
1246
1247         /*
1248          * Sanity check the domain. We don't support domains across
1249          * different SMMUs.
1250          */
1251         if (smmu_domain->smmu != smmu) {
1252                 dev_err(dev,
1253                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1254                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1255                 return -EINVAL;
1256         }
1257
1258         /* Looks ok, so add the device to the domain */
1259         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1260 }
1261
1262 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1263                         phys_addr_t paddr, size_t size, int prot)
1264 {
1265         int ret;
1266         unsigned long flags;
1267         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1268         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1269
1270         if (!ops)
1271                 return -ENODEV;
1272
1273         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1274         ret = ops->map(ops, iova, paddr, size, prot);
1275         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1276         return ret;
1277 }
1278
1279 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1280                              size_t size)
1281 {
1282         size_t ret;
1283         unsigned long flags;
1284         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1285         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1286
1287         if (!ops)
1288                 return 0;
1289
1290         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1291         ret = ops->unmap(ops, iova, size);
1292         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1293         return ret;
1294 }
1295
1296 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1297                                               dma_addr_t iova)
1298 {
1299         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1300         struct arm_smmu_device *smmu = smmu_domain->smmu;
1301         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1302         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1303         struct device *dev = smmu->dev;
1304         void __iomem *cb_base;
1305         u32 tmp;
1306         u64 phys;
1307         unsigned long va;
1308
1309         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1310
1311         /* ATS1 registers can only be written atomically */
1312         va = iova & ~0xfffUL;
1313         if (smmu->version == ARM_SMMU_V2)
1314                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1315         else /* Register is only 32-bit in v1 */
1316                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1317
1318         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1319                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1320                 dev_err(dev,
1321                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1322                         &iova);
1323                 return ops->iova_to_phys(ops, iova);
1324         }
1325
1326         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1327         if (phys & CB_PAR_F) {
1328                 dev_err(dev, "translation fault!\n");
1329                 dev_err(dev, "PAR = 0x%llx\n", phys);
1330                 return 0;
1331         }
1332
1333         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1334 }
1335
1336 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1337                                         dma_addr_t iova)
1338 {
1339         phys_addr_t ret;
1340         unsigned long flags;
1341         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1342         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1343
1344         if (!ops)
1345                 return 0;
1346
1347         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1348         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1349                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1350                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1351         } else {
1352                 ret = ops->iova_to_phys(ops, iova);
1353         }
1354
1355         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1356
1357         return ret;
1358 }
1359
1360 static bool arm_smmu_capable(enum iommu_cap cap)
1361 {
1362         switch (cap) {
1363         case IOMMU_CAP_CACHE_COHERENCY:
1364                 /*
1365                  * Return true here as the SMMU can always send out coherent
1366                  * requests.
1367                  */
1368                 return true;
1369         case IOMMU_CAP_INTR_REMAP:
1370                 return true; /* MSIs are just memory writes */
1371         case IOMMU_CAP_NOEXEC:
1372                 return true;
1373         default:
1374                 return false;
1375         }
1376 }
1377
1378 static int arm_smmu_match_node(struct device *dev, void *data)
1379 {
1380         return dev->of_node == data;
1381 }
1382
1383 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1384 {
1385         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1386                                                 np, arm_smmu_match_node);
1387         put_device(dev);
1388         return dev ? dev_get_drvdata(dev) : NULL;
1389 }
1390
1391 static int arm_smmu_add_device(struct device *dev)
1392 {
1393         struct arm_smmu_device *smmu;
1394         struct arm_smmu_master_cfg *cfg;
1395         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1396         int i, ret;
1397
1398         if (using_legacy_binding) {
1399                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1400                 fwspec = dev->iommu_fwspec;
1401                 if (ret)
1402                         goto out_free;
1403         } else if (fwspec) {
1404                 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1405         } else {
1406                 return -ENODEV;
1407         }
1408
1409         ret = -EINVAL;
1410         for (i = 0; i < fwspec->num_ids; i++) {
1411                 u16 sid = fwspec->ids[i];
1412                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1413
1414                 if (sid & ~smmu->streamid_mask) {
1415                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1416                                 sid, smmu->streamid_mask);
1417                         goto out_free;
1418                 }
1419                 if (mask & ~smmu->smr_mask_mask) {
1420                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1421                                 sid, smmu->smr_mask_mask);
1422                         goto out_free;
1423                 }
1424         }
1425
1426         ret = -ENOMEM;
1427         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1428                       GFP_KERNEL);
1429         if (!cfg)
1430                 goto out_free;
1431
1432         cfg->smmu = smmu;
1433         fwspec->iommu_priv = cfg;
1434         while (i--)
1435                 cfg->smendx[i] = INVALID_SMENDX;
1436
1437         ret = arm_smmu_master_alloc_smes(dev);
1438         if (ret)
1439                 goto out_free;
1440
1441         return 0;
1442
1443 out_free:
1444         if (fwspec)
1445                 kfree(fwspec->iommu_priv);
1446         iommu_fwspec_free(dev);
1447         return ret;
1448 }
1449
1450 static void arm_smmu_remove_device(struct device *dev)
1451 {
1452         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1453
1454         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1455                 return;
1456
1457         arm_smmu_master_free_smes(fwspec);
1458         iommu_group_remove_device(dev);
1459         kfree(fwspec->iommu_priv);
1460         iommu_fwspec_free(dev);
1461 }
1462
1463 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1464 {
1465         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1466         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1467         struct iommu_group *group = NULL;
1468         int i, idx;
1469
1470         for_each_cfg_sme(fwspec, i, idx) {
1471                 if (group && smmu->s2crs[idx].group &&
1472                     group != smmu->s2crs[idx].group)
1473                         return ERR_PTR(-EINVAL);
1474
1475                 group = smmu->s2crs[idx].group;
1476         }
1477
1478         if (group)
1479                 return group;
1480
1481         if (dev_is_pci(dev))
1482                 group = pci_device_group(dev);
1483         else
1484                 group = generic_device_group(dev);
1485
1486         return group;
1487 }
1488
1489 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1490                                     enum iommu_attr attr, void *data)
1491 {
1492         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1493
1494         switch (attr) {
1495         case DOMAIN_ATTR_NESTING:
1496                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1497                 return 0;
1498         default:
1499                 return -ENODEV;
1500         }
1501 }
1502
1503 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1504                                     enum iommu_attr attr, void *data)
1505 {
1506         int ret = 0;
1507         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1508
1509         mutex_lock(&smmu_domain->init_mutex);
1510
1511         switch (attr) {
1512         case DOMAIN_ATTR_NESTING:
1513                 if (smmu_domain->smmu) {
1514                         ret = -EPERM;
1515                         goto out_unlock;
1516                 }
1517
1518                 if (*(int *)data)
1519                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1520                 else
1521                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1522
1523                 break;
1524         default:
1525                 ret = -ENODEV;
1526         }
1527
1528 out_unlock:
1529         mutex_unlock(&smmu_domain->init_mutex);
1530         return ret;
1531 }
1532
1533 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1534 {
1535         u32 fwid = 0;
1536
1537         if (args->args_count > 0)
1538                 fwid |= (u16)args->args[0];
1539
1540         if (args->args_count > 1)
1541                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1542
1543         return iommu_fwspec_add_ids(dev, &fwid, 1);
1544 }
1545
1546 static struct iommu_ops arm_smmu_ops = {
1547         .capable                = arm_smmu_capable,
1548         .domain_alloc           = arm_smmu_domain_alloc,
1549         .domain_free            = arm_smmu_domain_free,
1550         .attach_dev             = arm_smmu_attach_dev,
1551         .map                    = arm_smmu_map,
1552         .unmap                  = arm_smmu_unmap,
1553         .map_sg                 = default_iommu_map_sg,
1554         .iova_to_phys           = arm_smmu_iova_to_phys,
1555         .add_device             = arm_smmu_add_device,
1556         .remove_device          = arm_smmu_remove_device,
1557         .device_group           = arm_smmu_device_group,
1558         .domain_get_attr        = arm_smmu_domain_get_attr,
1559         .domain_set_attr        = arm_smmu_domain_set_attr,
1560         .of_xlate               = arm_smmu_of_xlate,
1561         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1562 };
1563
1564 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1565 {
1566         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1567         void __iomem *cb_base;
1568         int i;
1569         u32 reg, major;
1570
1571         /* clear global FSR */
1572         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1573         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1574
1575         /*
1576          * Reset stream mapping groups: Initial values mark all SMRn as
1577          * invalid and all S2CRn as bypass unless overridden.
1578          */
1579         for (i = 0; i < smmu->num_mapping_groups; ++i)
1580                 arm_smmu_write_sme(smmu, i);
1581
1582         /*
1583          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1584          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1585          * bit is only present in MMU-500r2 onwards.
1586          */
1587         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1588         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1589         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1590                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1591                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1592                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1593         }
1594
1595         /* Make sure all context banks are disabled and clear CB_FSR  */
1596         for (i = 0; i < smmu->num_context_banks; ++i) {
1597                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1598                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1599                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1600                 /*
1601                  * Disable MMU-500's not-particularly-beneficial next-page
1602                  * prefetcher for the sake of errata #841119 and #826419.
1603                  */
1604                 if (smmu->model == ARM_MMU500) {
1605                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1606                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1607                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1608                 }
1609         }
1610
1611         /* Invalidate the TLB, just in case */
1612         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1613         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1614
1615         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1616
1617         /* Enable fault reporting */
1618         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1619
1620         /* Disable TLB broadcasting. */
1621         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1622
1623         /* Enable client access, handling unmatched streams as appropriate */
1624         reg &= ~sCR0_CLIENTPD;
1625         if (disable_bypass)
1626                 reg |= sCR0_USFCFG;
1627         else
1628                 reg &= ~sCR0_USFCFG;
1629
1630         /* Disable forced broadcasting */
1631         reg &= ~sCR0_FB;
1632
1633         /* Don't upgrade barriers */
1634         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1635
1636         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1637                 reg |= sCR0_VMID16EN;
1638
1639         /* Push the button */
1640         __arm_smmu_tlb_sync(smmu);
1641         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1642 }
1643
1644 static int arm_smmu_id_size_to_bits(int size)
1645 {
1646         switch (size) {
1647         case 0:
1648                 return 32;
1649         case 1:
1650                 return 36;
1651         case 2:
1652                 return 40;
1653         case 3:
1654                 return 42;
1655         case 4:
1656                 return 44;
1657         case 5:
1658         default:
1659                 return 48;
1660         }
1661 }
1662
1663 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1664 {
1665         unsigned long size;
1666         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1667         u32 id;
1668         bool cttw_dt, cttw_reg;
1669         int i;
1670
1671         dev_notice(smmu->dev, "probing hardware configuration...\n");
1672         dev_notice(smmu->dev, "SMMUv%d with:\n",
1673                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1674
1675         /* ID0 */
1676         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1677
1678         /* Restrict available stages based on module parameter */
1679         if (force_stage == 1)
1680                 id &= ~(ID0_S2TS | ID0_NTS);
1681         else if (force_stage == 2)
1682                 id &= ~(ID0_S1TS | ID0_NTS);
1683
1684         if (id & ID0_S1TS) {
1685                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1686                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1687         }
1688
1689         if (id & ID0_S2TS) {
1690                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1691                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1692         }
1693
1694         if (id & ID0_NTS) {
1695                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1696                 dev_notice(smmu->dev, "\tnested translation\n");
1697         }
1698
1699         if (!(smmu->features &
1700                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1701                 dev_err(smmu->dev, "\tno translation support!\n");
1702                 return -ENODEV;
1703         }
1704
1705         if ((id & ID0_S1TS) &&
1706                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1707                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1708                 dev_notice(smmu->dev, "\taddress translation ops\n");
1709         }
1710
1711         /*
1712          * In order for DMA API calls to work properly, we must defer to what
1713          * the DT says about coherency, regardless of what the hardware claims.
1714          * Fortunately, this also opens up a workaround for systems where the
1715          * ID register value has ended up configured incorrectly.
1716          */
1717         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1718         cttw_reg = !!(id & ID0_CTTW);
1719         if (cttw_dt)
1720                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1721         if (cttw_dt || cttw_reg)
1722                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1723                            cttw_dt ? "" : "non-");
1724         if (cttw_dt != cttw_reg)
1725                 dev_notice(smmu->dev,
1726                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1727
1728         /* Max. number of entries we have for stream matching/indexing */
1729         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1730         smmu->streamid_mask = size - 1;
1731         if (id & ID0_SMS) {
1732                 u32 smr;
1733
1734                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1735                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1736                 if (size == 0) {
1737                         dev_err(smmu->dev,
1738                                 "stream-matching supported, but no SMRs present!\n");
1739                         return -ENODEV;
1740                 }
1741
1742                 /*
1743                  * SMR.ID bits may not be preserved if the corresponding MASK
1744                  * bits are set, so check each one separately. We can reject
1745                  * masters later if they try to claim IDs outside these masks.
1746                  */
1747                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1748                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1749                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1750                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1751
1752                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1753                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1754                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1755                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1756
1757                 /* Zero-initialised to mark as invalid */
1758                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1759                                           GFP_KERNEL);
1760                 if (!smmu->smrs)
1761                         return -ENOMEM;
1762
1763                 dev_notice(smmu->dev,
1764                            "\tstream matching with %lu register groups, mask 0x%x",
1765                            size, smmu->smr_mask_mask);
1766         }
1767         /* s2cr->type == 0 means translation, so initialise explicitly */
1768         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1769                                          GFP_KERNEL);
1770         if (!smmu->s2crs)
1771                 return -ENOMEM;
1772         for (i = 0; i < size; i++)
1773                 smmu->s2crs[i] = s2cr_init_val;
1774
1775         smmu->num_mapping_groups = size;
1776         mutex_init(&smmu->stream_map_mutex);
1777
1778         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1779                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1780                 if (!(id & ID0_PTFS_NO_AARCH32S))
1781                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1782         }
1783
1784         /* ID1 */
1785         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1786         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1787
1788         /* Check for size mismatch of SMMU address space from mapped region */
1789         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1790         size *= 2 << smmu->pgshift;
1791         if (smmu->size != size)
1792                 dev_warn(smmu->dev,
1793                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1794                         size, smmu->size);
1795
1796         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1797         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1798         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1799                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1800                 return -ENODEV;
1801         }
1802         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1803                    smmu->num_context_banks, smmu->num_s2_context_banks);
1804         /*
1805          * Cavium CN88xx erratum #27704.
1806          * Ensure ASID and VMID allocation is unique across all SMMUs in
1807          * the system.
1808          */
1809         if (smmu->model == CAVIUM_SMMUV2) {
1810                 smmu->cavium_id_base =
1811                         atomic_add_return(smmu->num_context_banks,
1812                                           &cavium_smmu_context_count);
1813                 smmu->cavium_id_base -= smmu->num_context_banks;
1814         }
1815
1816         /* ID2 */
1817         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1818         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1819         smmu->ipa_size = size;
1820
1821         /* The output mask is also applied for bypass */
1822         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1823         smmu->pa_size = size;
1824
1825         if (id & ID2_VMID16)
1826                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1827
1828         /*
1829          * What the page table walker can address actually depends on which
1830          * descriptor format is in use, but since a) we don't know that yet,
1831          * and b) it can vary per context bank, this will have to do...
1832          */
1833         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1834                 dev_warn(smmu->dev,
1835                          "failed to set DMA mask for table walker\n");
1836
1837         if (smmu->version < ARM_SMMU_V2) {
1838                 smmu->va_size = smmu->ipa_size;
1839                 if (smmu->version == ARM_SMMU_V1_64K)
1840                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1841         } else {
1842                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1843                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1844                 if (id & ID2_PTFS_4K)
1845                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1846                 if (id & ID2_PTFS_16K)
1847                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1848                 if (id & ID2_PTFS_64K)
1849                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1850         }
1851
1852         /* Now we've corralled the various formats, what'll it do? */
1853         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1854                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1855         if (smmu->features &
1856             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1857                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1858         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1859                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1860         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1861                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1862
1863         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1864                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1865         else
1866                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1867         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1868                    smmu->pgsize_bitmap);
1869
1870
1871         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1872                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1873                            smmu->va_size, smmu->ipa_size);
1874
1875         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1876                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1877                            smmu->ipa_size, smmu->pa_size);
1878
1879         return 0;
1880 }
1881
1882 struct arm_smmu_match_data {
1883         enum arm_smmu_arch_version version;
1884         enum arm_smmu_implementation model;
1885 };
1886
1887 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1888 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1889
1890 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1891 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1892 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1893 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1894 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1895
1896 static const struct of_device_id arm_smmu_of_match[] = {
1897         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1898         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1899         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1900         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1901         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1902         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1903         { },
1904 };
1905 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1906
1907 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1908 {
1909         const struct arm_smmu_match_data *data;
1910         struct resource *res;
1911         struct arm_smmu_device *smmu;
1912         struct device *dev = &pdev->dev;
1913         int num_irqs, i, err;
1914         bool legacy_binding;
1915
1916         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1917         if (legacy_binding && !using_generic_binding) {
1918                 if (!using_legacy_binding)
1919                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1920                 using_legacy_binding = true;
1921         } else if (!legacy_binding && !using_legacy_binding) {
1922                 using_generic_binding = true;
1923         } else {
1924                 dev_err(dev, "not probing due to mismatched DT properties\n");
1925                 return -ENODEV;
1926         }
1927
1928         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1929         if (!smmu) {
1930                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1931                 return -ENOMEM;
1932         }
1933         smmu->dev = dev;
1934
1935         data = of_device_get_match_data(dev);
1936         smmu->version = data->version;
1937         smmu->model = data->model;
1938
1939         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1940         smmu->base = devm_ioremap_resource(dev, res);
1941         if (IS_ERR(smmu->base))
1942                 return PTR_ERR(smmu->base);
1943         smmu->size = resource_size(res);
1944
1945         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1946                                  &smmu->num_global_irqs)) {
1947                 dev_err(dev, "missing #global-interrupts property\n");
1948                 return -ENODEV;
1949         }
1950
1951         num_irqs = 0;
1952         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1953                 num_irqs++;
1954                 if (num_irqs > smmu->num_global_irqs)
1955                         smmu->num_context_irqs++;
1956         }
1957
1958         if (!smmu->num_context_irqs) {
1959                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1960                         num_irqs, smmu->num_global_irqs + 1);
1961                 return -ENODEV;
1962         }
1963
1964         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1965                                   GFP_KERNEL);
1966         if (!smmu->irqs) {
1967                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1968                 return -ENOMEM;
1969         }
1970
1971         for (i = 0; i < num_irqs; ++i) {
1972                 int irq = platform_get_irq(pdev, i);
1973
1974                 if (irq < 0) {
1975                         dev_err(dev, "failed to get irq index %d\n", i);
1976                         return -ENODEV;
1977                 }
1978                 smmu->irqs[i] = irq;
1979         }
1980
1981         err = arm_smmu_device_cfg_probe(smmu);
1982         if (err)
1983                 return err;
1984
1985         parse_driver_options(smmu);
1986
1987         if (smmu->version == ARM_SMMU_V2 &&
1988             smmu->num_context_banks != smmu->num_context_irqs) {
1989                 dev_err(dev,
1990                         "found only %d context interrupt(s) but %d required\n",
1991                         smmu->num_context_irqs, smmu->num_context_banks);
1992                 return -ENODEV;
1993         }
1994
1995         for (i = 0; i < smmu->num_global_irqs; ++i) {
1996                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1997                                        arm_smmu_global_fault,
1998                                        IRQF_SHARED,
1999                                        "arm-smmu global fault",
2000                                        smmu);
2001                 if (err) {
2002                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2003                                 i, smmu->irqs[i]);
2004                         return err;
2005                 }
2006         }
2007
2008         of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
2009         platform_set_drvdata(pdev, smmu);
2010         arm_smmu_device_reset(smmu);
2011
2012         /* Oh, for a proper bus abstraction */
2013         if (!iommu_present(&platform_bus_type))
2014                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2015 #ifdef CONFIG_ARM_AMBA
2016         if (!iommu_present(&amba_bustype))
2017                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2018 #endif
2019 #ifdef CONFIG_PCI
2020         if (!iommu_present(&pci_bus_type)) {
2021                 pci_request_acs();
2022                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2023         }
2024 #endif
2025         return 0;
2026 }
2027
2028 static int arm_smmu_device_remove(struct platform_device *pdev)
2029 {
2030         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2031
2032         if (!smmu)
2033                 return -ENODEV;
2034
2035         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2036                 dev_err(&pdev->dev, "removing device with active domains!\n");
2037
2038         /* Turn the thing off */
2039         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2040         return 0;
2041 }
2042
2043 static struct platform_driver arm_smmu_driver = {
2044         .driver = {
2045                 .name           = "arm-smmu",
2046                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2047         },
2048         .probe  = arm_smmu_device_dt_probe,
2049         .remove = arm_smmu_device_remove,
2050 };
2051
2052 static int __init arm_smmu_init(void)
2053 {
2054         static bool registered;
2055         int ret = 0;
2056
2057         if (!registered) {
2058                 ret = platform_driver_register(&arm_smmu_driver);
2059                 registered = !ret;
2060         }
2061         return ret;
2062 }
2063
2064 static void __exit arm_smmu_exit(void)
2065 {
2066         return platform_driver_unregister(&arm_smmu_driver);
2067 }
2068
2069 subsys_initcall(arm_smmu_init);
2070 module_exit(arm_smmu_exit);
2071
2072 static int __init arm_smmu_of_init(struct device_node *np)
2073 {
2074         int ret = arm_smmu_init();
2075
2076         if (ret)
2077                 return ret;
2078
2079         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2080                 return -ENODEV;
2081
2082         return 0;
2083 }
2084 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2085 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2086 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2087 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2088 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2089 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2090
2091 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2092 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2093 MODULE_LICENSE("GPL v2");