drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/iommu.h>
  43 #include <linux/iopoll.h>
  44 #include <linux/module.h>
  45 #include <linux/of.h>
  46 #include <linux/of_address.h>
  47 #include <linux/of_device.h>
  48 #include <linux/of_iommu.h>
  49 #include <linux/pci.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52 #include <linux/spinlock.h>
  53
  54 #include <linux/amba/bus.h>
  55
  56 #include "io-pgtable.h"
  57
  58 /* Maximum number of context banks per SMMU */
  59 #define ARM_SMMU_MAX_CBS                128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_EXIDENABLE                 (1 << 3)
  92 #define sCR0_GCFGFRE                    (1 << 4)
  93 #define sCR0_GCFGFIE                    (1 << 5)
  94 #define sCR0_USFCFG                     (1 << 10)
  95 #define sCR0_VMIDPNE                    (1 << 11)
  96 #define sCR0_PTM                        (1 << 12)
  97 #define sCR0_FB                         (1 << 13)
  98 #define sCR0_VMID16EN                   (1 << 31)
  99 #define sCR0_BSU_SHIFT                  14
 100 #define sCR0_BSU_MASK                   0x3
 101
 102 /* Auxiliary Configuration register */
 103 #define ARM_SMMU_GR0_sACR               0x10
 104
 105 /* Identification registers */
 106 #define ARM_SMMU_GR0_ID0                0x20
 107 #define ARM_SMMU_GR0_ID1                0x24
 108 #define ARM_SMMU_GR0_ID2                0x28
 109 #define ARM_SMMU_GR0_ID3                0x2c
 110 #define ARM_SMMU_GR0_ID4                0x30
 111 #define ARM_SMMU_GR0_ID5                0x34
 112 #define ARM_SMMU_GR0_ID6                0x38
 113 #define ARM_SMMU_GR0_ID7                0x3c
 114 #define ARM_SMMU_GR0_sGFSR              0x48
 115 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 116 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 117 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 118
 119 #define ID0_S1TS                        (1 << 30)
 120 #define ID0_S2TS                        (1 << 29)
 121 #define ID0_NTS                         (1 << 28)
 122 #define ID0_SMS                         (1 << 27)
 123 #define ID0_ATOSNS                      (1 << 26)
 124 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 125 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 126 #define ID0_CTTW                        (1 << 14)
 127 #define ID0_NUMIRPT_SHIFT               16
 128 #define ID0_NUMIRPT_MASK                0xff
 129 #define ID0_NUMSIDB_SHIFT               9
 130 #define ID0_NUMSIDB_MASK                0xf
 131 #define ID0_EXIDS                       (1 << 8)
 132 #define ID0_NUMSMRG_SHIFT               0
 133 #define ID0_NUMSMRG_MASK                0xff
 134
 135 #define ID1_PAGESIZE                    (1 << 31)
 136 #define ID1_NUMPAGENDXB_SHIFT           28
 137 #define ID1_NUMPAGENDXB_MASK            7
 138 #define ID1_NUMS2CB_SHIFT               16
 139 #define ID1_NUMS2CB_MASK                0xff
 140 #define ID1_NUMCB_SHIFT                 0
 141 #define ID1_NUMCB_MASK                  0xff
 142
 143 #define ID2_OAS_SHIFT                   4
 144 #define ID2_OAS_MASK                    0xf
 145 #define ID2_IAS_SHIFT                   0
 146 #define ID2_IAS_MASK                    0xf
 147 #define ID2_UBS_SHIFT                   8
 148 #define ID2_UBS_MASK                    0xf
 149 #define ID2_PTFS_4K                     (1 << 12)
 150 #define ID2_PTFS_16K                    (1 << 13)
 151 #define ID2_PTFS_64K                    (1 << 14)
 152 #define ID2_VMID16                      (1 << 15)
 153
 154 #define ID7_MAJOR_SHIFT                 4
 155 #define ID7_MAJOR_MASK                  0xf
 156
 157 /* Global TLB invalidation */
 158 #define ARM_SMMU_GR0_TLBIVMID           0x64
 159 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 160 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 161 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 162 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 163 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 164 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 165 #define TLB_SPIN_COUNT                  10
 166
 167 /* Stream mapping registers */
 168 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 169 #define SMR_VALID                       (1 << 31)
 170 #define SMR_MASK_SHIFT                  16
 171 #define SMR_ID_SHIFT                    0
 172
 173 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 174 #define S2CR_CBNDX_SHIFT                0
 175 #define S2CR_CBNDX_MASK                 0xff
 176 #define S2CR_EXIDVALID                  (1 << 10)
 177 #define S2CR_TYPE_SHIFT                 16
 178 #define S2CR_TYPE_MASK                  0x3
 179 enum arm_smmu_s2cr_type {
 180         S2CR_TYPE_TRANS,
 181         S2CR_TYPE_BYPASS,
 182         S2CR_TYPE_FAULT,
 183 };
 184
 185 #define S2CR_PRIVCFG_SHIFT              24
 186 #define S2CR_PRIVCFG_MASK               0x3
 187 enum arm_smmu_s2cr_privcfg {
 188         S2CR_PRIVCFG_DEFAULT,
 189         S2CR_PRIVCFG_DIPAN,
 190         S2CR_PRIVCFG_UNPRIV,
 191         S2CR_PRIVCFG_PRIV,
 192 };
 193
 194 /* Context bank attribute registers */
 195 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 196 #define CBAR_VMID_SHIFT                 0
 197 #define CBAR_VMID_MASK                  0xff
 198 #define CBAR_S1_BPSHCFG_SHIFT           8
 199 #define CBAR_S1_BPSHCFG_MASK            3
 200 #define CBAR_S1_BPSHCFG_NSH             3
 201 #define CBAR_S1_MEMATTR_SHIFT           12
 202 #define CBAR_S1_MEMATTR_MASK            0xf
 203 #define CBAR_S1_MEMATTR_WB              0xf
 204 #define CBAR_TYPE_SHIFT                 16
 205 #define CBAR_TYPE_MASK                  0x3
 206 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 207 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 208 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 209 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 210 #define CBAR_IRPTNDX_SHIFT              24
 211 #define CBAR_IRPTNDX_MASK               0xff
 212
 213 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 214 #define CBA2R_RW64_32BIT                (0 << 0)
 215 #define CBA2R_RW64_64BIT                (1 << 0)
 216 #define CBA2R_VMID_SHIFT                16
 217 #define CBA2R_VMID_MASK                 0xffff
 218
 219 /* Translation context bank */
 220 #define ARM_SMMU_CB(smmu, n)    ((smmu)->cb_base + ((n) << (smmu)->pgshift))
 221
 222 #define ARM_SMMU_CB_SCTLR               0x0
 223 #define ARM_SMMU_CB_ACTLR               0x4
 224 #define ARM_SMMU_CB_RESUME              0x8
 225 #define ARM_SMMU_CB_TTBCR2              0x10
 226 #define ARM_SMMU_CB_TTBR0               0x20
 227 #define ARM_SMMU_CB_TTBR1               0x28
 228 #define ARM_SMMU_CB_TTBCR               0x30
 229 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 230 #define ARM_SMMU_CB_S1_MAIR0            0x38
 231 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 232 #define ARM_SMMU_CB_PAR                 0x50
 233 #define ARM_SMMU_CB_FSR                 0x58
 234 #define ARM_SMMU_CB_FAR                 0x60
 235 #define ARM_SMMU_CB_FSYNR0              0x68
 236 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 237 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 238 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 239 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 240 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 241 #define ARM_SMMU_CB_TLBSYNC             0x7f0
 242 #define ARM_SMMU_CB_TLBSTATUS           0x7f4
 243 #define ARM_SMMU_CB_ATS1PR              0x800
 244 #define ARM_SMMU_CB_ATSR                0x8f0
 245
 246 #define SCTLR_S1_ASIDPNE                (1 << 12)
 247 #define SCTLR_CFCFG                     (1 << 7)
 248 #define SCTLR_CFIE                      (1 << 6)
 249 #define SCTLR_CFRE                      (1 << 5)
 250 #define SCTLR_E                         (1 << 4)
 251 #define SCTLR_AFE                       (1 << 2)
 252 #define SCTLR_TRE                       (1 << 1)
 253 #define SCTLR_M                         (1 << 0)
 254
 255 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 256
 257 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 258 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 259
 260 #define CB_PAR_F                        (1 << 0)
 261
 262 #define ATSR_ACTIVE                     (1 << 0)
 263
 264 #define RESUME_RETRY                    (0 << 0)
 265 #define RESUME_TERMINATE                (1 << 0)
 266
 267 #define TTBCR2_SEP_SHIFT                15
 268 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 269 #define TTBCR2_AS                       (1 << 4)
 270
 271 #define TTBRn_ASID_SHIFT                48
 272
 273 #define FSR_MULTI                       (1 << 31)
 274 #define FSR_SS                          (1 << 30)
 275 #define FSR_UUT                         (1 << 8)
 276 #define FSR_ASF                         (1 << 7)
 277 #define FSR_TLBLKF                      (1 << 6)
 278 #define FSR_TLBMCF                      (1 << 5)
 279 #define FSR_EF                          (1 << 4)
 280 #define FSR_PF                          (1 << 3)
 281 #define FSR_AFF                         (1 << 2)
 282 #define FSR_TF                          (1 << 1)
 283
 284 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 285                                          FSR_TLBMCF | FSR_TLBLKF)
 286 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 287                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 288
 289 #define FSYNR0_WNR                      (1 << 4)
 290
 291 #define MSI_IOVA_BASE                   0x8000000
 292 #define MSI_IOVA_LENGTH                 0x100000
 293
 294 static int force_stage;
 295 module_param(force_stage, int, S_IRUGO);
 296 MODULE_PARM_DESC(force_stage,
 297         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 298 static bool disable_bypass;
 299 module_param(disable_bypass, bool, S_IRUGO);
 300 MODULE_PARM_DESC(disable_bypass,
 301         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 302
 303 enum arm_smmu_arch_version {
 304         ARM_SMMU_V1,
 305         ARM_SMMU_V1_64K,
 306         ARM_SMMU_V2,
 307 };
 308
 309 enum arm_smmu_implementation {
 310         GENERIC_SMMU,
 311         ARM_MMU500,
 312         CAVIUM_SMMUV2,
 313 };
 314
 315 /* Until ACPICA headers cover IORT rev. C */
 316 #ifndef ACPI_IORT_SMMU_CORELINK_MMU401
 317 #define ACPI_IORT_SMMU_CORELINK_MMU401  0x4
 318 #endif
 319 #ifndef ACPI_IORT_SMMU_CAVIUM_THUNDERX
 320 #define ACPI_IORT_SMMU_CAVIUM_THUNDERX  0x5
 321 #endif
 322
 323 struct arm_smmu_s2cr {
 324         struct iommu_group              *group;
 325         int                             count;
 326         enum arm_smmu_s2cr_type         type;
 327         enum arm_smmu_s2cr_privcfg      privcfg;
 328         u8                              cbndx;
 329 };
 330
 331 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 332         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 333 }
 334
 335 struct arm_smmu_smr {
 336         u16                             mask;
 337         u16                             id;
 338         bool                            valid;
 339 };
 340
 341 struct arm_smmu_master_cfg {
 342         struct arm_smmu_device          *smmu;
 343         s16                             smendx[];
 344 };
 345 #define INVALID_SMENDX                  -1
 346 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 347 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 348 #define fwspec_smendx(fw, i) \
 349         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 350 #define for_each_cfg_sme(fw, i, idx) \
 351         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 352
 353 struct arm_smmu_device {
 354         struct device                   *dev;
 355
 356         void __iomem                    *base;
 357         void __iomem                    *cb_base;
 358         unsigned long                   pgshift;
 359
 360 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 361 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 362 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 363 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 364 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 365 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 366 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 367 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 368 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 369 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 370 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 371 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 372 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 373         u32                             features;
 374
 375 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 376         u32                             options;
 377         enum arm_smmu_arch_version      version;
 378         enum arm_smmu_implementation    model;
 379
 380         u32                             num_context_banks;
 381         u32                             num_s2_context_banks;
 382         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 383         atomic_t                        irptndx;
 384
 385         u32                             num_mapping_groups;
 386         u16                             streamid_mask;
 387         u16                             smr_mask_mask;
 388         struct arm_smmu_smr             *smrs;
 389         struct arm_smmu_s2cr            *s2crs;
 390         struct mutex                    stream_map_mutex;
 391
 392         unsigned long                   va_size;
 393         unsigned long                   ipa_size;
 394         unsigned long                   pa_size;
 395         unsigned long                   pgsize_bitmap;
 396
 397         u32                             num_global_irqs;
 398         u32                             num_context_irqs;
 399         unsigned int                    *irqs;
 400
 401         u32                             cavium_id_base; /* Specific to Cavium */
 402
 403         /* IOMMU core code handle */
 404         struct iommu_device             iommu;
 405 };
 406
 407 enum arm_smmu_context_fmt {
 408         ARM_SMMU_CTX_FMT_NONE,
 409         ARM_SMMU_CTX_FMT_AARCH64,
 410         ARM_SMMU_CTX_FMT_AARCH32_L,
 411         ARM_SMMU_CTX_FMT_AARCH32_S,
 412 };
 413
 414 struct arm_smmu_cfg {
 415         u8                              cbndx;
 416         u8                              irptndx;
 417         union {
 418                 u16                     asid;
 419                 u16                     vmid;
 420         };
 421         u32                             cbar;
 422         enum arm_smmu_context_fmt       fmt;
 423 };
 424 #define INVALID_IRPTNDX                 0xff
 425
 426 enum arm_smmu_domain_stage {
 427         ARM_SMMU_DOMAIN_S1 = 0,
 428         ARM_SMMU_DOMAIN_S2,
 429         ARM_SMMU_DOMAIN_NESTED,
 430         ARM_SMMU_DOMAIN_BYPASS,
 431 };
 432
 433 struct arm_smmu_domain {
 434         struct arm_smmu_device          *smmu;
 435         struct io_pgtable_ops           *pgtbl_ops;
 436         spinlock_t                      pgtbl_lock;
 437         struct arm_smmu_cfg             cfg;
 438         enum arm_smmu_domain_stage      stage;
 439         struct mutex                    init_mutex; /* Protects smmu pointer */
 440         struct iommu_domain             domain;
 441 };
 442
 443 struct arm_smmu_option_prop {
 444         u32 opt;
 445         const char *prop;
 446 };
 447
 448 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 449
 450 static bool using_legacy_binding, using_generic_binding;
 451
 452 static struct arm_smmu_option_prop arm_smmu_options[] = {
 453         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 454         { 0, NULL},
 455 };
 456
 457 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 458 {
 459         return container_of(dom, struct arm_smmu_domain, domain);
 460 }
 461
 462 static void parse_driver_options(struct arm_smmu_device *smmu)
 463 {
 464         int i = 0;
 465
 466         do {
 467                 if (of_property_read_bool(smmu->dev->of_node,
 468                                                 arm_smmu_options[i].prop)) {
 469                         smmu->options |= arm_smmu_options[i].opt;
 470                         dev_notice(smmu->dev, "option %s\n",
 471                                 arm_smmu_options[i].prop);
 472                 }
 473         } while (arm_smmu_options[++i].opt);
 474 }
 475
 476 static struct device_node *dev_get_dev_node(struct device *dev)
 477 {
 478         if (dev_is_pci(dev)) {
 479                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 480
 481                 while (!pci_is_root_bus(bus))
 482                         bus = bus->parent;
 483                 return of_node_get(bus->bridge->parent->of_node);
 484         }
 485
 486         return of_node_get(dev->of_node);
 487 }
 488
 489 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 490 {
 491         *((__be32 *)data) = cpu_to_be32(alias);
 492         return 0; /* Continue walking */
 493 }
 494
 495 static int __find_legacy_master_phandle(struct device *dev, void *data)
 496 {
 497         struct of_phandle_iterator *it = *(void **)data;
 498         struct device_node *np = it->node;
 499         int err;
 500
 501         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 502                             "#stream-id-cells", 0)
 503                 if (it->node == np) {
 504                         *(void **)data = dev;
 505                         return 1;
 506                 }
 507         it->node = np;
 508         return err == -ENOENT ? 0 : err;
 509 }
 510
 511 static struct platform_driver arm_smmu_driver;
 512 static struct iommu_ops arm_smmu_ops;
 513
 514 static int arm_smmu_register_legacy_master(struct device *dev,
 515                                            struct arm_smmu_device **smmu)
 516 {
 517         struct device *smmu_dev;
 518         struct device_node *np;
 519         struct of_phandle_iterator it;
 520         void *data = &it;
 521         u32 *sids;
 522         __be32 pci_sid;
 523         int err;
 524
 525         np = dev_get_dev_node(dev);
 526         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 527                 of_node_put(np);
 528                 return -ENODEV;
 529         }
 530
 531         it.node = np;
 532         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 533                                      __find_legacy_master_phandle);
 534         smmu_dev = data;
 535         of_node_put(np);
 536         if (err == 0)
 537                 return -ENODEV;
 538         if (err < 0)
 539                 return err;
 540
 541         if (dev_is_pci(dev)) {
 542                 /* "mmu-masters" assumes Stream ID == Requester ID */
 543                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 544                                        &pci_sid);
 545                 it.cur = &pci_sid;
 546                 it.cur_count = 1;
 547         }
 548
 549         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 550                                 &arm_smmu_ops);
 551         if (err)
 552                 return err;
 553
 554         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 555         if (!sids)
 556                 return -ENOMEM;
 557
 558         *smmu = dev_get_drvdata(smmu_dev);
 559         of_phandle_iterator_args(&it, sids, it.cur_count);
 560         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 561         kfree(sids);
 562         return err;
 563 }
 564
 565 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 566 {
 567         int idx;
 568
 569         do {
 570                 idx = find_next_zero_bit(map, end, start);
 571                 if (idx == end)
 572                         return -ENOSPC;
 573         } while (test_and_set_bit(idx, map));
 574
 575         return idx;
 576 }
 577
 578 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 579 {
 580         clear_bit(idx, map);
 581 }
 582
 583 /* Wait for any pending TLB invalidations to complete */
 584 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
 585                                 void __iomem *sync, void __iomem *status)
 586 {
 587         unsigned int spin_cnt, delay;
 588
 589         writel_relaxed(0, sync);
 590         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 591                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 592                         if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
 593                                 return;
 594                         cpu_relax();
 595                 }
 596                 udelay(delay);
 597         }
 598         dev_err_ratelimited(smmu->dev,
 599                             "TLB sync timed out -- SMMU may be deadlocked\n");
 600 }
 601
 602 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 603 {
 604         void __iomem *base = ARM_SMMU_GR0(smmu);
 605
 606         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
 607                             base + ARM_SMMU_GR0_sTLBGSTATUS);
 608 }
 609
 610 static void arm_smmu_tlb_sync_context(void *cookie)
 611 {
 612         struct arm_smmu_domain *smmu_domain = cookie;
 613         struct arm_smmu_device *smmu = smmu_domain->smmu;
 614         void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 615
 616         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
 617                             base + ARM_SMMU_CB_TLBSTATUS);
 618 }
 619
 620 static void arm_smmu_tlb_sync_vmid(void *cookie)
 621 {
 622         struct arm_smmu_domain *smmu_domain = cookie;
 623
 624         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 625 }
 626
 627 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 628 {
 629         struct arm_smmu_domain *smmu_domain = cookie;
 630         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 631         void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 632
 633         writel_relaxed(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
 634         arm_smmu_tlb_sync_context(cookie);
 635 }
 636
 637 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 638 {
 639         struct arm_smmu_domain *smmu_domain = cookie;
 640         struct arm_smmu_device *smmu = smmu_domain->smmu;
 641         void __iomem *base = ARM_SMMU_GR0(smmu);
 642
 643         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 644         arm_smmu_tlb_sync_global(smmu);
 645 }
 646
 647 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 648                                           size_t granule, bool leaf, void *cookie)
 649 {
 650         struct arm_smmu_domain *smmu_domain = cookie;
 651         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 652         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 653         void __iomem *reg = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 654
 655         if (stage1) {
 656                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 657
 658                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 659                         iova &= ~12UL;
 660                         iova |= cfg->asid;
 661                         do {
 662                                 writel_relaxed(iova, reg);
 663                                 iova += granule;
 664                         } while (size -= granule);
 665                 } else {
 666                         iova >>= 12;
 667                         iova |= (u64)cfg->asid << 48;
 668                         do {
 669                                 writeq_relaxed(iova, reg);
 670                                 iova += granule >> 12;
 671                         } while (size -= granule);
 672                 }
 673         } else {
 674                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 675                               ARM_SMMU_CB_S2_TLBIIPAS2;
 676                 iova >>= 12;
 677                 do {
 678                         smmu_write_atomic_lq(iova, reg);
 679                         iova += granule >> 12;
 680                 } while (size -= granule);
 681         }
 682 }
 683
 684 /*
 685  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 686  * almost negligible, but the benefit of getting the first one in as far ahead
 687  * of the sync as possible is significant, hence we don't just make this a
 688  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 689  */
 690 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 691                                          size_t granule, bool leaf, void *cookie)
 692 {
 693         struct arm_smmu_domain *smmu_domain = cookie;
 694         void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
 695
 696         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 697 }
 698
 699 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 700         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 701         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 702         .tlb_sync       = arm_smmu_tlb_sync_context,
 703 };
 704
 705 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 706         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 707         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 708         .tlb_sync       = arm_smmu_tlb_sync_context,
 709 };
 710
 711 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 712         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 713         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 714         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 715 };
 716
 717 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 718 {
 719         u32 fsr, fsynr;
 720         unsigned long iova;
 721         struct iommu_domain *domain = dev;
 722         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 723         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 724         struct arm_smmu_device *smmu = smmu_domain->smmu;
 725         void __iomem *cb_base;
 726
 727         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 728         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 729
 730         if (!(fsr & FSR_FAULT))
 731                 return IRQ_NONE;
 732
 733         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 734         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 735
 736         dev_err_ratelimited(smmu->dev,
 737         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 738                             fsr, iova, fsynr, cfg->cbndx);
 739
 740         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 741         return IRQ_HANDLED;
 742 }
 743
 744 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 745 {
 746         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 747         struct arm_smmu_device *smmu = dev;
 748         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 749
 750         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 751         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 752         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 753         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 754
 755         if (!gfsr)
 756                 return IRQ_NONE;
 757
 758         dev_err_ratelimited(smmu->dev,
 759                 "Unexpected global fault, this could be serious\n");
 760         dev_err_ratelimited(smmu->dev,
 761                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 762                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 763
 764         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 765         return IRQ_HANDLED;
 766 }
 767
 768 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 769                                        struct io_pgtable_cfg *pgtbl_cfg)
 770 {
 771         u32 reg, reg2;
 772         u64 reg64;
 773         bool stage1;
 774         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 775         struct arm_smmu_device *smmu = smmu_domain->smmu;
 776         void __iomem *cb_base, *gr1_base;
 777
 778         gr1_base = ARM_SMMU_GR1(smmu);
 779         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 780         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 781
 782         if (smmu->version > ARM_SMMU_V1) {
 783                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 784                         reg = CBA2R_RW64_64BIT;
 785                 else
 786                         reg = CBA2R_RW64_32BIT;
 787                 /* 16-bit VMIDs live in CBA2R */
 788                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 789                         reg |= cfg->vmid << CBA2R_VMID_SHIFT;
 790
 791                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 792         }
 793
 794         /* CBAR */
 795         reg = cfg->cbar;
 796         if (smmu->version < ARM_SMMU_V2)
 797                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 798
 799         /*
 800          * Use the weakest shareability/memory types, so they are
 801          * overridden by the ttbcr/pte.
 802          */
 803         if (stage1) {
 804                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 805                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 806         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 807                 /* 8-bit VMIDs live in CBAR */
 808                 reg |= cfg->vmid << CBAR_VMID_SHIFT;
 809         }
 810         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 811
 812         /*
 813          * TTBCR
 814          * We must write this before the TTBRs, since it determines the
 815          * access behaviour of some fields (in particular, ASID[15:8]).
 816          */
 817         if (stage1) {
 818                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 819                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 820                         reg2 = 0;
 821                 } else {
 822                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 823                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 824                         reg2 |= TTBCR2_SEP_UPSTREAM;
 825                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 826                                 reg2 |= TTBCR2_AS;
 827                 }
 828                 if (smmu->version > ARM_SMMU_V1)
 829                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 830         } else {
 831                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 832         }
 833         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 834
 835         /* TTBRs */
 836         if (stage1) {
 837                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 838                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 839                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 840                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 841                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 842                         writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 843                 } else {
 844                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 845                         reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 846                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 847                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 848                         reg64 |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 849                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 850                 }
 851         } else {
 852                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 853                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 854         }
 855
 856         /* MAIRs (stage-1 only) */
 857         if (stage1) {
 858                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 859                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 860                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 861                 } else {
 862                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 863                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 864                 }
 865                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 866                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 867         }
 868
 869         /* SCTLR */
 870         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 871         if (stage1)
 872                 reg |= SCTLR_S1_ASIDPNE;
 873 #ifdef __BIG_ENDIAN
 874         reg |= SCTLR_E;
 875 #endif
 876         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 877 }
 878
 879 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 880                                         struct arm_smmu_device *smmu)
 881 {
 882         int irq, start, ret = 0;
 883         unsigned long ias, oas;
 884         struct io_pgtable_ops *pgtbl_ops;
 885         struct io_pgtable_cfg pgtbl_cfg;
 886         enum io_pgtable_fmt fmt;
 887         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 888         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 889         const struct iommu_gather_ops *tlb_ops;
 890
 891         mutex_lock(&smmu_domain->init_mutex);
 892         if (smmu_domain->smmu)
 893                 goto out_unlock;
 894
 895         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 896                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 897                 smmu_domain->smmu = smmu;
 898                 goto out_unlock;
 899         }
 900
 901         /*
 902          * Mapping the requested stage onto what we support is surprisingly
 903          * complicated, mainly because the spec allows S1+S2 SMMUs without
 904          * support for nested translation. That means we end up with the
 905          * following table:
 906          *
 907          * Requested        Supported        Actual
 908          *     S1               N              S1
 909          *     S1             S1+S2            S1
 910          *     S1               S2             S2
 911          *     S1               S1             S1
 912          *     N                N              N
 913          *     N              S1+S2            S2
 914          *     N                S2             S2
 915          *     N                S1             S1
 916          *
 917          * Note that you can't actually request stage-2 mappings.
 918          */
 919         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 920                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 921         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 922                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 923
 924         /*
 925          * Choosing a suitable context format is even more fiddly. Until we
 926          * grow some way for the caller to express a preference, and/or move
 927          * the decision into the io-pgtable code where it arguably belongs,
 928          * just aim for the closest thing to the rest of the system, and hope
 929          * that the hardware isn't esoteric enough that we can't assume AArch64
 930          * support to be a superset of AArch32 support...
 931          */
 932         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 933                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 934         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 935             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 936             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 937             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 938                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 939         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 940             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 941                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 942                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 943                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 944
 945         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 946                 ret = -EINVAL;
 947                 goto out_unlock;
 948         }
 949
 950         switch (smmu_domain->stage) {
 951         case ARM_SMMU_DOMAIN_S1:
 952                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 953                 start = smmu->num_s2_context_banks;
 954                 ias = smmu->va_size;
 955                 oas = smmu->ipa_size;
 956                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 957                         fmt = ARM_64_LPAE_S1;
 958                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 959                         fmt = ARM_32_LPAE_S1;
 960                         ias = min(ias, 32UL);
 961                         oas = min(oas, 40UL);
 962                 } else {
 963                         fmt = ARM_V7S;
 964                         ias = min(ias, 32UL);
 965                         oas = min(oas, 32UL);
 966                 }
 967                 tlb_ops = &arm_smmu_s1_tlb_ops;
 968                 break;
 969         case ARM_SMMU_DOMAIN_NESTED:
 970                 /*
 971                  * We will likely want to change this if/when KVM gets
 972                  * involved.
 973                  */
 974         case ARM_SMMU_DOMAIN_S2:
 975                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 976                 start = 0;
 977                 ias = smmu->ipa_size;
 978                 oas = smmu->pa_size;
 979                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 980                         fmt = ARM_64_LPAE_S2;
 981                 } else {
 982                         fmt = ARM_32_LPAE_S2;
 983                         ias = min(ias, 40UL);
 984                         oas = min(oas, 40UL);
 985                 }
 986                 if (smmu->version == ARM_SMMU_V2)
 987                         tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 988                 else
 989                         tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 990                 break;
 991         default:
 992                 ret = -EINVAL;
 993                 goto out_unlock;
 994         }
 995         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 996                                       smmu->num_context_banks);
 997         if (ret < 0)
 998                 goto out_unlock;
 999
1000         cfg->cbndx = ret;
1001         if (smmu->version < ARM_SMMU_V2) {
1002                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
1003                 cfg->irptndx %= smmu->num_context_irqs;
1004         } else {
1005                 cfg->irptndx = cfg->cbndx;
1006         }
1007
1008         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
1009                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
1010         else
1011                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
1012
1013         pgtbl_cfg = (struct io_pgtable_cfg) {
1014                 .pgsize_bitmap  = smmu->pgsize_bitmap,
1015                 .ias            = ias,
1016                 .oas            = oas,
1017                 .tlb            = tlb_ops,
1018                 .iommu_dev      = smmu->dev,
1019         };
1020
1021         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
1022                 pgtbl_cfg.quirks = IO_PGTABLE_QUIRK_NO_DMA;
1023
1024         smmu_domain->smmu = smmu;
1025         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
1026         if (!pgtbl_ops) {
1027                 ret = -ENOMEM;
1028                 goto out_clear_smmu;
1029         }
1030
1031         /* Update the domain's page sizes to reflect the page table format */
1032         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
1033         domain->geometry.aperture_end = (1UL << ias) - 1;
1034         domain->geometry.force_aperture = true;
1035
1036         /* Initialise the context bank with our page table cfg */
1037         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
1038
1039         /*
1040          * Request context fault interrupt. Do this last to avoid the
1041          * handler seeing a half-initialised domain state.
1042          */
1043         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1044         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
1045                                IRQF_SHARED, "arm-smmu-context-fault", domain);
1046         if (ret < 0) {
1047                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
1048                         cfg->irptndx, irq);
1049                 cfg->irptndx = INVALID_IRPTNDX;
1050         }
1051
1052         mutex_unlock(&smmu_domain->init_mutex);
1053
1054         /* Publish page table ops for map/unmap */
1055         smmu_domain->pgtbl_ops = pgtbl_ops;
1056         return 0;
1057
1058 out_clear_smmu:
1059         smmu_domain->smmu = NULL;
1060 out_unlock:
1061         mutex_unlock(&smmu_domain->init_mutex);
1062         return ret;
1063 }
1064
1065 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1066 {
1067         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1068         struct arm_smmu_device *smmu = smmu_domain->smmu;
1069         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1070         void __iomem *cb_base;
1071         int irq;
1072
1073         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
1074                 return;
1075
1076         /*
1077          * Disable the context bank and free the page tables before freeing
1078          * it.
1079          */
1080         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1081         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1082
1083         if (cfg->irptndx != INVALID_IRPTNDX) {
1084                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1085                 devm_free_irq(smmu->dev, irq, domain);
1086         }
1087
1088         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1089         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1090 }
1091
1092 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1093 {
1094         struct arm_smmu_domain *smmu_domain;
1095
1096         if (type != IOMMU_DOMAIN_UNMANAGED &&
1097             type != IOMMU_DOMAIN_DMA &&
1098             type != IOMMU_DOMAIN_IDENTITY)
1099                 return NULL;
1100         /*
1101          * Allocate the domain and initialise some of its data structures.
1102          * We can't really do anything meaningful until we've added a
1103          * master.
1104          */
1105         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1106         if (!smmu_domain)
1107                 return NULL;
1108
1109         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1110             iommu_get_dma_cookie(&smmu_domain->domain))) {
1111                 kfree(smmu_domain);
1112                 return NULL;
1113         }
1114
1115         mutex_init(&smmu_domain->init_mutex);
1116         spin_lock_init(&smmu_domain->pgtbl_lock);
1117
1118         return &smmu_domain->domain;
1119 }
1120
1121 static void arm_smmu_domain_free(struct iommu_domain *domain)
1122 {
1123         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1124
1125         /*
1126          * Free the domain resources. We assume that all devices have
1127          * already been detached.
1128          */
1129         iommu_put_dma_cookie(domain);
1130         arm_smmu_destroy_domain_context(domain);
1131         kfree(smmu_domain);
1132 }
1133
1134 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1135 {
1136         struct arm_smmu_smr *smr = smmu->smrs + idx;
1137         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1138
1139         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1140                 reg |= SMR_VALID;
1141         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1142 }
1143
1144 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1145 {
1146         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1147         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1148                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1149                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1150
1151         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1152             smmu->smrs[idx].valid)
1153                 reg |= S2CR_EXIDVALID;
1154         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1155 }
1156
1157 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1158 {
1159         arm_smmu_write_s2cr(smmu, idx);
1160         if (smmu->smrs)
1161                 arm_smmu_write_smr(smmu, idx);
1162 }
1163
1164 /*
1165  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1166  * should be called after sCR0 is written.
1167  */
1168 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1169 {
1170         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1171         u32 smr;
1172
1173         if (!smmu->smrs)
1174                 return;
1175
1176         /*
1177          * SMR.ID bits may not be preserved if the corresponding MASK
1178          * bits are set, so check each one separately. We can reject
1179          * masters later if they try to claim IDs outside these masks.
1180          */
1181         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1182         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1183         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1184         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1185
1186         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1187         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1188         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1189         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1190 }
1191
1192 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1193 {
1194         struct arm_smmu_smr *smrs = smmu->smrs;
1195         int i, free_idx = -ENOSPC;
1196
1197         /* Stream indexing is blissfully easy */
1198         if (!smrs)
1199                 return id;
1200
1201         /* Validating SMRs is... less so */
1202         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1203                 if (!smrs[i].valid) {
1204                         /*
1205                          * Note the first free entry we come across, which
1206                          * we'll claim in the end if nothing else matches.
1207                          */
1208                         if (free_idx < 0)
1209                                 free_idx = i;
1210                         continue;
1211                 }
1212                 /*
1213                  * If the new entry is _entirely_ matched by an existing entry,
1214                  * then reuse that, with the guarantee that there also cannot
1215                  * be any subsequent conflicting entries. In normal use we'd
1216                  * expect simply identical entries for this case, but there's
1217                  * no harm in accommodating the generalisation.
1218                  */
1219                 if ((mask & smrs[i].mask) == mask &&
1220                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1221                         return i;
1222                 /*
1223                  * If the new entry has any other overlap with an existing one,
1224                  * though, then there always exists at least one stream ID
1225                  * which would cause a conflict, and we can't allow that risk.
1226                  */
1227                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1228                         return -EINVAL;
1229         }
1230
1231         return free_idx;
1232 }
1233
1234 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1235 {
1236         if (--smmu->s2crs[idx].count)
1237                 return false;
1238
1239         smmu->s2crs[idx] = s2cr_init_val;
1240         if (smmu->smrs)
1241                 smmu->smrs[idx].valid = false;
1242
1243         return true;
1244 }
1245
1246 static int arm_smmu_master_alloc_smes(struct device *dev)
1247 {
1248         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1249         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1250         struct arm_smmu_device *smmu = cfg->smmu;
1251         struct arm_smmu_smr *smrs = smmu->smrs;
1252         struct iommu_group *group;
1253         int i, idx, ret;
1254
1255         mutex_lock(&smmu->stream_map_mutex);
1256         /* Figure out a viable stream map entry allocation */
1257         for_each_cfg_sme(fwspec, i, idx) {
1258                 u16 sid = fwspec->ids[i];
1259                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1260
1261                 if (idx != INVALID_SMENDX) {
1262                         ret = -EEXIST;
1263                         goto out_err;
1264                 }
1265
1266                 ret = arm_smmu_find_sme(smmu, sid, mask);
1267                 if (ret < 0)
1268                         goto out_err;
1269
1270                 idx = ret;
1271                 if (smrs && smmu->s2crs[idx].count == 0) {
1272                         smrs[idx].id = sid;
1273                         smrs[idx].mask = mask;
1274                         smrs[idx].valid = true;
1275                 }
1276                 smmu->s2crs[idx].count++;
1277                 cfg->smendx[i] = (s16)idx;
1278         }
1279
1280         group = iommu_group_get_for_dev(dev);
1281         if (!group)
1282                 group = ERR_PTR(-ENOMEM);
1283         if (IS_ERR(group)) {
1284                 ret = PTR_ERR(group);
1285                 goto out_err;
1286         }
1287         iommu_group_put(group);
1288
1289         /* It worked! Now, poke the actual hardware */
1290         for_each_cfg_sme(fwspec, i, idx) {
1291                 arm_smmu_write_sme(smmu, idx);
1292                 smmu->s2crs[idx].group = group;
1293         }
1294
1295         mutex_unlock(&smmu->stream_map_mutex);
1296         return 0;
1297
1298 out_err:
1299         while (i--) {
1300                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1301                 cfg->smendx[i] = INVALID_SMENDX;
1302         }
1303         mutex_unlock(&smmu->stream_map_mutex);
1304         return ret;
1305 }
1306
1307 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1308 {
1309         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1310         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1311         int i, idx;
1312
1313         mutex_lock(&smmu->stream_map_mutex);
1314         for_each_cfg_sme(fwspec, i, idx) {
1315                 if (arm_smmu_free_sme(smmu, idx))
1316                         arm_smmu_write_sme(smmu, idx);
1317                 cfg->smendx[i] = INVALID_SMENDX;
1318         }
1319         mutex_unlock(&smmu->stream_map_mutex);
1320 }
1321
1322 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1323                                       struct iommu_fwspec *fwspec)
1324 {
1325         struct arm_smmu_device *smmu = smmu_domain->smmu;
1326         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1327         u8 cbndx = smmu_domain->cfg.cbndx;
1328         enum arm_smmu_s2cr_type type;
1329         int i, idx;
1330
1331         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1332                 type = S2CR_TYPE_BYPASS;
1333         else
1334                 type = S2CR_TYPE_TRANS;
1335
1336         for_each_cfg_sme(fwspec, i, idx) {
1337                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1338                         continue;
1339
1340                 s2cr[idx].type = type;
1341                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1342                 s2cr[idx].cbndx = cbndx;
1343                 arm_smmu_write_s2cr(smmu, idx);
1344         }
1345         return 0;
1346 }
1347
1348 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1349 {
1350         int ret;
1351         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1352         struct arm_smmu_device *smmu;
1353         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1354
1355         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1356                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1357                 return -ENXIO;
1358         }
1359
1360         /*
1361          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1362          * domains between of_xlate() and add_device() - we have no way to cope
1363          * with that, so until ARM gets converted to rely on groups and default
1364          * domains, just say no (but more politely than by dereferencing NULL).
1365          * This should be at least a WARN_ON once that's sorted.
1366          */
1367         if (!fwspec->iommu_priv)
1368                 return -ENODEV;
1369
1370         smmu = fwspec_smmu(fwspec);
1371         /* Ensure that the domain is finalised */
1372         ret = arm_smmu_init_domain_context(domain, smmu);
1373         if (ret < 0)
1374                 return ret;
1375
1376         /*
1377          * Sanity check the domain. We don't support domains across
1378          * different SMMUs.
1379          */
1380         if (smmu_domain->smmu != smmu) {
1381                 dev_err(dev,
1382                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1383                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1384                 return -EINVAL;
1385         }
1386
1387         /* Looks ok, so add the device to the domain */
1388         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1389 }
1390
1391 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1392                         phys_addr_t paddr, size_t size, int prot)
1393 {
1394         int ret;
1395         unsigned long flags;
1396         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1397         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1398
1399         if (!ops)
1400                 return -ENODEV;
1401
1402         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1403         ret = ops->map(ops, iova, paddr, size, prot);
1404         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1405         return ret;
1406 }
1407
1408 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1409                              size_t size)
1410 {
1411         size_t ret;
1412         unsigned long flags;
1413         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1414         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1415
1416         if (!ops)
1417                 return 0;
1418
1419         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1420         ret = ops->unmap(ops, iova, size);
1421         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1422         return ret;
1423 }
1424
1425 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1426                                               dma_addr_t iova)
1427 {
1428         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1429         struct arm_smmu_device *smmu = smmu_domain->smmu;
1430         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1431         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1432         struct device *dev = smmu->dev;
1433         void __iomem *cb_base;
1434         u32 tmp;
1435         u64 phys;
1436         unsigned long va;
1437
1438         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1439
1440         /* ATS1 registers can only be written atomically */
1441         va = iova & ~0xfffUL;
1442         if (smmu->version == ARM_SMMU_V2)
1443                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1444         else /* Register is only 32-bit in v1 */
1445                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1446
1447         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1448                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1449                 dev_err(dev,
1450                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1451                         &iova);
1452                 return ops->iova_to_phys(ops, iova);
1453         }
1454
1455         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1456         if (phys & CB_PAR_F) {
1457                 dev_err(dev, "translation fault!\n");
1458                 dev_err(dev, "PAR = 0x%llx\n", phys);
1459                 return 0;
1460         }
1461
1462         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1463 }
1464
1465 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1466                                         dma_addr_t iova)
1467 {
1468         phys_addr_t ret;
1469         unsigned long flags;
1470         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1471         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1472
1473         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1474                 return iova;
1475
1476         if (!ops)
1477                 return 0;
1478
1479         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1480         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1481                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1482                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1483         } else {
1484                 ret = ops->iova_to_phys(ops, iova);
1485         }
1486
1487         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1488
1489         return ret;
1490 }
1491
1492 static bool arm_smmu_capable(enum iommu_cap cap)
1493 {
1494         switch (cap) {
1495         case IOMMU_CAP_CACHE_COHERENCY:
1496                 /*
1497                  * Return true here as the SMMU can always send out coherent
1498                  * requests.
1499                  */
1500                 return true;
1501         case IOMMU_CAP_NOEXEC:
1502                 return true;
1503         default:
1504                 return false;
1505         }
1506 }
1507
1508 static int arm_smmu_match_node(struct device *dev, void *data)
1509 {
1510         return dev->fwnode == data;
1511 }
1512
1513 static
1514 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1515 {
1516         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1517                                                 fwnode, arm_smmu_match_node);
1518         put_device(dev);
1519         return dev ? dev_get_drvdata(dev) : NULL;
1520 }
1521
1522 static int arm_smmu_add_device(struct device *dev)
1523 {
1524         struct arm_smmu_device *smmu;
1525         struct arm_smmu_master_cfg *cfg;
1526         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1527         int i, ret;
1528
1529         if (using_legacy_binding) {
1530                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1531                 fwspec = dev->iommu_fwspec;
1532                 if (ret)
1533                         goto out_free;
1534         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1535                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1536         } else {
1537                 return -ENODEV;
1538         }
1539
1540         ret = -EINVAL;
1541         for (i = 0; i < fwspec->num_ids; i++) {
1542                 u16 sid = fwspec->ids[i];
1543                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1544
1545                 if (sid & ~smmu->streamid_mask) {
1546                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1547                                 sid, smmu->streamid_mask);
1548                         goto out_free;
1549                 }
1550                 if (mask & ~smmu->smr_mask_mask) {
1551                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1552                                 mask, smmu->smr_mask_mask);
1553                         goto out_free;
1554                 }
1555         }
1556
1557         ret = -ENOMEM;
1558         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1559                       GFP_KERNEL);
1560         if (!cfg)
1561                 goto out_free;
1562
1563         cfg->smmu = smmu;
1564         fwspec->iommu_priv = cfg;
1565         while (i--)
1566                 cfg->smendx[i] = INVALID_SMENDX;
1567
1568         ret = arm_smmu_master_alloc_smes(dev);
1569         if (ret)
1570                 goto out_free;
1571
1572         iommu_device_link(&smmu->iommu, dev);
1573
1574         return 0;
1575
1576 out_free:
1577         if (fwspec)
1578                 kfree(fwspec->iommu_priv);
1579         iommu_fwspec_free(dev);
1580         return ret;
1581 }
1582
1583 static void arm_smmu_remove_device(struct device *dev)
1584 {
1585         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1586         struct arm_smmu_master_cfg *cfg;
1587         struct arm_smmu_device *smmu;
1588
1589
1590         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1591                 return;
1592
1593         cfg  = fwspec->iommu_priv;
1594         smmu = cfg->smmu;
1595
1596         iommu_device_unlink(&smmu->iommu, dev);
1597         arm_smmu_master_free_smes(fwspec);
1598         iommu_group_remove_device(dev);
1599         kfree(fwspec->iommu_priv);
1600         iommu_fwspec_free(dev);
1601 }
1602
1603 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1604 {
1605         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1606         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1607         struct iommu_group *group = NULL;
1608         int i, idx;
1609
1610         for_each_cfg_sme(fwspec, i, idx) {
1611                 if (group && smmu->s2crs[idx].group &&
1612                     group != smmu->s2crs[idx].group)
1613                         return ERR_PTR(-EINVAL);
1614
1615                 group = smmu->s2crs[idx].group;
1616         }
1617
1618         if (group)
1619                 return iommu_group_ref_get(group);
1620
1621         if (dev_is_pci(dev))
1622                 group = pci_device_group(dev);
1623         else
1624                 group = generic_device_group(dev);
1625
1626         return group;
1627 }
1628
1629 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1630                                     enum iommu_attr attr, void *data)
1631 {
1632         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1633
1634         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1635                 return -EINVAL;
1636
1637         switch (attr) {
1638         case DOMAIN_ATTR_NESTING:
1639                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1640                 return 0;
1641         default:
1642                 return -ENODEV;
1643         }
1644 }
1645
1646 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1647                                     enum iommu_attr attr, void *data)
1648 {
1649         int ret = 0;
1650         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1651
1652         if (domain->type != IOMMU_DOMAIN_UNMANAGED)
1653                 return -EINVAL;
1654
1655         mutex_lock(&smmu_domain->init_mutex);
1656
1657         switch (attr) {
1658         case DOMAIN_ATTR_NESTING:
1659                 if (smmu_domain->smmu) {
1660                         ret = -EPERM;
1661                         goto out_unlock;
1662                 }
1663
1664                 if (*(int *)data)
1665                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1666                 else
1667                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1668
1669                 break;
1670         default:
1671                 ret = -ENODEV;
1672         }
1673
1674 out_unlock:
1675         mutex_unlock(&smmu_domain->init_mutex);
1676         return ret;
1677 }
1678
1679 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1680 {
1681         u32 mask, fwid = 0;
1682
1683         if (args->args_count > 0)
1684                 fwid |= (u16)args->args[0];
1685
1686         if (args->args_count > 1)
1687                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1688         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1689                 fwid |= (u16)mask << SMR_MASK_SHIFT;
1690
1691         return iommu_fwspec_add_ids(dev, &fwid, 1);
1692 }
1693
1694 static void arm_smmu_get_resv_regions(struct device *dev,
1695                                       struct list_head *head)
1696 {
1697         struct iommu_resv_region *region;
1698         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1699
1700         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1701                                          prot, IOMMU_RESV_SW_MSI);
1702         if (!region)
1703                 return;
1704
1705         list_add_tail(&region->list, head);
1706
1707         iommu_dma_get_resv_regions(dev, head);
1708 }
1709
1710 static void arm_smmu_put_resv_regions(struct device *dev,
1711                                       struct list_head *head)
1712 {
1713         struct iommu_resv_region *entry, *next;
1714
1715         list_for_each_entry_safe(entry, next, head, list)
1716                 kfree(entry);
1717 }
1718
1719 static struct iommu_ops arm_smmu_ops = {
1720         .capable                = arm_smmu_capable,
1721         .domain_alloc           = arm_smmu_domain_alloc,
1722         .domain_free            = arm_smmu_domain_free,
1723         .attach_dev             = arm_smmu_attach_dev,
1724         .map                    = arm_smmu_map,
1725         .unmap                  = arm_smmu_unmap,
1726         .map_sg                 = default_iommu_map_sg,
1727         .iova_to_phys           = arm_smmu_iova_to_phys,
1728         .add_device             = arm_smmu_add_device,
1729         .remove_device          = arm_smmu_remove_device,
1730         .device_group           = arm_smmu_device_group,
1731         .domain_get_attr        = arm_smmu_domain_get_attr,
1732         .domain_set_attr        = arm_smmu_domain_set_attr,
1733         .of_xlate               = arm_smmu_of_xlate,
1734         .get_resv_regions       = arm_smmu_get_resv_regions,
1735         .put_resv_regions       = arm_smmu_put_resv_regions,
1736         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1737 };
1738
1739 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1740 {
1741         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1742         void __iomem *cb_base;
1743         int i;
1744         u32 reg, major;
1745
1746         /* clear global FSR */
1747         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1748         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1749
1750         /*
1751          * Reset stream mapping groups: Initial values mark all SMRn as
1752          * invalid and all S2CRn as bypass unless overridden.
1753          */
1754         for (i = 0; i < smmu->num_mapping_groups; ++i)
1755                 arm_smmu_write_sme(smmu, i);
1756
1757         if (smmu->model == ARM_MMU500) {
1758                 /*
1759                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1760                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1761                  * bit is only present in MMU-500r2 onwards.
1762                  */
1763                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1764                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1765                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1766                 if (major >= 2)
1767                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1768                 /*
1769                  * Allow unmatched Stream IDs to allocate bypass
1770                  * TLB entries for reduced latency.
1771                  */
1772                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1773                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1774         }
1775
1776         /* Make sure all context banks are disabled and clear CB_FSR  */
1777         for (i = 0; i < smmu->num_context_banks; ++i) {
1778                 cb_base = ARM_SMMU_CB(smmu, i);
1779                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1780                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1781                 /*
1782                  * Disable MMU-500's not-particularly-beneficial next-page
1783                  * prefetcher for the sake of errata #841119 and #826419.
1784                  */
1785                 if (smmu->model == ARM_MMU500) {
1786                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1787                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1788                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1789                 }
1790         }
1791
1792         /* Invalidate the TLB, just in case */
1793         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1794         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1795
1796         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1797
1798         /* Enable fault reporting */
1799         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1800
1801         /* Disable TLB broadcasting. */
1802         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1803
1804         /* Enable client access, handling unmatched streams as appropriate */
1805         reg &= ~sCR0_CLIENTPD;
1806         if (disable_bypass)
1807                 reg |= sCR0_USFCFG;
1808         else
1809                 reg &= ~sCR0_USFCFG;
1810
1811         /* Disable forced broadcasting */
1812         reg &= ~sCR0_FB;
1813
1814         /* Don't upgrade barriers */
1815         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1816
1817         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1818                 reg |= sCR0_VMID16EN;
1819
1820         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1821                 reg |= sCR0_EXIDENABLE;
1822
1823         /* Push the button */
1824         arm_smmu_tlb_sync_global(smmu);
1825         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1826 }
1827
1828 static int arm_smmu_id_size_to_bits(int size)
1829 {
1830         switch (size) {
1831         case 0:
1832                 return 32;
1833         case 1:
1834                 return 36;
1835         case 2:
1836                 return 40;
1837         case 3:
1838                 return 42;
1839         case 4:
1840                 return 44;
1841         case 5:
1842         default:
1843                 return 48;
1844         }
1845 }
1846
1847 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1848 {
1849         unsigned long size;
1850         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1851         u32 id;
1852         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1853         int i;
1854
1855         dev_notice(smmu->dev, "probing hardware configuration...\n");
1856         dev_notice(smmu->dev, "SMMUv%d with:\n",
1857                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1858
1859         /* ID0 */
1860         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1861
1862         /* Restrict available stages based on module parameter */
1863         if (force_stage == 1)
1864                 id &= ~(ID0_S2TS | ID0_NTS);
1865         else if (force_stage == 2)
1866                 id &= ~(ID0_S1TS | ID0_NTS);
1867
1868         if (id & ID0_S1TS) {
1869                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1870                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1871         }
1872
1873         if (id & ID0_S2TS) {
1874                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1875                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1876         }
1877
1878         if (id & ID0_NTS) {
1879                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1880                 dev_notice(smmu->dev, "\tnested translation\n");
1881         }
1882
1883         if (!(smmu->features &
1884                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1885                 dev_err(smmu->dev, "\tno translation support!\n");
1886                 return -ENODEV;
1887         }
1888
1889         if ((id & ID0_S1TS) &&
1890                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1891                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1892                 dev_notice(smmu->dev, "\taddress translation ops\n");
1893         }
1894
1895         /*
1896          * In order for DMA API calls to work properly, we must defer to what
1897          * the FW says about coherency, regardless of what the hardware claims.
1898          * Fortunately, this also opens up a workaround for systems where the
1899          * ID register value has ended up configured incorrectly.
1900          */
1901         cttw_reg = !!(id & ID0_CTTW);
1902         if (cttw_fw || cttw_reg)
1903                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1904                            cttw_fw ? "" : "non-");
1905         if (cttw_fw != cttw_reg)
1906                 dev_notice(smmu->dev,
1907                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1908
1909         /* Max. number of entries we have for stream matching/indexing */
1910         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1911                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1912                 size = 1 << 16;
1913         } else {
1914                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1915         }
1916         smmu->streamid_mask = size - 1;
1917         if (id & ID0_SMS) {
1918                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1919                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1920                 if (size == 0) {
1921                         dev_err(smmu->dev,
1922                                 "stream-matching supported, but no SMRs present!\n");
1923                         return -ENODEV;
1924                 }
1925
1926                 /* Zero-initialised to mark as invalid */
1927                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1928                                           GFP_KERNEL);
1929                 if (!smmu->smrs)
1930                         return -ENOMEM;
1931
1932                 dev_notice(smmu->dev,
1933                            "\tstream matching with %lu register groups", size);
1934         }
1935         /* s2cr->type == 0 means translation, so initialise explicitly */
1936         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1937                                          GFP_KERNEL);
1938         if (!smmu->s2crs)
1939                 return -ENOMEM;
1940         for (i = 0; i < size; i++)
1941                 smmu->s2crs[i] = s2cr_init_val;
1942
1943         smmu->num_mapping_groups = size;
1944         mutex_init(&smmu->stream_map_mutex);
1945
1946         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1947                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1948                 if (!(id & ID0_PTFS_NO_AARCH32S))
1949                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1950         }
1951
1952         /* ID1 */
1953         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1954         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1955
1956         /* Check for size mismatch of SMMU address space from mapped region */
1957         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1958         size <<= smmu->pgshift;
1959         if (smmu->cb_base != gr0_base + size)
1960                 dev_warn(smmu->dev,
1961                         "SMMU address space size (0x%lx) differs from mapped region size (0x%tx)!\n",
1962                         size * 2, (smmu->cb_base - gr0_base) * 2);
1963
1964         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1965         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1966         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1967                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1968                 return -ENODEV;
1969         }
1970         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1971                    smmu->num_context_banks, smmu->num_s2_context_banks);
1972         /*
1973          * Cavium CN88xx erratum #27704.
1974          * Ensure ASID and VMID allocation is unique across all SMMUs in
1975          * the system.
1976          */
1977         if (smmu->model == CAVIUM_SMMUV2) {
1978                 smmu->cavium_id_base =
1979                         atomic_add_return(smmu->num_context_banks,
1980                                           &cavium_smmu_context_count);
1981                 smmu->cavium_id_base -= smmu->num_context_banks;
1982                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1983         }
1984
1985         /* ID2 */
1986         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1987         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1988         smmu->ipa_size = size;
1989
1990         /* The output mask is also applied for bypass */
1991         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1992         smmu->pa_size = size;
1993
1994         if (id & ID2_VMID16)
1995                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1996
1997         /*
1998          * What the page table walker can address actually depends on which
1999          * descriptor format is in use, but since a) we don't know that yet,
2000          * and b) it can vary per context bank, this will have to do...
2001          */
2002         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
2003                 dev_warn(smmu->dev,
2004                          "failed to set DMA mask for table walker\n");
2005
2006         if (smmu->version < ARM_SMMU_V2) {
2007                 smmu->va_size = smmu->ipa_size;
2008                 if (smmu->version == ARM_SMMU_V1_64K)
2009                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2010         } else {
2011                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
2012                 smmu->va_size = arm_smmu_id_size_to_bits(size);
2013                 if (id & ID2_PTFS_4K)
2014                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
2015                 if (id & ID2_PTFS_16K)
2016                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
2017                 if (id & ID2_PTFS_64K)
2018                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2019         }
2020
2021         /* Now we've corralled the various formats, what'll it do? */
2022         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2023                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2024         if (smmu->features &
2025             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2026                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2027         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2028                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2029         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2030                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2031
2032         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2033                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2034         else
2035                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2036         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2037                    smmu->pgsize_bitmap);
2038
2039
2040         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2041                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2042                            smmu->va_size, smmu->ipa_size);
2043
2044         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2045                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2046                            smmu->ipa_size, smmu->pa_size);
2047
2048         return 0;
2049 }
2050
2051 struct arm_smmu_match_data {
2052         enum arm_smmu_arch_version version;
2053         enum arm_smmu_implementation model;
2054 };
2055
2056 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
2057 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
2058
2059 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2060 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2061 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2062 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2063 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2064
2065 static const struct of_device_id arm_smmu_of_match[] = {
2066         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2067         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2068         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2069         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2070         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2071         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2072         { },
2073 };
2074 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
2075
2076 #ifdef CONFIG_ACPI
2077 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2078 {
2079         int ret = 0;
2080
2081         switch (model) {
2082         case ACPI_IORT_SMMU_V1:
2083         case ACPI_IORT_SMMU_CORELINK_MMU400:
2084                 smmu->version = ARM_SMMU_V1;
2085                 smmu->model = GENERIC_SMMU;
2086                 break;
2087         case ACPI_IORT_SMMU_CORELINK_MMU401:
2088                 smmu->version = ARM_SMMU_V1_64K;
2089                 smmu->model = GENERIC_SMMU;
2090                 break;
2091         case ACPI_IORT_SMMU_V2:
2092                 smmu->version = ARM_SMMU_V2;
2093                 smmu->model = GENERIC_SMMU;
2094                 break;
2095         case ACPI_IORT_SMMU_CORELINK_MMU500:
2096                 smmu->version = ARM_SMMU_V2;
2097                 smmu->model = ARM_MMU500;
2098                 break;
2099         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2100                 smmu->version = ARM_SMMU_V2;
2101                 smmu->model = CAVIUM_SMMUV2;
2102                 break;
2103         default:
2104                 ret = -ENODEV;
2105         }
2106
2107         return ret;
2108 }
2109
2110 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2111                                       struct arm_smmu_device *smmu)
2112 {
2113         struct device *dev = smmu->dev;
2114         struct acpi_iort_node *node =
2115                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2116         struct acpi_iort_smmu *iort_smmu;
2117         int ret;
2118
2119         /* Retrieve SMMU1/2 specific data */
2120         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2121
2122         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2123         if (ret < 0)
2124                 return ret;
2125
2126         /* Ignore the configuration access interrupt */
2127         smmu->num_global_irqs = 1;
2128
2129         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2130                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2131
2132         return 0;
2133 }
2134 #else
2135 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2136                                              struct arm_smmu_device *smmu)
2137 {
2138         return -ENODEV;
2139 }
2140 #endif
2141
2142 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2143                                     struct arm_smmu_device *smmu)
2144 {
2145         const struct arm_smmu_match_data *data;
2146         struct device *dev = &pdev->dev;
2147         bool legacy_binding;
2148
2149         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2150                                  &smmu->num_global_irqs)) {
2151                 dev_err(dev, "missing #global-interrupts property\n");
2152                 return -ENODEV;
2153         }
2154
2155         data = of_device_get_match_data(dev);
2156         smmu->version = data->version;
2157         smmu->model = data->model;
2158
2159         parse_driver_options(smmu);
2160
2161         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2162         if (legacy_binding && !using_generic_binding) {
2163                 if (!using_legacy_binding)
2164                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2165                 using_legacy_binding = true;
2166         } else if (!legacy_binding && !using_legacy_binding) {
2167                 using_generic_binding = true;
2168         } else {
2169                 dev_err(dev, "not probing due to mismatched DT properties\n");
2170                 return -ENODEV;
2171         }
2172
2173         if (of_dma_is_coherent(dev->of_node))
2174                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2175
2176         return 0;
2177 }
2178
2179 static void arm_smmu_bus_init(void)
2180 {
2181         /* Oh, for a proper bus abstraction */
2182         if (!iommu_present(&platform_bus_type))
2183                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2184 #ifdef CONFIG_ARM_AMBA
2185         if (!iommu_present(&amba_bustype))
2186                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2187 #endif
2188 #ifdef CONFIG_PCI
2189         if (!iommu_present(&pci_bus_type)) {
2190                 pci_request_acs();
2191                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2192         }
2193 #endif
2194 }
2195
2196 static int arm_smmu_device_probe(struct platform_device *pdev)
2197 {
2198         struct resource *res;
2199         resource_size_t ioaddr;
2200         struct arm_smmu_device *smmu;
2201         struct device *dev = &pdev->dev;
2202         int num_irqs, i, err;
2203
2204         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2205         if (!smmu) {
2206                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2207                 return -ENOMEM;
2208         }
2209         smmu->dev = dev;
2210
2211         if (dev->of_node)
2212                 err = arm_smmu_device_dt_probe(pdev, smmu);
2213         else
2214                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2215
2216         if (err)
2217                 return err;
2218
2219         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2220         ioaddr = res->start;
2221         smmu->base = devm_ioremap_resource(dev, res);
2222         if (IS_ERR(smmu->base))
2223                 return PTR_ERR(smmu->base);
2224         smmu->cb_base = smmu->base + resource_size(res) / 2;
2225
2226         num_irqs = 0;
2227         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2228                 num_irqs++;
2229                 if (num_irqs > smmu->num_global_irqs)
2230                         smmu->num_context_irqs++;
2231         }
2232
2233         if (!smmu->num_context_irqs) {
2234                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2235                         num_irqs, smmu->num_global_irqs + 1);
2236                 return -ENODEV;
2237         }
2238
2239         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2240                                   GFP_KERNEL);
2241         if (!smmu->irqs) {
2242                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2243                 return -ENOMEM;
2244         }
2245
2246         for (i = 0; i < num_irqs; ++i) {
2247                 int irq = platform_get_irq(pdev, i);
2248
2249                 if (irq < 0) {
2250                         dev_err(dev, "failed to get irq index %d\n", i);
2251                         return -ENODEV;
2252                 }
2253                 smmu->irqs[i] = irq;
2254         }
2255
2256         err = arm_smmu_device_cfg_probe(smmu);
2257         if (err)
2258                 return err;
2259
2260         if (smmu->version == ARM_SMMU_V2 &&
2261             smmu->num_context_banks != smmu->num_context_irqs) {
2262                 dev_err(dev,
2263                         "found only %d context interrupt(s) but %d required\n",
2264                         smmu->num_context_irqs, smmu->num_context_banks);
2265                 return -ENODEV;
2266         }
2267
2268         for (i = 0; i < smmu->num_global_irqs; ++i) {
2269                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2270                                        arm_smmu_global_fault,
2271                                        IRQF_SHARED,
2272                                        "arm-smmu global fault",
2273                                        smmu);
2274                 if (err) {
2275                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2276                                 i, smmu->irqs[i]);
2277                         return err;
2278                 }
2279         }
2280
2281         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2282                                      "smmu.%pa", &ioaddr);
2283         if (err) {
2284                 dev_err(dev, "Failed to register iommu in sysfs\n");
2285                 return err;
2286         }
2287
2288         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2289         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2290
2291         err = iommu_device_register(&smmu->iommu);
2292         if (err) {
2293                 dev_err(dev, "Failed to register iommu\n");
2294                 return err;
2295         }
2296
2297         platform_set_drvdata(pdev, smmu);
2298         arm_smmu_device_reset(smmu);
2299         arm_smmu_test_smr_masks(smmu);
2300
2301         /*
2302          * For ACPI and generic DT bindings, an SMMU will be probed before
2303          * any device which might need it, so we want the bus ops in place
2304          * ready to handle default domain setup as soon as any SMMU exists.
2305          */
2306         if (!using_legacy_binding)
2307                 arm_smmu_bus_init();
2308
2309         return 0;
2310 }
2311
2312 /*
2313  * With the legacy DT binding in play, though, we have no guarantees about
2314  * probe order, but then we're also not doing default domains, so we can
2315  * delay setting bus ops until we're sure every possible SMMU is ready,
2316  * and that way ensure that no add_device() calls get missed.
2317  */
2318 static int arm_smmu_legacy_bus_init(void)
2319 {
2320         if (using_legacy_binding)
2321                 arm_smmu_bus_init();
2322         return 0;
2323 }
2324 device_initcall_sync(arm_smmu_legacy_bus_init);
2325
2326 static int arm_smmu_device_remove(struct platform_device *pdev)
2327 {
2328         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2329
2330         if (!smmu)
2331                 return -ENODEV;
2332
2333         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2334                 dev_err(&pdev->dev, "removing device with active domains!\n");
2335
2336         /* Turn the thing off */
2337         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2338         return 0;
2339 }
2340
2341 static struct platform_driver arm_smmu_driver = {
2342         .driver = {
2343                 .name           = "arm-smmu",
2344                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2345         },
2346         .probe  = arm_smmu_device_probe,
2347         .remove = arm_smmu_device_remove,
2348 };
2349 module_platform_driver(arm_smmu_driver);
2350
2351 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", NULL);
2352 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", NULL);
2353 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", NULL);
2354 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", NULL);
2355 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", NULL);
2356 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", NULL);
2357
2358 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2359 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2360 MODULE_LICENSE("GPL v2");