/* Maximum number of mapping groups per SMMU */
#define ARM_SMMU_MAX_SMRS 128
-/* Number of VMIDs per SMMU */
-#define ARM_SMMU_NUM_VMIDS 256
-
/* SMMU global address space */
#define ARM_SMMU_GR0(smmu) ((smmu)->base)
#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize)
#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6)
#define ARM_SMMU_PTE_AP_RDONLY (((pteval_t)2) << 6)
#define ARM_SMMU_PTE_ATTRINDX_SHIFT 2
+#define ARM_SMMU_PTE_nG (((pteval_t)1) << 11)
/* Stage-2 PTE */
#define ARM_SMMU_PTE_HAP_FAULT (((pteval_t)0) << 6)
#define ARM_SMMU_CB_FAR_LO 0x60
#define ARM_SMMU_CB_FAR_HI 0x64
#define ARM_SMMU_CB_FSYNR0 0x68
+#define ARM_SMMU_CB_S1_TLBIASID 0x610
#define SCTLR_S1_ASIDPNE (1 << 12)
#define SCTLR_CFCFG (1 << 7)
#define TTBCR2_ADDR_44 4
#define TTBCR2_ADDR_48 5
+#define TTBRn_HI_ASID_SHIFT 16
+
#define MAIR_ATTR_SHIFT(n) ((n) << 3)
#define MAIR_ATTR_MASK 0xff
#define MAIR_ATTR_DEVICE 0x04
#define FSR_IGN (FSR_AFF | FSR_ASF | FSR_TLBMCF | \
FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
- FSR_EF | FSR_PF | FSR_TF)
+ FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define FSYNR0_WNR (1 << 4)
u32 num_context_irqs;
unsigned int *irqs;
- DECLARE_BITMAP(vmid_map, ARM_SMMU_NUM_VMIDS);
-
struct list_head list;
struct rb_root masters;
};
struct arm_smmu_cfg {
struct arm_smmu_device *smmu;
- u8 vmid;
u8 cbndx;
u8 irptndx;
u32 cbar;
pgd_t *pgd;
};
+#define INVALID_IRPTNDX 0xff
+
+#define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx)
+#define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1)
struct arm_smmu_domain {
/*
}
}
+static void arm_smmu_tlb_inv_context(struct arm_smmu_cfg *cfg)
+{
+ struct arm_smmu_device *smmu = cfg->smmu;
+ void __iomem *base = ARM_SMMU_GR0(smmu);
+ bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+
+ if (stage1) {
+ base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+ writel_relaxed(ARM_SMMU_CB_ASID(cfg),
+ base + ARM_SMMU_CB_S1_TLBIASID);
+ } else {
+ base = ARM_SMMU_GR0(smmu);
+ writel_relaxed(ARM_SMMU_CB_VMID(cfg),
+ base + ARM_SMMU_GR0_TLBIVMID);
+ }
+
+ arm_smmu_tlb_sync(smmu);
+}
+
static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
{
int flags, ret;
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
+ if (!gfsr)
+ return IRQ_NONE;
+
gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
gfsr, gfsynr0, gfsynr1, gfsynr2);
writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
- return IRQ_NONE;
+ return IRQ_HANDLED;
}
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
/* CBAR */
- reg = root_cfg->cbar |
- (root_cfg->vmid << CBAR_VMID_SHIFT);
+ reg = root_cfg->cbar;
if (smmu->version == 1)
reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
/* Use the weakest memory type, so it is overridden by the pte */
if (stage1)
reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
+ else
+ reg |= ARM_SMMU_CB_VMID(root_cfg) << CBAR_VMID_SHIFT;
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
if (smmu->version > 1) {
/* TTBR0 */
reg = __pa(root_cfg->pgd);
-#ifndef __BIG_ENDIAN
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
+ if (stage1)
+ reg |= ARM_SMMU_CB_ASID(root_cfg) << TTBRn_HI_ASID_SHIFT;
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI);
-#else
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI);
- reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
- writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
-#endif
/*
* TTBCR
writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
}
- /* Nuke the TLB */
- writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
- arm_smmu_tlb_sync(smmu);
-
/* SCTLR */
reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
if (stage1)
return -ENODEV;
}
- ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 0, ARM_SMMU_NUM_VMIDS);
- if (IS_ERR_VALUE(ret))
- return ret;
-
- root_cfg->vmid = ret;
if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) {
/*
* We will likely want to change this if/when KVM gets
ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
smmu->num_context_banks);
if (IS_ERR_VALUE(ret))
- goto out_free_vmid;
+ return ret;
root_cfg->cbndx = ret;
-
if (smmu->version == 1) {
root_cfg->irptndx = atomic_inc_return(&smmu->irptndx);
root_cfg->irptndx %= smmu->num_context_irqs;
if (IS_ERR_VALUE(ret)) {
dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
root_cfg->irptndx, irq);
- root_cfg->irptndx = -1;
+ root_cfg->irptndx = INVALID_IRPTNDX;
goto out_free_context;
}
out_free_context:
__arm_smmu_free_bitmap(smmu->context_map, root_cfg->cbndx);
-out_free_vmid:
- __arm_smmu_free_bitmap(smmu->vmid_map, root_cfg->vmid);
return ret;
}
struct arm_smmu_domain *smmu_domain = domain->priv;
struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
struct arm_smmu_device *smmu = root_cfg->smmu;
+ void __iomem *cb_base;
int irq;
if (!smmu)
return;
- if (root_cfg->irptndx != -1) {
+ /* Disable the context bank and nuke the TLB before freeing it. */
+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+ arm_smmu_tlb_inv_context(root_cfg);
+
+ if (root_cfg->irptndx != INVALID_IRPTNDX) {
irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx];
free_irq(irq, domain);
}
- __arm_smmu_free_bitmap(smmu->vmid_map, root_cfg->vmid);
__arm_smmu_free_bitmap(smmu->context_map, root_cfg->cbndx);
}
static void arm_smmu_domain_destroy(struct iommu_domain *domain)
{
struct arm_smmu_domain *smmu_domain = domain->priv;
+
+ /*
+ * Free the domain resources. We assume that all devices have
+ * already been detached.
+ */
arm_smmu_destroy_domain_context(domain);
arm_smmu_free_pgtables(smmu_domain);
kfree(smmu_domain);
}
if (stage == 1) {
- pteval |= ARM_SMMU_PTE_AP_UNPRIV;
+ pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG;
if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
pteval |= ARM_SMMU_PTE_AP_RDONLY;
{
int ret;
struct arm_smmu_domain *smmu_domain = domain->priv;
- struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
- struct arm_smmu_device *smmu = root_cfg->smmu;
- void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0);
- writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
- arm_smmu_tlb_sync(smmu);
+ arm_smmu_tlb_inv_context(&smmu_domain->root_cfg);
return ret ? ret : size;
}
static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
{
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+ void __iomem *sctlr_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB_SCTLR;
int i = 0;
u32 scr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i));
}
+ /* Make sure all context banks are disabled */
+ for (i = 0; i < smmu->num_context_banks; ++i)
+ writel_relaxed(0, sctlr_base + ARM_SMMU_CB(smmu, i));
+
/* Invalidate the TLB, just in case */
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL);
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
of_node_put(master->of_node);
}
- if (!bitmap_empty(smmu->vmid_map, ARM_SMMU_NUM_VMIDS))
+ if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
dev_err(dev, "removing device with active domains!\n");
for (i = 0; i < smmu->num_global_irqs; ++i)
return ret;
/* Oh, for a proper bus abstraction */
- if (!iommu_present(&platform_bus_type));
+ if (!iommu_present(&platform_bus_type))
bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
- if (!iommu_present(&amba_bustype));
+ if (!iommu_present(&amba_bustype))
bus_set_iommu(&amba_bustype, &arm_smmu_ops);
return 0;
projects / karo-tx-linux.git / blobdiff