iommu/arm-smmu: Don't use VMIDs for stage-1 translations

[~andy/linux] / drivers / iommu / arm-smmu.c

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 3a595bb5b8248af59d27d067cdae863536806521..07a03087db331af563a72b709cb7d7d821dc1fe9 100644 (file)
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -87,6 +87,7 @@
 #define ARM_SMMU_PTE_AP_UNPRIV         (((pteval_t)1) << 6)
 #define ARM_SMMU_PTE_AP_RDONLY         (((pteval_t)2) << 6)
 #define ARM_SMMU_PTE_ATTRINDX_SHIFT    2
+#define ARM_SMMU_PTE_nG                        (((pteval_t)1) << 11)
 
 /* Stage-2 PTE */
 #define ARM_SMMU_PTE_HAP_FAULT         (((pteval_t)0) << 6)
@@ -223,6 +224,7 @@
 #define ARM_SMMU_CB_FAR_LO             0x60
 #define ARM_SMMU_CB_FAR_HI             0x64
 #define ARM_SMMU_CB_FSYNR0             0x68
+#define ARM_SMMU_CB_S1_TLBIASID                0x610
 
 #define SCTLR_S1_ASIDPNE               (1 << 12)
 #define SCTLR_CFCFG                    (1 << 7)
@@ -282,6 +284,8 @@
 #define TTBCR2_ADDR_44                 4
 #define TTBCR2_ADDR_48                 5
 
+#define TTBRn_HI_ASID_SHIFT            16
+
 #define MAIR_ATTR_SHIFT(n)             ((n) << 3)
 #define MAIR_ATTR_MASK                 0xff
 #define MAIR_ATTR_DEVICE               0x04
@@ -533,6 +537,23 @@ static void arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
        }
 }
 
+static void arm_smmu_tlb_inv_context(struct arm_smmu_cfg *cfg)
+{
+       struct arm_smmu_device *smmu = cfg->smmu;
+       void __iomem *base = ARM_SMMU_GR0(smmu);
+       bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+
+       if (stage1) {
+               base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+               writel_relaxed(cfg->vmid, base + ARM_SMMU_CB_S1_TLBIASID);
+       } else {
+               base = ARM_SMMU_GR0(smmu);
+               writel_relaxed(cfg->vmid, base + ARM_SMMU_GR0_TLBIVMID);
+       }
+
+       arm_smmu_tlb_sync(smmu);
+}
+
 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 {
        int flags, ret;
@@ -621,14 +642,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
        cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
 
        /* CBAR */
-       reg = root_cfg->cbar |
-             (root_cfg->vmid << CBAR_VMID_SHIFT);
+       reg = root_cfg->cbar;
        if (smmu->version == 1)
              reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 
        /* Use the weakest memory type, so it is overridden by the pte */
        if (stage1)
                reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
+       else
+               reg |= root_cfg->vmid << CBAR_VMID_SHIFT;
        writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
 
        if (smmu->version > 1) {
@@ -692,6 +714,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
        reg = __pa(root_cfg->pgd);
        writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
        reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
+       if (stage1)
+               reg |= root_cfg->vmid << TTBRn_HI_ASID_SHIFT;
        writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI);
 
        /*
@@ -747,10 +771,6 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain)
                writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
        }
 
-       /* Nuke the TLB */
-       writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
-       arm_smmu_tlb_sync(smmu);
-
        /* SCTLR */
        reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
        if (stage1)
@@ -787,7 +807,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
                return -ENODEV;
        }
 
-       ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 0, ARM_SMMU_NUM_VMIDS);
+       /* VMID zero is reserved for stage-1 mappings */
+       ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 1, ARM_SMMU_NUM_VMIDS);
        if (IS_ERR_VALUE(ret))
                return ret;
 
@@ -847,11 +868,17 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
        struct arm_smmu_domain *smmu_domain = domain->priv;
        struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
        struct arm_smmu_device *smmu = root_cfg->smmu;
+       void __iomem *cb_base;
        int irq;
 
        if (!smmu)
                return;
 
+       /* Disable the context bank and nuke the TLB before freeing it. */
+       cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
+       writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+       arm_smmu_tlb_inv_context(root_cfg);
+
        if (root_cfg->irptndx != -1) {
                irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx];
                free_irq(irq, domain);
@@ -956,6 +983,11 @@ static void arm_smmu_free_pgtables(struct arm_smmu_domain *smmu_domain)
 static void arm_smmu_domain_destroy(struct iommu_domain *domain)
 {
        struct arm_smmu_domain *smmu_domain = domain->priv;
+
+       /*
+        * Free the domain resources. We assume that all devices have
+        * already been detached.
+        */
        arm_smmu_destroy_domain_context(domain);
        arm_smmu_free_pgtables(smmu_domain);
        kfree(smmu_domain);
@@ -1196,7 +1228,7 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
        }
 
        if (stage == 1) {
-               pteval |= ARM_SMMU_PTE_AP_UNPRIV;
+               pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG;
                if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
                        pteval |= ARM_SMMU_PTE_AP_RDONLY;
 
@@ -1412,13 +1444,9 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
 {
        int ret;
        struct arm_smmu_domain *smmu_domain = domain->priv;
-       struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
-       struct arm_smmu_device *smmu = root_cfg->smmu;
-       void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 
        ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0);
-       writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
-       arm_smmu_tlb_sync(smmu);
+       arm_smmu_tlb_inv_context(&smmu_domain->root_cfg);
        return ret ? ret : size;
 }
 
@@ -1541,6 +1569,7 @@ static struct iommu_ops arm_smmu_ops = {
 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 {
        void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+       void __iomem *sctlr_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB_SCTLR;
        int i = 0;
        u32 scr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
 
@@ -1550,6 +1579,10 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
                writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i));
        }
 
+       /* Make sure all context banks are disabled */
+       for (i = 0; i < smmu->num_context_banks; ++i)
+               writel_relaxed(0, sctlr_base + ARM_SMMU_CB(smmu, i));
+
        /* Invalidate the TLB, just in case */
        writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL);
        writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);