Private PASID and per-instance pagetables

index d7e2f54086e4..366254e4b07f 100644
@@ -153,6 +153,7 @@ enum iommu_attr {
 	DOMAIN_ATTR_FSL_PAMU_ENABLE,
 	DOMAIN_ATTR_FSL_PAMUV1,
 	DOMAIN_ATTR_NESTING,	/* two stages of translation */
+	DOMAIN_ATTR_SPLIT_TABLES,
 	DOMAIN_ATTR_MAX,
 };
 

index a1226e4ab5f8..56f97093f46a 100644
@@ -193,7 +193,25 @@ enum arm_smmu_s2cr_privcfg {
 #define RESUME_RETRY			(0 << 0)
 #define RESUME_TERMINATE		(1 << 0)
 
+#define TTBCR_EPD1			(1 << 23)
+#define TTBCR_T1SZ_SHIFT		16
+#define TTBCR_IRGN1_SHIFT		24
+#define TTBCR_ORGN1_SHIFT		26
+#define TTBCR_RGN_WBWA			1
+#define TTBCR_SH1_SHIFT			28
+#define TTBCR_SH_IS			3
+
+#define TTBCR_TG1_16K			(1 << 30)
+#define TTBCR_TG1_4K			(2 << 30)
+#define TTBCR_TG1_64K			(3 << 30)
+
 #define TTBCR2_SEP_SHIFT		15
+#define TTBCR2_SEP_31			(0x0 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_SEP_35			(0x1 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_SEP_39			(0x2 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_SEP_41			(0x3 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_SEP_43			(0x4 << TTBCR2_SEP_SHIFT)
+#define TTBCR2_SEP_47			(0x5 << TTBCR2_SEP_SHIFT)
 #define TTBCR2_SEP_UPSTREAM		(0x7 << TTBCR2_SEP_SHIFT)
 #define TTBCR2_AS			(1 << 4)
 

270 lines not on this page · raw message

index fe851eae9057..920d9faa2a76 100644
@@ -422,8 +422,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova,
 	if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE)))
 		return 0;
 
-	if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||
-		    paddr >= (1ULL << data->iop.cfg.oas)))
+	if (WARN_ON(paddr >= (1ULL << data->iop.cfg.oas)))
 		return -ERANGE;
 
 	prot = arm_lpae_prot_to_pte(data, iommu_prot);

index 22e7b80a7682..d23e0092f917 100644
@@ -522,7 +522,7 @@ arm_smmu_alloc_priv_cd(struct iommu_pasid_table_ops *ops,
 
 	switch (fmt) {
 	case ARM_64_LPAE_S1:
-		cd->ttbr	= cfg->arm_lpae_s1_cfg.ttbr[0];
+		cd->ttbr	= cfg->arm_lpae_s1_cfg.ttbr;
 		cd->tcr		= cfg->arm_lpae_s1_cfg.tcr;
 		cd->mair	= cfg->arm_lpae_s1_cfg.mair[0];
 		break;

index 3568e8b073ec..d459909877c3 100644
@@ -644,7 +644,7 @@ static void arm_smmu_init_ttbr1(struct arm_smmu_domain *smmu_domain,
 		cb->split_table_mask = (1ULL << 48);
 	}
 
-	cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+	cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 	cb->ttbr[1] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 }
 

@@ -675,10 +675,10 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 	/* TTBRs */
 	if (stage1) {
 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
-			cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
-			cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
+			cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
+			cb->ttbr[1] = 0;
 		} else {
-			cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
+			cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 			cb->ttbr[0] |= (u64)cfg->asid << TTBRn_ASID_SHIFT;
 
 			/*

index 10e4a3d11c02..37d607ac5153 100644
@@ -767,11 +767,10 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
 	wmb();
 
 	/* TTBRs */
-	cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) |
+	cfg->arm_v7s_cfg.ttbr = virt_to_phys(data->pgd) |
 				   ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS |
 				   ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) |
 				   ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA);
-	cfg->arm_v7s_cfg.ttbr[1] = 0;
 	return &data->iop;
 
 out_free_data:

index 920d9faa2a76..5bba30d901b4 100644
@@ -793,9 +793,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 	/* Ensure the empty pgd is visible before any actual TTBR write */
 	wmb();
 
-	/* TTBRs */
-	cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd);
-	cfg->arm_lpae_s1_cfg.ttbr[1] = 0;
+	/* TTBR */
+	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
 	return &data->iop;
 
 out_free_data:

index 2df79093cad9..fd9f0fc4eb60 100644
@@ -87,7 +87,7 @@ struct io_pgtable_cfg {
 	/* Low-level data specific to the table format */
 	union {
 		struct {
-			u64	ttbr[2];
+			u64	ttbr;
 			u64	tcr;
 			u64	mair[2];
 		} arm_lpae_s1_cfg;

@@ -98,7 +98,7 @@ struct io_pgtable_cfg {
 		} arm_lpae_s2_cfg;
 
 		struct {
-			u32	ttbr[2];
+			u32	ttbr;
 			u32	tcr;
 			u32	nmrr;
 			u32	prrr;

index 40ae6e87cb88..7cdaa0fef85a 100644
@@ -447,7 +447,7 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
 	}
 
 	/* TTBR0 */
-	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0];
+	ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
 	ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
 	ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
 

index 0d3350463a3f..ef323301c574 100644
@@ -281,8 +281,8 @@ static void __program_context(void __iomem *base, int ctx,
 	SET_V2PCFG(base, ctx, 0x3);
 
 	SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);
-	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]);
-	SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]);
+	SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr);
+	SET_TTBR1(base, ctx, 0);
 
 	/* Set prrr and nmrr */
 	SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);

index f2832a10fcea..db710d99fc5f 100644
@@ -344,7 +344,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
 	/* Update the pgtable base address register of the M4U HW */
 	if (!data->m4u_dom) {
 		data->m4u_dom = dom;
-		writel(dom->cfg.arm_v7s_cfg.ttbr[0],
+		writel(dom->cfg.arm_v7s_cfg.ttbr,
 		       data->base + REG_MMU_PT_BASE_ADDR);
 	}
 

@@ -725,7 +725,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
 	writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
 	writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
 	if (data->m4u_dom)
-		writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
+		writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr,
 		       base + REG_MMU_PT_BASE_ADDR);
 	return 0;
 }

index 65b9c99707f8..b0f5fd5b33c0 100644
@@ -257,10 +257,9 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain,
 
 		/* TTBRs */
 		iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
-				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
+				pgtbl_cfg.arm_lpae_s1_cfg.ttbr |
 				((u64)ctx->asid << TTBRn_ASID_SHIFT));
 		iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
-				pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
 				((u64)ctx->asid << TTBRn_ASID_SHIFT));
 
 		/* TTBCR */

index e98b994c15f1..26f0da9692d4 100644
@@ -156,6 +156,7 @@ io_mm_alloc(struct iommu_domain *domain, struct device *dev,
 	mmgrab(mm);
 
 	io_mm->flags		= flags;
+	io_mm->type		= IO_TYPE_SHARED;
 	io_mm->mm		= mm;
 	io_mm->notifier.ops	= &iommu_mmu_notifier;
 	io_mm->release		= domain->ops->mm_free;

@@ -544,13 +545,10 @@ int iommu_sva_device_init(struct device *dev, unsigned long features,
 			  unsigned int max_pasid,
 			  iommu_mm_exit_handler_t mm_exit)
 {
-	int ret;
+	int ret = 0;
 	struct iommu_sva_param *param;
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
 
-	if (!domain || !domain->ops->sva_device_init)
-		return -ENODEV;
-
 	if (features & ~IOMMU_SVA_FEAT_IOPF)
 		return -EINVAL;
 

@@ -576,9 +574,12 @@ int iommu_sva_device_init(struct device *dev, unsigned long features,
 	 * IOMMU driver updates the limits depending on the IOMMU and device
 	 * capabilities.
 	 */
-	ret = domain->ops->sva_device_init(dev, param);
-	if (ret)
-		goto err_free_param;
+
+	if (domain && domain->ops->sva_device_init) {
+		ret = domain->ops->sva_device_init(dev, param);
+		if (ret)
+			goto err_free_param;
+	}
 
 	mutex_lock(&dev->iommu_param->lock);
 	if (dev->iommu_param->sva_param)

@@ -790,3 +791,127 @@ struct mm_struct *iommu_sva_find(int pasid)
 	return mm;
 }
 EXPORT_SYMBOL_GPL(iommu_sva_find);
+
+int iommu_sva_alloc_pasid(struct iommu_domain *domain, struct device *dev)
+{
+	int ret, pasid;
+	struct io_mm *io_mm;
+	struct iommu_sva_param *param = dev->iommu_param->sva_param;
+
+	if (!domain->ops->mm_attach || !domain->ops->mm_detach)
+		return -ENODEV;
+
+	if (domain->ops->mm_alloc)
+		io_mm = domain->ops->mm_alloc(domain, NULL, 0);
+	else
+		io_mm = kzalloc(sizeof(*io_mm), GFP_KERNEL);
+
+	if (IS_ERR(io_mm))
+		return PTR_ERR(io_mm);
+	if (!io_mm)
+		return -ENOMEM;
+
+	io_mm->domain = domain;
+	io_mm->type = IO_TYPE_PRIVATE;
+
+	idr_preload(GFP_KERNEL);
+	spin_lock(&iommu_sva_lock);
+	pasid = idr_alloc_cyclic(&iommu_pasid_idr, io_mm, param->min_pasid,
+		param->max_pasid + 1, GFP_ATOMIC);
+	io_mm->pasid = pasid;
+	spin_unlock(&iommu_sva_lock);
+	idr_preload_end();
+
+	if (pasid < 0) {
+		kfree(io_mm);
+		return pasid;
+	}
+
+	ret = domain->ops->mm_attach(domain, dev, io_mm, false);
+	if (!ret)
+		return pasid;
+
+	spin_lock(&iommu_sva_lock);
+	idr_remove(&iommu_pasid_idr, io_mm->pasid);
+	spin_unlock(&iommu_sva_lock);
+
+	if (domain->ops->mm_free)
+		domain->ops->mm_free(io_mm);
+	else
+		kfree(io_mm);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(iommu_sva_alloc_pasid);
+
+static struct io_mm *get_io_mm(int pasid)
+{
+	struct io_mm *io_mm;
+
+	spin_lock(&iommu_sva_lock);
+	io_mm = idr_find(&iommu_pasid_idr, pasid);
+	spin_unlock(&iommu_sva_lock);
+
+	return io_mm;
+}
+
+int iommu_sva_map(int pasid, unsigned long iova,
+	      phys_addr_t paddr, size_t size, int prot)
+{
+	struct io_mm *io_mm = get_io_mm(pasid);
+
+	if (!io_mm || io_mm->type != IO_TYPE_PRIVATE)
+		return -ENODEV;
+
+	return __iommu_map(io_mm->domain, &pasid, iova, paddr, size, prot);
+}
+EXPORT_SYMBOL_GPL(iommu_sva_map);
+
+size_t iommu_sva_map_sg(int pasid, unsigned long iova, struct scatterlist *sg,
+		unsigned int nents, int prot)
+{
+	struct io_mm *io_mm = get_io_mm(pasid);
+	struct iommu_domain *domain;
+
+	if (!io_mm || io_mm->type != IO_TYPE_PRIVATE)
+		return -ENODEV;
+
+	domain = io_mm->domain;
+
+	return domain->ops->map_sg(domain, &pasid, iova, sg, nents, prot);
+}
+EXPORT_SYMBOL_GPL(iommu_sva_map_sg);
+
+size_t iommu_sva_unmap(int pasid, unsigned long iova, size_t size)
+{
+	struct io_mm *io_mm = get_io_mm(pasid);
+
+	if (!io_mm || io_mm->type != IO_TYPE_PRIVATE)
+		return -ENODEV;
+
+	return __iommu_unmap(io_mm->domain, &pasid, iova, size, false);
+}
+EXPORT_SYMBOL_GPL(iommu_sva_unmap);
+
+void iommu_sva_free_pasid(int pasid, struct device *dev)
+{
+	struct io_mm *io_mm = get_io_mm(pasid);
+	struct iommu_domain *domain;
+
+	if (!io_mm || io_mm->type != IO_TYPE_PRIVATE)
+		return;
+
+	domain = io_mm->domain;
+
+	domain->ops->mm_detach(domain, dev, io_mm, false);
+
+	spin_lock(&iommu_sva_lock);
+	idr_remove(&iommu_pasid_idr, io_mm->pasid);
+	spin_unlock(&iommu_sva_lock);
+
+	if (domain->ops->mm_free)
+		domain->ops->mm_free(io_mm);
+	else
+		kfree(io_mm);
+}
+EXPORT_SYMBOL_GPL(iommu_sva_free_pasid);

132 lines not on this page · raw message

129 lines not on this page · raw message

index d459909877c3..5c7c135bbb44 100644
@@ -201,7 +201,6 @@ struct arm_smmu_device {
 	unsigned long			va_size;
 	unsigned long			ipa_size;
 	unsigned long			pa_size;
-	unsigned long			ubs_size;
 	unsigned long			pgsize_bitmap;
 
 	u32				num_global_irqs;

@@ -252,6 +251,9 @@ struct arm_smmu_domain {
 	spinlock_t			cb_lock; /* Serialises ATS1* ops and TLB syncs */
 	u32 attributes;
 	struct iommu_domain		domain;
+
+	spinlock_t			pasid_lock;
+	struct list_head		pasid_list;
 };
 
 struct arm_smmu_option_prop {

@@ -259,6 +261,144 @@ struct arm_smmu_option_prop {
 	const char *prop;
 };
 
+static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
+{
+	return container_of(dom, struct arm_smmu_domain, domain);
+}
+
+struct arm_smmu_pasid {
+	struct iommu_domain *domain;
+	struct io_pgtable_ops		*pgtbl_ops;
+	struct list_head node;
+	int pasid;
+};
+
+struct arm_smmu_pasid *arm_smmu_get_pasid(struct arm_smmu_domain *smmu_domain,
+		int pasid)
+{
+	struct arm_smmu_pasid *node, *obj = NULL;
+
+	spin_lock(&smmu_domain->pasid_lock);
+	list_for_each_entry(node, &smmu_domain->pasid_list, node) {
+		if (node->pasid == pasid) {
+			obj = node;
+			break;
+		}
+	}
+	spin_unlock(&smmu_domain->pasid_lock);
+
+	return obj;
+}
+
+static void arm_smmu_mm_detach(struct iommu_domain *domain, struct device *dev,
+		struct io_mm *io_mm, bool unused)
+{
+	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct arm_smmu_pasid *node, *obj = NULL;
+
+	spin_lock(&smmu_domain->pasid_lock);
+	list_for_each_entry(node, &smmu_domain->pasid_list, node) {
+		if (node->pasid == io_mm->pasid) {
+			obj = node;
+			list_del(&obj->node);
+			break;
+		}
+	}
+	spin_unlock(&smmu_domain->pasid_lock);
+
+	if (obj)
+		free_io_pgtable_ops(obj->pgtbl_ops);
+
+	kfree(obj);
+}
+
+static size_t arm_smmu_sva_unmap(struct iommu_domain *domain, int pasid,
+		unsigned long iova, size_t size)
+{
+	struct arm_smmu_pasid *obj =
+		arm_smmu_get_pasid(to_smmu_domain(domain), pasid);
+
+	if (!obj)
+		return -ENODEV;
+
+	return obj->pgtbl_ops->unmap(obj->pgtbl_ops, iova, size);
+}
+
+
+static int arm_smmu_sva_map(struct iommu_domain *domain, int pasid,
+		unsigned long iova, phys_addr_t paddr, size_t size, int prot)
+{
+	struct arm_smmu_pasid *obj =
+		arm_smmu_get_pasid(to_smmu_domain(domain), pasid);
+
+	if (!obj)
+		return -ENODEV;
+
+	return obj->pgtbl_ops->map(obj->pgtbl_ops, iova, paddr, size, prot);
+}
+
+static int arm_smmu_mm_attach(struct iommu_domain *domain, struct device *dev,
+		struct io_mm *io_mm, bool unused)
+{
+	struct arm_smmu_pasid *obj;
+	struct io_pgtable_cfg pgtbl_cfg;
+	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct arm_smmu_device *smmu = smmu_domain->smmu;
+	enum io_pgtable_fmt fmt;
+	unsigned long ias, oas;
+
+	/* Only allow private pasids */
+	if (io_mm->type != IO_TYPE_PRIVATE || io_mm->mm)
+		return -ENODEV;
+
+	/* Only allow pasid backed tables to be created on S1 domains */
+	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
+		return -ENODEV;
+
+	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+	if (!obj)
+		return -ENOMEM;
+
+	/* Get the same exact format as the parent domain */
+	ias = smmu->va_size;
+	oas = smmu->ipa_size;
+
+	if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
+		fmt = ARM_64_LPAE_S1;
+	else if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
+		fmt = ARM_32_LPAE_S1;
+		ias = min(ias, 32UL);
+		oas = min(oas, 40UL);
+	} else {
+		fmt = ARM_V7S;
+		ias = min(ias, 32UL);
+		oas = min(oas, 32UL);
+	}
+
+	pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap = smmu->pgsize_bitmap,
+		.ias = ias,
+		.oas = oas,
+		.tlb = NULL,
+		.iommu_dev = smmu->dev
+	};
+
+	obj->pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
+	if (!obj->pgtbl_ops) {
+		kfree(obj);
+		return -ENOMEM;
+	}
+
+	obj->domain = domain;
+	obj->pasid = io_mm->pasid;
+
+	spin_lock(&smmu_domain->pasid_lock);
+	list_add_tail(&obj->node, &smmu_domain->pasid_list);
+	spin_unlock(&smmu_domain->pasid_lock);
+
+	return 0;
+}
+
 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 
 static bool using_legacy_binding, using_generic_binding;

@@ -268,11 +408,6 @@ static struct arm_smmu_option_prop arm_smmu_options[] = {
 	{ 0, NULL},
 };
 
-static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
-{
-	return container_of(dom, struct arm_smmu_domain, domain);
-}
-
 static void parse_driver_options(struct arm_smmu_device *smmu)
 {
 	int i = 0;

@@ -1055,6 +1190,9 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 	mutex_init(&smmu_domain->init_mutex);
 	spin_lock_init(&smmu_domain->cb_lock);
 
+	spin_lock_init(&smmu_domain->pasid_lock);
+	INIT_LIST_HEAD(&smmu_domain->pasid_list);
+
 	return &smmu_domain->domain;
 }
 

@@ -1694,6 +1832,10 @@ static struct iommu_ops arm_smmu_ops = {
 	.of_xlate		= arm_smmu_of_xlate,
 	.get_resv_regions	= arm_smmu_get_resv_regions,
 	.put_resv_regions	= arm_smmu_put_resv_regions,
+	.mm_attach		= arm_smmu_mm_attach,
+	.sva_map		= arm_smmu_sva_map,
+	.sva_unmap		= arm_smmu_sva_unmap,
+	.mm_detach		= arm_smmu_mm_detach,
 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
 };
 

index fd9f0fc4eb60..69fcee763446 100644
@@ -173,18 +173,22 @@ struct io_pgtable {
 
 static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop)
 {
-	iop->cfg.tlb->tlb_flush_all(iop->cookie);
+	if (iop->cfg.tlb)
+		iop->cfg.tlb->tlb_flush_all(iop->cookie);
 }
 
 static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop,
 		unsigned long iova, size_t size, size_t granule, bool leaf)
 {
-	iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie);
+	if (iop->cfg.tlb)
+		iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf,
+			iop->cookie);
 }
 
 static inline void io_pgtable_tlb_sync(struct io_pgtable *iop)
 {
-	iop->cfg.tlb->tlb_sync(iop->cookie);
+	if (iop->cfg.tlb)
+		iop->cfg.tlb->tlb_sync(iop->cookie);
 }
 
 /**

index 5c7c135bbb44..100797a07be0 100644
@@ -50,6 +50,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/arm-smmu.h>
 
 #include <linux/amba/bus.h>
 

@@ -254,6 +255,8 @@ struct arm_smmu_domain {
 
 	spinlock_t			pasid_lock;
 	struct list_head		pasid_list;
+	const struct arm_smmu_pasid_ops	*pasid_ops;
+	void				*pasid_data;
 };
 
 struct arm_smmu_option_prop {

@@ -296,6 +299,10 @@ static void arm_smmu_mm_detach(struct iommu_domain *domain, struct device *dev,
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 	struct arm_smmu_pasid *node, *obj = NULL;
 
+	if (smmu_domain->pasid_ops && smmu_domain->pasid_ops->remove_pasid)
+		smmu_domain->pasid_ops->remove_pasid(io_mm->pasid,
+			smmu_domain->pasid_data);
+
 	spin_lock(&smmu_domain->pasid_lock);
 	list_for_each_entry(node, &smmu_domain->pasid_list, node) {
 		if (node->pasid == io_mm->pasid) {

@@ -392,6 +399,26 @@ static int arm_smmu_mm_attach(struct iommu_domain *domain, struct device *dev,
 	obj->domain = domain;
 	obj->pasid = io_mm->pasid;
 
+	if (smmu_domain->pasid_ops && smmu_domain->pasid_ops->install_pasid) {
+		int ret;
+		u64 ttbr;
+
+		if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH32_S)
+			ttbr = pgtbl_cfg.arm_v7s_cfg.ttbr;
+		else
+			ttbr = pgtbl_cfg.arm_lpae_s1_cfg.ttbr;
+
+		ret = smmu_domain->pasid_ops->install_pasid(io_mm->pasid, ttbr,
+			smmu_domain->cfg.asid, smmu_domain->pasid_data);
+
+		if (ret) {
+			free_io_pgtable_ops(obj->pgtbl_ops);
+			kfree(obj);
+
+			return ret;
+		}
+	}
+
 	spin_lock(&smmu_domain->pasid_lock);
 	list_add_tail(&obj->node, &smmu_domain->pasid_list);
 	spin_unlock(&smmu_domain->pasid_lock);

@@ -2156,6 +2183,19 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 	return 0;
 }
 
+void arm_smmu_add_pasid_ops(struct iommu_domain *domain,
+	const struct arm_smmu_pasid_ops *ops, void *data)
+{
+	struct arm_smmu_domain *smmu_domain;
+
+	if (domain) {
+		smmu_domain = to_smmu_domain(domain);
+		smmu_domain->pasid_ops = ops;
+		smmu_domain->pasid_data = data;
+	}
+}
+EXPORT_SYMBOL_GPL(arm_smmu_add_pasid_ops);
+
 struct arm_smmu_match_data {
 	enum arm_smmu_arch_version version;
 	enum arm_smmu_implementation model;

new file mode 100644
index 000000000000..c14ca52231bf
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2018, The Linux Foundation. All rights reserved. */
+
+#ifndef ARM_SMMU_H_
+#define ARM_SMMU_H_
+
+struct iommu_domain;
+
+struct arm_smmu_pasid_ops {
+	int (*install_pasid)(int pasid, u64 ttbr, u32 asid, void *data);
+	void (*remove_pasid)(int pasid, void *data);
+};
+
+
+void arm_smmu_add_pasid_ops(struct iommu_domain *domain,
+	const struct arm_smmu_pasid_ops *ops, void *data);
+
+#endif

index d39400e5bc42..b2c0370072dd 100644
@@ -741,6 +741,20 @@ static int a5xx_hw_init(struct msm_gpu *gpu)
 		REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
 	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
 
+	/* Put the GPU into 64 bit by default */
+	gpu_write(gpu, REG_A5XX_CP_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_VSC_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_RB_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_PC_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_VFD_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_VPC_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_SP_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
+	gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
+
 	ret = adreno_hw_init(gpu);
 	if (ret)
 		return ret;

index b23d33622f37..fdbe1a8372f0 100644
@@ -30,7 +30,7 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
 	struct msm_iommu *iommu = arg;
 	if (iommu->base.handler)
 		return iommu->base.handler(iommu->base.arg, iova, flags);
-	pr_warn_ratelimited("*** fault: iova=%08lx, flags=%d\n", iova, flags);
+	pr_warn_ratelimited("*** fault: iova=%16lx, flags=%d\n", iova, flags);
 	return 0;
 }
 

index 2f45bea04221..78e8e56d2499 100644
@@ -703,7 +703,8 @@ static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu)
 
 static struct msm_gem_address_space *
 msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev,
-		uint64_t va_start, uint64_t va_end)
+		u64 va_start, u64 va_end,
+		u64 va_global_start, u64 va_global_end)
 {
 	struct iommu_domain *iommu;
 	struct msm_gem_address_space *aspace;

@@ -721,6 +722,19 @@ msm_gpu_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev,
 	iommu->geometry.aperture_start = va_start;
 	iommu->geometry.aperture_end = va_end;
 
+	/* If a va_global range was specified then try to set up split tables */
+	if (va_global_start && va_global_end) {
+		int val = 1;
+
+		ret = iommu_domain_set_attr(iommu, DOMAIN_ATTR_SPLIT_TABLES,
+			&val);
+
+		if (!WARN(ret, "Unable to enable split pagetables for the IOMMU\n")) {
+			iommu->geometry.aperture_start = va_global_start;
+			iommu->geometry.aperture_end = va_global_end;
+		}
+	}
+
 	dev_info(gpu->dev->dev, "%s: using IOMMU\n", gpu->name);
 
 	aspace = msm_gem_address_space_create(&pdev->dev, iommu, "gpu");

@@ -813,7 +827,8 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	msm_devfreq_init(gpu);
 
 	gpu->aspace = msm_gpu_create_address_space(gpu, pdev,
-		config->va_start, config->va_end);
+		config->va_start, config->va_end, config->va_start_global,
+		config->va_end_global);
 
 	if (gpu->aspace == NULL)
 		dev_info(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name);

index b8241179175a..da58aa6c12c8 100644
@@ -31,8 +31,8 @@ struct msm_gpu_perfcntr;
 struct msm_gpu_config {
 	const char *ioname;
 	const char *irqname;
-	uint64_t va_start;
-	uint64_t va_end;
+	uint64_t va_start, va_end;
+	uint64_t va_start_global, va_end_global;
 	unsigned int nr_rings;
 };
 

index aa2c5d4580c8..85df78d71398 100644
@@ -35,6 +35,7 @@ struct msm_mmu {
 	struct device *dev;
 	int (*handler)(void *arg, unsigned long iova, int flags);
 	void *arg;
+	unsigned long features;
 };
 
 static inline void msm_mmu_init(struct msm_mmu *mmu, struct device *dev,

@@ -54,4 +55,16 @@ static inline void msm_mmu_set_fault_handler(struct msm_mmu *mmu, void *arg,
 	mmu->handler = handler;
 }
 
+static inline void msm_mmu_set_feature(struct msm_mmu *mmu,
+		unsigned long feature)
+{
+	mmu->features |= feature;
+}
+
+static inline bool msm_mmu_has_feature(struct msm_mmu *mmu,
+		unsigned long feature)
+{
+	return (mmu->features & feature) ? true : false;
+}
+
 #endif /* __MSM_MMU_H__ */

index fdbe1a8372f0..99e6611969d4 100644
@@ -15,6 +15,9 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/hashtable.h>
+#include <linux/arm-smmu.h>
+
 #include "msm_drv.h"
 #include "msm_mmu.h"
 

@@ -34,12 +37,32 @@ static int msm_fault_handler(struct iommu_domain *domain, struct device *dev,
 	return 0;
 }
 
+static bool msm_iommu_check_per_instance(struct msm_iommu *iommu)
+{
+	int val;
+
+	if (!IS_ENABLED(CONFIG_IOMMU_SVA))
+		return false;
+
+	if (iommu_domain_get_attr(iommu->domain, DOMAIN_ATTR_SPLIT_TABLES,
+		&val))
+		return false;
+
+	return val ? true : false;
+}
+
 static int msm_iommu_attach(struct msm_mmu *mmu, const char * const *names,
 			    int cnt)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 	int ret;
 
+	if (msm_iommu_check_per_instance(iommu)) {
+		if (!iommu_sva_device_init(mmu->dev, 0, (1 << 31), NULL))
+			msm_mmu_set_feature(mmu,
+				MMU_FEATURE_PER_INSTANCE_TABLES);
+	}
+
 	pm_runtime_get_sync(mmu->dev);
 	ret = iommu_attach_device(iommu->domain, mmu->dev);
 	pm_runtime_put_sync(mmu->dev);

@@ -112,3 +135,170 @@ struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain)
 
 	return &iommu->base;
 }
+
+struct pasid_entry {
+	int pasid;
+	u64 ttbr;
+	u32 asid;
+	struct hlist_node node;
+};
+
+DECLARE_HASHTABLE(pasid_table, 4);
+
+static int install_pasid_cb(int pasid, u64 ttbr, u32 asid, void *data)
+{
+	struct pasid_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+
+	if (!entry)
+		return -ENOMEM;
+
+	entry->pasid = pasid;
+	entry->ttbr = ttbr;
+	entry->asid = asid;
+
+	/* FIXME: Assume that we'll never have a pasid conflict? */
+	/* FIXME: locks? RCU? */
+	hash_add(pasid_table, &entry->node, pasid);
+	return 0;
+}
+
+static void remove_pasid_cb(int pasid, void *data)
+{
+	struct pasid_entry *entry;
+
+	hash_for_each_possible(pasid_table, entry, node, pasid) {
+		if (pasid == entry->pasid) {
+			hash_del(&entry->node);
+			kfree(entry);
+			return;
+		}
+	}
+}
+
+struct msm_iommu_pasid {
+	struct msm_mmu base;
+	struct device *dev;
+	int pasid;
+	u64 ttbr;
+	u32 asid;
+};
+#define to_msm_iommu_pasid(x) container_of(x, struct msm_iommu_pasid, base)
+
+static int msm_iommu_pasid_attach(struct msm_mmu *mmu,
+		const char * const *names, int cnt)
+{
+	return 0;
+}
+
+static int msm_iommu_pasid_map(struct msm_mmu *mmu, uint64_t iova,
+		struct sg_table *sgt, unsigned len, int prot)
+{
+	struct msm_iommu_pasid *pasid = to_msm_iommu_pasid(mmu);
+	int ret;
+
+	ret = iommu_sva_map_sg(pasid->pasid, iova, sgt->sgl, sgt->nents, prot);
+	WARN_ON(ret < 0);
+
+	return (ret == len) ? 0 : -EINVAL;
+}
+
+static int msm_iommu_pasid_unmap(struct msm_mmu *mmu, uint64_t iova,
+		struct sg_table *sgt, unsigned len)
+{
+	struct msm_iommu_pasid *pasid = to_msm_iommu_pasid(mmu);
+
+	iommu_sva_unmap(pasid->pasid, iova, len);
+
+	return 0;
+}
+
+static void msm_iommu_pasid_detach(struct msm_mmu *mmu,
+		const char * const *names, int cnt)
+{
+}
+
+static void msm_iommu_pasid_destroy(struct msm_mmu *mmu)
+{
+	struct msm_iommu_pasid *pasid = to_msm_iommu_pasid(mmu);
+
+	iommu_sva_free_pasid(pasid->pasid, pasid->dev);
+	kfree(pasid);
+}
+
+static const struct msm_mmu_funcs pasid_funcs = {
+		.attach = msm_iommu_pasid_attach,
+		.detach = msm_iommu_pasid_detach,
+		.map = msm_iommu_pasid_map,
+		.unmap = msm_iommu_pasid_unmap,
+		.destroy = msm_iommu_pasid_destroy,
+};
+
+static const struct arm_smmu_pasid_ops msm_iommu_pasid_ops = {
+	.install_pasid = install_pasid_cb,
+	.remove_pasid = remove_pasid_cb,
+};
+
+struct msm_mmu *msm_iommu_pasid_new(struct msm_mmu *parent)
+{
+	struct msm_iommu *parent_iommu = to_msm_iommu(parent);
+	struct msm_iommu_pasid *pasid;
+	int id;
+
+	if (!msm_mmu_has_feature(parent, MMU_FEATURE_PER_INSTANCE_TABLES))
+		return ERR_PTR(-EOPNOTSUPP);
+
+	pasid = kzalloc(sizeof(*pasid), GFP_KERNEL);
+	if (!pasid)
+		return ERR_PTR(-ENOMEM);
+
+	arm_smmu_add_pasid_ops(parent_iommu->domain, &msm_iommu_pasid_ops,
+		NULL);
+
+	id = iommu_sva_alloc_pasid(parent_iommu->domain, parent->dev);
+	if (id < 0) {
+		kfree(pasid);
+		return ERR_PTR(id);
+	}
+
+	pasid->pasid = id;
+	pasid->dev = parent->dev;
+
+	msm_mmu_init(&pasid->base, parent->dev, &pasid_funcs);
+
+	return &pasid->base;
+}
+
+/* Given a pasid return the TTBR and ASID associated with it */
+int msm_iommu_pasid_info(struct msm_mmu *mmu, u64 *ttbr, u32 *asid)
+{
+	struct msm_iommu_pasid *pasid;
+	struct pasid_entry *entry;
+
+	if (mmu->funcs->map != msm_iommu_pasid_map)
+		return -ENODEV;
+
+	pasid = to_msm_iommu_pasid(mmu);
+
+	if (!pasid->ttbr) {
+		/* Find the pasid entry in the hash */
+		hash_for_each_possible(pasid_table, entry, node, pasid->pasid) {
+			if (pasid->pasid == entry->pasid) {
+				pasid->ttbr = entry->ttbr;
+				pasid->asid = entry->asid;
+				goto out;
+			}
+		}
+
+		WARN(1, "Couldn't find the entry for pasid %d\n", pasid->pasid);
+		return -EINVAL;
+	}
+
+out:
+	if (*ttbr)
+		*ttbr = pasid->ttbr;
+
+	if (*asid)
+		*asid = pasid->asid;
+
+	return 0;
+}

index 85df78d71398..29436b9daa73 100644
@@ -30,6 +30,9 @@ struct msm_mmu_funcs {
 	void (*destroy)(struct msm_mmu *mmu);
 };
 
+/* MMU features */
+#define MMU_FEATURE_PER_INSTANCE_TABLES (1 << 0)
+
 struct msm_mmu {
 	const struct msm_mmu_funcs *funcs;
 	struct device *dev;

@@ -48,6 +51,9 @@ static inline void msm_mmu_init(struct msm_mmu *mmu, struct device *dev,
 struct msm_mmu *msm_iommu_new(struct device *dev, struct iommu_domain *domain);
 struct msm_mmu *msm_gpummu_new(struct device *dev, struct msm_gpu *gpu);
 
+struct msm_mmu *msm_iommu_pasid_new(struct msm_mmu *parent);
+int msm_iommu_pasid_info(struct msm_mmu *mmu, u64 *ttbr, u32 *asid);
+
 static inline void msm_mmu_set_fault_handler(struct msm_mmu *mmu, void *arg,
 		int (*handler)(void *arg, unsigned long iova, int flags))
 {

index 897b08135927..d92b009dfef4 100644
@@ -177,6 +177,9 @@ void msm_gem_address_space_put(struct msm_gem_address_space *aspace);
 struct msm_gem_address_space *
 msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
 		const char *name);
+struct msm_gem_address_space *
+msm_gem_address_space_create_instance(struct msm_mmu *parent, const char *name,
+		u64 start, u64 end);
 
 void msm_gem_submit_free(struct msm_gem_submit *submit);
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,

index ffbec224551b..d75b56119752 100644
@@ -92,12 +92,11 @@ msm_gem_map_vma(struct msm_gem_address_space *aspace,
 }
 
 struct msm_gem_address_space *
-msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
-		const char *name)
+msm_gem_address_space_new(struct msm_mmu *mmu, const char *name,
+		u64 start, u64 end)
 {
 	struct msm_gem_address_space *aspace;
-	u64 size = domain->geometry.aperture_end -
-		domain->geometry.aperture_start;
+	u64 size = end - start;
 
 	aspace = kzalloc(sizeof(*aspace), GFP_KERNEL);
 	if (!aspace)

@@ -105,12 +104,36 @@ msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
 
 	spin_lock_init(&aspace->lock);
 	aspace->name = name;
-	aspace->mmu = msm_iommu_new(dev, domain);
+	aspace->mmu = mmu;
 
-	drm_mm_init(&aspace->mm, (domain->geometry.aperture_start >> PAGE_SHIFT),
-		size >> PAGE_SHIFT);
+	drm_mm_init(&aspace->mm, (start >> PAGE_SHIFT), size >> PAGE_SHIFT);
 
 	kref_init(&aspace->kref);
 
 	return aspace;
 }
+
+struct msm_gem_address_space *
+msm_gem_address_space_create(struct device *dev, struct iommu_domain *domain,
+		const char *name)
+{
+	struct msm_mmu *mmu = msm_iommu_new(dev, domain);
+
+	if (IS_ERR(mmu))
+		return ERR_CAST(mmu);
+
+	return msm_gem_address_space_new(mmu, name,
+		domain->geometry.aperture_start,
+		domain->geometry.aperture_end);
+}
+
+struct msm_gem_address_space *
+msm_gem_address_space_create_instance(struct msm_mmu *parent, const char *name,
+		u64 start, u64 end)
+{
+	struct msm_mmu *instance = msm_iommu_pasid_new(parent);
+	if (IS_ERR(instance))
+		return ERR_CAST(instance);
+
+	return msm_gem_address_space_new(instance, name, start, end);
+}

67 lines not on this page · raw message

index 7d71860c4bee..9387d6085576 100644
@@ -45,6 +45,9 @@ struct a5xx_gpu {
 
 	atomic_t preempt_state;
 	struct timer_list preempt_timer;
+	struct a5xx_smmu_info *smmu_info;
+	struct drm_gem_object *smmu_info_bo;
+	uint64_t smmu_info_iova;
 };
 
 #define to_a5xx_gpu(x) container_of(x, struct a5xx_gpu, base)

@@ -132,6 +135,20 @@ struct a5xx_preempt_record {
  */
 #define A5XX_PREEMPT_COUNTER_SIZE (16 * 4)
 
+/*
+ * This is a global structure that the preemption code uses to switch in the
+ * pagetable for the preempted process - the code switches in whatever we
+ * after preempting in a new ring.
+ */
+struct a5xx_smmu_info {
+	uint32_t  magic;
+	uint32_t  _pad4;
+	uint64_t  ttbr0;
+	uint32_t  asid;
+	uint32_t  contextidr;
+};
+
+#define A5XX_SMMU_INFO_MAGIC 0x3618CDA3UL
 
 int a5xx_power_init(struct msm_gpu *gpu);
 void a5xx_gpmu_ucode_init(struct msm_gpu *gpu);

108 lines not on this page · raw message

index 17d0506d058c..b681edec4560 100644
@@ -558,6 +558,17 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	adreno_gpu_config.ioname = "kgsl_3d0_reg_memory";
 	adreno_gpu_config.irqname = "kgsl_3d0_irq";
 
+	if (adreno_is_a5xx(adreno_gpu)) {
+		/*
+		 * If possible use the TTBR1 virtual address space for all the
+		 * "global" buffer objects which are shared between processes.
+		 * This leaves the lower virtual address space open for
+		 * per-instance pagables if they are available
+		 */
+		adreno_gpu_config.va_start_global = 0xfffffff800000000ULL;
+		adreno_gpu_config.va_end_global = 0xfffffff8ffffffffULL;
+	}
+
 	adreno_gpu_config.va_start = SZ_16M;
 	adreno_gpu_config.va_end = 0xffffffff;
 

index d6b0e7b813f4..dc4b21ea3e65 100644
@@ -203,6 +203,11 @@ static inline int adreno_is_a530(struct adreno_gpu *gpu)
 	return gpu->revn == 530;
 }
 
+static inline bool adreno_is_a5xx(struct adreno_gpu *gpu)
+{
+	return ((gpu->revn >= 500) & (gpu->revn < 600));
+}
+
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value);
 const struct firmware *adreno_request_fw(struct adreno_gpu *adreno_gpu,
 		const char *fwname);

index cffce094aecb..fd71484d5894 100644
@@ -26,6 +26,7 @@
 struct msm_rbmemptrs {
 	volatile uint32_t rptr;
 	volatile uint32_t fence;
+	volatile uint64_t ttbr0;
 };
 
 struct msm_ringbuffer {

index 2b663435a3f7..31d1e7589892 100644
@@ -22,6 +22,7 @@
 #include "msm_fence.h"
 #include "msm_gpu.h"
 #include "msm_kms.h"
+#include "msm_gem.h"
 
 
 /*

@@ -511,7 +512,27 @@ static int context_init(struct drm_device *dev, struct drm_file *file)
 
 	msm_submitqueue_init(dev, ctx);
 
-	ctx->aspace = priv->gpu->aspace;
+	/* FIXME: Do we want a dynamic name of some sort? */
+	/* FIXME: We need a smarter way to set the range based on target */
+
+	ctx->aspace = msm_gem_address_space_create_instance(
+		priv->gpu->aspace->mmu, "gpu", 0x100000000, 0x1ffffffff);
+
+	if (IS_ERR(ctx->aspace)) {
+		int ret = PTR_ERR(ctx->aspace);
+
+		/*
+		 * if per-instance pagetables are not supported, fall back to
+		 * using the generic address space
+		 */
+		if (ret == -EOPNOTSUPP)
+			ctx->aspace = priv->gpu->aspace;
+		else {
+			kfree(ctx);
+			return ret;
+		}
+	}
+
 	file->driver_priv = ctx;
 
 	return 0;

@@ -527,8 +548,12 @@ static int msm_open(struct drm_device *dev, struct drm_file *file)
 	return context_init(dev, file);
 }
 
-static void context_close(struct msm_file_private *ctx)
+static void context_close(struct msm_drm_private *priv,
+		struct msm_file_private *ctx)
 {
+	if (ctx && ctx->aspace != priv->gpu->aspace)
+		msm_gem_address_space_put(ctx->aspace);
+
 	msm_submitqueue_close(ctx);
 	kfree(ctx);
 }

@@ -543,7 +568,7 @@ static void msm_postclose(struct drm_device *dev, struct drm_file *file)
 		priv->lastctx = NULL;
 	mutex_unlock(&dev->struct_mutex);
 
-	context_close(ctx);
+	context_close(priv, ctx);
 }
 
 static irqreturn_t msm_irq(int irq, void *arg)

index 0ba3d27f2300..a255b5d6c495 100644
@@ -599,7 +599,7 @@ int iommu_group_add_device(struct iommu_group *group, struct device *dev)
 	if (group->domain)
 		ret = __iommu_attach_device(group->domain, dev);
 	mutex_unlock(&group->mutex);
-	if (ret)
+	if (ret && ret != -ENOTSUPP)
 		goto err_put_group;
 
 	/* Notify any listeners about change to group. */

@@ -625,7 +625,8 @@ int iommu_group_add_device(struct iommu_group *group, struct device *dev)
 	sysfs_remove_link(&dev->kobj, "iommu_group");
 err_free_device:
 	kfree(device);
-	pr_err("Failed to add device %s to group %d: %d\n", dev_name(dev), group->id, ret);
+	if (ret != -ENOTSUPP)
+		pr_err("Failed to add device %s to group %d: %d\n", dev_name(dev), group->id, ret);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(iommu_group_add_device);

@@ -1238,8 +1239,16 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev)
 
 	ret = iommu_group_add_device(group, dev);
 	if (ret) {
-		iommu_group_put(group);
-		return ERR_PTR(ret);
+		/*
+		 * If the driver chooses not to bind the device, reset
+		 * group->domain so a new domain can be added later
+		 */
+		if (ret == -ENOTSUPP)
+			group->domain = NULL;
+		else {
+			iommu_group_put(group);
+			return ERR_PTR(ret);
+		}
 	}
 
 	return group;

index 100797a07be0..df6e4eacf727 100644
@@ -1080,6 +1080,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 		goto out_unlock;
 
 	cfg->cbndx = ret;
+
 	if (smmu->version < ARM_SMMU_V2) {
 		cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 		cfg->irptndx %= smmu->num_context_irqs;

@@ -1450,6 +1451,15 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
 	return 0;
 }
 
+/*
+ * This is a list of compatible strings for devices that wish to manage their
+ * own IOMMU space instead of the DMA IOMMU ops. Devices on this list will not
+ * allow themselves to be attached to a IOMMU_DOMAIN_DMA domain
+ */
+static const char *arm_smmu_dma_blacklist[] = {
+	"qcom,adreno",
+};
+
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 {
 	int ret;

@@ -1472,6 +1482,20 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	if (!fwspec->iommu_priv)
 		return -ENODEV;
 
+	/*
+	 * If this is the dfeault DMA domain, check to see if the device is on
+	 * the blacklist and reject if so
+	 */
+	if (domain->type == IOMMU_DOMAIN_DMA && dev->of_node) {
+		int i;
+
+		for(i = 0; i < ARRAY_SIZE(arm_smmu_dma_blacklist); i++) {
+			if (of_device_is_compatible(dev->of_node,
+				arm_smmu_dma_blacklist[i]))
+				return -ENOTSUPP;
+		}
+	}
+
 	smmu = fwspec_smmu(fwspec);
 	/* Ensure that the domain is finalised */
 	ret = arm_smmu_init_domain_context(domain, smmu);

index 30cd514d8f7c..2b663435a3f7 100644
@@ -502,6 +502,7 @@ static void load_gpu(struct drm_device *dev)
 
 static int context_init(struct drm_device *dev, struct drm_file *file)
 {
+	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_file_private *ctx;
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);

@@ -510,6 +511,7 @@ static int context_init(struct drm_device *dev, struct drm_file *file)
 
 	msm_submitqueue_init(dev, ctx);
 
+	ctx->aspace = priv->gpu->aspace;
 	file->driver_priv = ctx;
 
 	return 0;

@@ -683,17 +685,6 @@ static int msm_ioctl_gem_cpu_fini(struct drm_device *dev, void *data,
 	return ret;
 }
 
-static int msm_ioctl_gem_info_iova(struct drm_device *dev,
-		struct drm_gem_object *obj, uint64_t *iova)
-{
-	struct msm_drm_private *priv = dev->dev_private;
-
-	if (!priv->gpu)
-		return -EINVAL;
-
-	return msm_gem_get_iova(obj, priv->gpu->aspace, iova);
-}
-
 static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 		struct drm_file *file)
 {

@@ -709,9 +700,10 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data,
 		return -ENOENT;
 
 	if (args->flags & MSM_INFO_IOVA) {
+		struct msm_file_private *ctx = file->driver_priv;
 		uint64_t iova;
 
-		ret = msm_ioctl_gem_info_iova(dev, obj, &iova);
+		ret = msm_gem_get_iova(obj, ctx->aspace, &iova);
 		if (!ret)
 			args->offset = iova;
 	} else {

index 48ed5b9a8580..897b08135927 100644
@@ -58,6 +58,7 @@ struct msm_file_private {
 	rwlock_t queuelock;
 	struct list_head submitqueues;
 	int queueid;
+	struct msm_gem_address_space *aspace;
 };
 
 enum msm_mdp_plane_property {

index c5d9bd3e47a8..fe8b3aa7d76f 100644
@@ -138,6 +138,7 @@ void msm_gem_vunmap(struct drm_gem_object *obj, enum msm_gem_lock subclass);
 struct msm_gem_submit {
 	struct drm_device *dev;
 	struct msm_gpu *gpu;
+	struct msm_gem_address_space *aspace;
 	struct list_head node;   /* node in ring submit list */
 	struct list_head bo_list;
 	struct ww_acquire_ctx ticket;

index 7bd83e0afa97..d5dffcba9919 100644
@@ -31,8 +31,8 @@
 #define BO_PINNED   0x2000
 
 static struct msm_gem_submit *submit_create(struct drm_device *dev,
-		struct msm_gpu *gpu, struct msm_gpu_submitqueue *queue,
-		uint32_t nr_bos, uint32_t nr_cmds)
+		struct msm_gpu *gpu, struct msm_gem_address_space *aspace,
+		struct msm_gpu_submitqueue *queue, uint32_t nr_bos, uint32_t nr_cmds)
 {
 	struct msm_gem_submit *submit;
 	uint64_t sz = sizeof(*submit) + ((u64)nr_bos * sizeof(submit->bos[0])) +

@@ -46,6 +46,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 		return NULL;
 
 	submit->dev = dev;
+	submit->aspace = aspace;
 	submit->gpu = gpu;
 	submit->fence = NULL;
 	submit->pid = get_pid(task_pid(current));

@@ -167,7 +168,7 @@ static void submit_unlock_unpin_bo(struct msm_gem_submit *submit,
 	struct msm_gem_object *msm_obj = submit->bos[i].obj;
 
 	if (submit->bos[i].flags & BO_PINNED)
-		msm_gem_put_iova(&msm_obj->base, submit->gpu->aspace);
+		msm_gem_put_iova(&msm_obj->base, submit->aspace);
 
 	if (submit->bos[i].flags & BO_LOCKED)
 		ww_mutex_unlock(&msm_obj->resv->lock);

@@ -270,7 +271,7 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
 
 		/* if locking succeeded, pin bo: */
 		ret = msm_gem_get_iova(&msm_obj->base,
-				submit->gpu->aspace, &iova);
+				submit->aspace, &iova);
 
 		if (ret)
 			break;

@@ -471,7 +472,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		}
 	}
 
-	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
+	submit = submit_create(dev, gpu, ctx->aspace, queue, args->nr_bos, args->nr_cmds);
 	if (!submit) {
 		ret = -ENOMEM;
 		goto out_unlock;

index 1c09acfb4028..2f45bea04221 100644
@@ -551,7 +551,7 @@ static void retire_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
 		/* move to inactive: */
 		msm_gem_move_to_inactive(&msm_obj->base);
-		msm_gem_put_iova(&msm_obj->base, gpu->aspace);
+		msm_gem_put_iova(&msm_obj->base, submit->aspace);
 		drm_gem_object_put(&msm_obj->base);
 	}
 

@@ -635,8 +635,7 @@ void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 
 		/* submit takes a reference to the bo and iova until retired: */
 		drm_gem_object_get(&msm_obj->base);
-		msm_gem_get_iova(&msm_obj->base,
-				submit->gpu->aspace, &iova);
+		msm_gem_get_iova(&msm_obj->base, submit->aspace, &iova);
 
 		if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
 			msm_gem_move_to_active(&msm_obj->base, gpu, true, submit->fence);