Skip to contentSkip to conversation

Add PASID support to SMMUv3 unmanaged domains

PATCH Superseded Michael Shavit 5 participants 35 messages Jun 6, 2023 – Jul 11, 2023

Subsystems (3)

v2 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v2 cover Add PASID support to SMMUv3 unmanaged domains

Hi all,

This patch series implements the set_dev_pasid operation for DMA and UNMANAGED iommu domains.

As a first step, the patch series refactors stage 1 domains so that they describe a single CD entry. On attach, stage 1 domains are inserted into a CD table that is now owned by the arm_smmu_master struct. Note that this does not preclude from attaching domains that represent a CD table, such as for the proposed iommufd NESTED domains.

The patch series also heavily refactors the arm-smmu-v3-sva implementation. The first set of clean-up patches were enabled by earlier changes in the iommu pasid framework. Later patches take advantage of the arm-smmu-v3 changes and makes use of its set_dev_pasid implementation.

This patch series is also available on gerrit with Jean's SMMU test engine patches rebased on top:
https://linux-review.googlesource.com/id/I0fcd9adc058d1c58a12d2599cc82fba73da7697a
This allowed testing of basic SVA functionality (e.g.: attaching, page fault handling, and detaching).

Thanks,
Michael Shavit

Changelog
v2:
* Reworded cover letter and commits based on v1 feedback.
* Added SVA clean-up and refactor.
* Split and reworked iommu/arm-smmu-v3: Move cdtable to
arm_smmu_master
* A few other small bug fixes and cosmetics.
v1: https://lore.kernel.org/all/[email protected]/

Michael Shavit (18):
iommu/arm-smmu-v3: Move ctx_desc out of s1_cfg iommu/arm-smmu-v3: Add smmu_s1_cfg to smmu_master iommu/arm-smmu-v3: Refactor write_strtab_ent iommu/arm-smmu-v3: Refactor write_ctx_desc iommu/arm-smmu-v3: Use the master-owned s1_cfg iommu/arm-smmu-v3: Simplify arm_smmu_enable_ats iommu/arm-smmu-v3: Keep track of attached ssids iommu/arm-smmu-v3: Add helper for atc invalidation iommu/arm-smmu-v3: Implement set_dev_pasid iommu/arm-smmu-v3-sva: Remove bond refcount iommu/arm-smmu-v3-sva: Clean unused iommu_sva iommu/arm-smmu-v3-sva: Remove arm_smmu_bond iommu/arm-smmu-v3-sva: Add check when enabling sva iommu/arm-smmu-v3: Support domains with shared CDs iommu/arm-smmu-v3: Allow more re-use for SVA iommu/arm-smmu-v3-sva: Attach S1_SHARED_CD domain iommu/arm-smmu-v3-sva: Alloc notifier for {smmu,mn} iommu/arm-smmu-v3-sva: Remove atc_inv_domain_ssid

 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   | 164 +++---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 493 ++++++++++++------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |  62 ++-
 3 files changed, 457 insertions(+), 262 deletions(-)

base-commit: 884fe9da1b7ccbea31b118f902fbc78f58366b4a

Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#1 lore raw

PATCH v2 1/18 iommu/arm-smmu-v3: Move ctx_desc out of s1_cfg

s1_cfg describes the CD table that is inserted into an SMMU's STEs. It's weird for s1_cfg to also own ctx_desc which describes a CD that is inserted into that table. It is more appropriate for arm_smmu_domain to own ctx_desc.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   |  2 +-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 23 +++++++--------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   | 28 ++++++++++---------
 3 files changed, 28 insertions(+), 25 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +1 -1
index a5a63b1c947eb..968559d625c40 100644
@@ -62,7 +62,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
 		return cd;
 	}
 
-	smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd);
+	smmu_domain = container_of(cd, struct arm_smmu_domain, cd);
 	smmu = smmu_domain->smmu;
 
 	ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd,
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +12 -11
index 3fd83fb757227..beff04b897718 100644
@@ -1863,7 +1863,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
 	 * careful, 007.
 	 */
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
+		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
 	} else {
 		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
@@ -1946,7 +1946,7 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
 				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
-		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
+		cmd.tlbi.asid	= smmu_domain->cd.asid;
 	} else {
 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
@@ -2077,7 +2077,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 		mutex_lock(&arm_smmu_asid_lock);
 		if (cfg->cdcfg.cdtab)
 			arm_smmu_free_cd_tables(smmu_domain);
-		arm_smmu_free_asid(&cfg->cd);
+		arm_smmu_free_asid(&smmu_domain->cd);
 		mutex_unlock(&arm_smmu_asid_lock);
 	} else {
 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
@@ -2096,13 +2096,14 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	u32 asid;
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
+	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 
-	refcount_set(&cfg->cd.refs, 1);
+	refcount_set(&cd->refs, 1);
 
 	/* Prevent SVA from modifying the ASID until it is written to the CD */
 	mutex_lock(&arm_smmu_asid_lock);
-	ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
+	ret = xa_alloc(&arm_smmu_asid_xa, &asid, cd,
 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
 	if (ret)
 		goto out_unlock;
@@ -2115,23 +2116,23 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	if (ret)
 		goto out_free_asid;
 
-	cfg->cd.asid	= (u16)asid;
-	cfg->cd.ttbr	= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
-	cfg->cd.tcr	= FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
+	cd->asid	= (u16)asid;
+	cd->ttbr	= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
+	cd->tcr		= FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
 			  FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
 			  FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
 			  FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
 			  FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
 			  FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
 			  CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
-	cfg->cd.mair	= pgtbl_cfg->arm_lpae_s1_cfg.mair;
+	cd->mair	= pgtbl_cfg->arm_lpae_s1_cfg.mair;
 
 	/*
 	 * Note that this will end up calling arm_smmu_sync_cd() before
 	 * the master has been added to the devices list for this domain.
 	 * This isn't an issue because the STE hasn't been installed yet.
 	 */
-	ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
+	ret = arm_smmu_write_ctx_desc(smmu_domain, 0, cd);
 	if (ret)
 		goto out_free_cd_tables;
 
@@ -2141,7 +2142,7 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 out_free_cd_tables:
 	arm_smmu_free_cd_tables(smmu_domain);
 out_free_asid:
-	arm_smmu_free_asid(&cfg->cd);
+	arm_smmu_free_asid(cd);
 out_unlock:
 	mutex_unlock(&arm_smmu_asid_lock);
 	return ret;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +15 -13
index b574c58a34876..68d519f21dbd8 100644
@@ -593,7 +593,6 @@ struct arm_smmu_ctx_desc_cfg {
 
 struct arm_smmu_s1_cfg {
 	struct arm_smmu_ctx_desc_cfg	cdcfg;
-	struct arm_smmu_ctx_desc	cd;
 	u8				s1fmt;
 	u8				s1cdmax;
 };
@@ -707,25 +706,28 @@ enum arm_smmu_domain_stage {
 };
 
 struct arm_smmu_domain {
-	struct arm_smmu_device		*smmu;
-	struct mutex			init_mutex; /* Protects smmu pointer */
+	struct arm_smmu_device			*smmu;
+	struct mutex				init_mutex; /* Protects smmu pointer */
 
-	struct io_pgtable_ops		*pgtbl_ops;
-	bool				stall_enabled;
-	atomic_t			nr_ats_masters;
+	struct io_pgtable_ops			*pgtbl_ops;
+	bool					stall_enabled;
+	atomic_t				nr_ats_masters;
 
-	enum arm_smmu_domain_stage	stage;
+	enum arm_smmu_domain_stage		stage;
 	union {
-		struct arm_smmu_s1_cfg	s1_cfg;
-		struct arm_smmu_s2_cfg	s2_cfg;
+		struct {
+		struct arm_smmu_ctx_desc	cd;
+		struct arm_smmu_s1_cfg		s1_cfg;
+		};
+		struct arm_smmu_s2_cfg		s2_cfg;
 	};
 
-	struct iommu_domain		domain;
+	struct iommu_domain			domain;
 
-	struct list_head		devices;
-	spinlock_t			devices_lock;
+	struct list_head			devices;
+	spinlock_t				devices_lock;
 
-	struct list_head		mmu_notifiers;
+	struct list_head			mmu_notifiers;
 };
 
 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#2 lore raw

PATCH v2 2/18 iommu/arm-smmu-v3: Add smmu_s1_cfg to smmu_master

In reply to Michael Shavit

Except for Nested domains, arm_smmu_master will own the STEs that are inserted into the arm_smmu_device's STE table.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 28 +++++++++++++--------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  1 +
 2 files changed, 18 insertions(+), 11 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +17 -11
index beff04b897718..023769f5ca79a 100644
@@ -1126,15 +1126,16 @@ int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
 	return 0;
 }
 
-static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
+static int arm_smmu_init_s1_cfg(struct arm_smmu_master *master,
+				struct arm_smmu_s1_cfg *cfg)
 {
 	int ret;
 	size_t l1size;
 	size_t max_contexts;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
+	struct arm_smmu_device *smmu = master->smmu;
 	struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
 
+	cfg->s1cdmax = master->ssid_bits;
 	max_contexts = 1 << cfg->s1cdmax;
 
 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
@@ -1175,12 +1176,11 @@ static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
 	return ret;
 }
 
-static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
+static void arm_smmu_free_cd_tables(struct arm_smmu_device *smmu,
+				    struct arm_smmu_ctx_desc_cfg *cdcfg)
 {
 	int i;
 	size_t size, l1size;
-	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
 
 	if (cdcfg->l1_desc) {
 		size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
@@ -2076,7 +2076,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 		/* Prevent SVA from touching the CD while we're freeing it */
 		mutex_lock(&arm_smmu_asid_lock);
 		if (cfg->cdcfg.cdtab)
-			arm_smmu_free_cd_tables(smmu_domain);
+			arm_smmu_free_cd_tables(smmu, &cfg->cdcfg);
 		arm_smmu_free_asid(&smmu_domain->cd);
 		mutex_unlock(&arm_smmu_asid_lock);
 	} else {
@@ -2108,11 +2108,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	if (ret)
 		goto out_unlock;
 
-	cfg->s1cdmax = master->ssid_bits;
-
 	smmu_domain->stall_enabled = master->stall_enabled;
 
-	ret = arm_smmu_alloc_cd_tables(smmu_domain);
+	ret = arm_smmu_init_s1_cfg(master, cfg);
 	if (ret)
 		goto out_free_asid;
 
@@ -2140,7 +2138,7 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	return 0;
 
 out_free_cd_tables:
-	arm_smmu_free_cd_tables(smmu_domain);
+	arm_smmu_free_cd_tables(smmu, &cfg->cdcfg);
 out_free_asid:
 	arm_smmu_free_asid(cd);
 out_unlock:
@@ -2704,6 +2702,13 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
 		master->stall_enabled = true;
 
+	ret = arm_smmu_init_s1_cfg(master, &master->owned_s1_cfg);
+	if (ret) {
+		arm_smmu_disable_pasid(master);
+		arm_smmu_remove_master(master);
+		goto err_free_master;
+	}
+
 	return &smmu->iommu;
 
 err_free_master:
@@ -2719,6 +2724,7 @@ static void arm_smmu_release_device(struct device *dev)
 	if (WARN_ON(arm_smmu_master_sva_enabled(master)))
 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
 	arm_smmu_detach_dev(master);
+	arm_smmu_free_cd_tables(master->smmu, &master->owned_s1_cfg.cdcfg);
 	arm_smmu_disable_pasid(master);
 	arm_smmu_remove_master(master);
 	kfree(master);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +1 -0
index 68d519f21dbd8..053cc14c23969 100644
@@ -688,6 +688,7 @@ struct arm_smmu_master {
 	struct arm_smmu_domain		*domain;
 	struct list_head		domain_head;
 	struct arm_smmu_stream		*streams;
+	struct arm_smmu_s1_cfg		owned_s1_cfg;
 	unsigned int			num_streams;
 	bool				ats_enabled;
 	bool				stall_enabled;
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#3 lore raw

PATCH v2 3/18 iommu/arm-smmu-v3: Refactor write_strtab_ent

In reply to Michael Shavit

Explicity keep track of the s1_cfg and s2_cfg that are attached to a master in arm_smmu_master, regardless of whether they are owned by arm_smmu_master, arm_smmu_domain or userspace.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 37 +++++++++------------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  2 ++
 2 files changed, 17 insertions(+), 22 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +15 -22
index 023769f5ca79a..d79c6ef5d6ed4 100644
@@ -1269,10 +1269,9 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 	 */
 	u64 val = le64_to_cpu(dst[0]);
 	bool ste_live = false;
-	struct arm_smmu_device *smmu = NULL;
+	struct arm_smmu_device *smmu = master->smmu;
 	struct arm_smmu_s1_cfg *s1_cfg = NULL;
 	struct arm_smmu_s2_cfg *s2_cfg = NULL;
-	struct arm_smmu_domain *smmu_domain = NULL;
 	struct arm_smmu_cmdq_ent prefetch_cmd = {
 		.opcode		= CMDQ_OP_PREFETCH_CFG,
 		.prefetch	= {
@@ -1280,24 +1279,10 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 		},
 	};
 
-	if (master) {
-		smmu_domain = master->domain;
-		smmu = master->smmu;
-	}
-
-	if (smmu_domain) {
-		switch (smmu_domain->stage) {
-		case ARM_SMMU_DOMAIN_S1:
-			s1_cfg = &smmu_domain->s1_cfg;
-			break;
-		case ARM_SMMU_DOMAIN_S2:
-		case ARM_SMMU_DOMAIN_NESTED:
-			s2_cfg = &smmu_domain->s2_cfg;
-			break;
-		default:
-			break;
-		}
-	}
+	if (master->s1_cfg)
+		s1_cfg = master->s1_cfg;
+	else if (master->s2_cfg)
+		s2_cfg = master->s2_cfg;
 
 	if (val & STRTAB_STE_0_V) {
 		switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
@@ -1319,8 +1304,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
 	val = STRTAB_STE_0_V;
 
 	/* Bypass/fault */
-	if (!smmu_domain || !(s1_cfg || s2_cfg)) {
-		if (!smmu_domain && disable_bypass)
+	if (!(s1_cfg || s2_cfg)) {
+		if (disable_bypass)
 			val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
 		else
 			val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
@@ -2401,6 +2386,8 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
 
 	master->domain = NULL;
 	master->ats_enabled = false;
+	master->s1_cfg = NULL;
+	master->s2_cfg = NULL;
 	arm_smmu_install_ste_for_dev(master);
 }
 
@@ -2454,6 +2441,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	}
 
 	master->domain = smmu_domain;
+	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+		master->s1_cfg = &smmu_domain->s1_cfg;
+	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2 ||
+		   smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED) {
+		master->s2_cfg = &smmu_domain->s2_cfg;
+	}
 
 	/*
 	 * The SMMU does not support enabling ATS with bypass. When the STE is
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +2 -0
index 053cc14c23969..3c614fbe2b8b9 100644
@@ -689,6 +689,8 @@ struct arm_smmu_master {
 	struct list_head		domain_head;
 	struct arm_smmu_stream		*streams;
 	struct arm_smmu_s1_cfg		owned_s1_cfg;
+	struct arm_smmu_s1_cfg		*s1_cfg;
+	struct arm_smmu_s2_cfg		*s2_cfg;
 	unsigned int			num_streams;
 	bool				ats_enabled;
 	bool				stall_enabled;
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#4 lore raw

PATCH v2 4/18 iommu/arm-smmu-v3: Refactor write_ctx_desc

In reply to Michael Shavit

Update arm_smmu_write_ctx_desc and downstream functions to be agnostic of the CD table's ownership. Note that in practice, it will only be called in cases where the table is owned by arm_smmu_master. Whether arm_smmu_write_ctx_desc will trigger a sync is also made part of the API.

Note that this change isn't a nop refactor since SVA will call arm_smmu_write_ctx_desc in a loop for every master the domain is attached to despite the fact that they all share the same CD table. Note that the next commit ceases this sharing of the s1_cfg which makes that loop necessary.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   | 36 ++++++++-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 80 +++++++++++--------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |  6 +-
 3 files changed, 81 insertions(+), 41 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +32 -4
index 968559d625c40..48fa8eb271a45 100644
@@ -45,10 +45,12 @@ static struct arm_smmu_ctx_desc *
 arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
 {
 	int ret;
+	unsigned long flags;
 	u32 new_asid;
 	struct arm_smmu_ctx_desc *cd;
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_domain *smmu_domain;
+	struct arm_smmu_master *master;
 
 	cd = xa_load(&arm_smmu_asid_xa, asid);
 	if (!cd)
@@ -80,7 +82,11 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
 	 * be some overlap between use of both ASIDs, until we invalidate the
 	 * TLB.
 	 */
-	arm_smmu_write_ctx_desc(smmu_domain, 0, cd);
+	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+		arm_smmu_write_ctx_desc(smmu, master->s1_cfg, master, 0, cd);
+	}
+	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
 	/* Invalidate TLB entries previously associated with that context */
 	arm_smmu_tlb_inv_asid(smmu, asid);
@@ -211,6 +217,8 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 {
 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+	struct arm_smmu_master *master;
+	unsigned long flags;
 
 	mutex_lock(&sva_lock);
 	if (smmu_mn->cleared) {
@@ -222,7 +230,12 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
 	 * but disable translation.
 	 */
-	arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, &quiet_cd);
+	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
+					mm->pasid, &quiet_cd);
+	}
+	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
 	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
 	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
@@ -248,8 +261,10 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 			  struct mm_struct *mm)
 {
 	int ret;
+	unsigned long flags;
 	struct arm_smmu_ctx_desc *cd;
 	struct arm_smmu_mmu_notifier *smmu_mn;
+	struct arm_smmu_master *master;
 
 	list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
 		if (smmu_mn->mn.mm == mm) {
@@ -279,7 +294,12 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 		goto err_free_cd;
 	}
 
-	ret = arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, cd);
+	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+		ret = arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg,
+					      master, mm->pasid, cd);
+	}
+	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 	if (ret)
 		goto err_put_notifier;
 
@@ -296,15 +316,23 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 
 static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 {
+	unsigned long flags;
 	struct mm_struct *mm = smmu_mn->mn.mm;
 	struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
+	struct arm_smmu_master *master;
 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
 
 	if (!refcount_dec_and_test(&smmu_mn->refs))
 		return;
 
 	list_del(&smmu_mn->list);
-	arm_smmu_write_ctx_desc(smmu_domain, mm->pasid, NULL);
+
+	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
+					mm->pasid, NULL);
+	}
+	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 
 	/*
 	 * If we went through clear(), we've already invalidated, and no
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +45 -35

178 lines not on this page · raw message

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +4 -2
index 3c614fbe2b8b9..00a493442d6f9 100644
@@ -595,6 +595,7 @@ struct arm_smmu_s1_cfg {
 	struct arm_smmu_ctx_desc_cfg	cdcfg;
 	u8				s1fmt;
 	u8				s1cdmax;
+	bool				stall_enabled;
 };
 
 struct arm_smmu_s2_cfg {
@@ -713,7 +714,6 @@ struct arm_smmu_domain {
 	struct mutex				init_mutex; /* Protects smmu pointer */
 
 	struct io_pgtable_ops			*pgtbl_ops;
-	bool					stall_enabled;
 	atomic_t				nr_ats_masters;
 
 	enum arm_smmu_domain_stage		stage;
@@ -742,7 +742,9 @@ extern struct xarray arm_smmu_asid_xa;
 extern struct mutex arm_smmu_asid_lock;
 extern struct arm_smmu_ctx_desc quiet_cd;
 
-int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
+int arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
+			    struct arm_smmu_s1_cfg *s1_cfg,
+			    struct arm_smmu_master *smmu_master, int ssid,
 			    struct arm_smmu_ctx_desc *cd);
 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#5 lore raw

PATCH v2 5/18 iommu/arm-smmu-v3: Use the master-owned s1_cfg

In reply to Michael Shavit

Insert CDs for STAGE_1 domains into a CD table owned by the arm_smmu_master. Remove the CD table that was owned by arm_smmu_domain.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 43 ++++++---------------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  3 --
 2 files changed, 12 insertions(+), 34 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +12 -31
index b6f7cf60f8f3d..08f440fe1da6d 100644
@@ -2065,12 +2065,8 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 
 	/* Free the CD and ASID, if we allocated them */
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
-
 		/* Prevent SVA from touching the CD while we're freeing it */
 		mutex_lock(&arm_smmu_asid_lock);
-		if (cfg->cdcfg.cdtab)
-			arm_smmu_free_cd_tables(smmu, &cfg->cdcfg);
 		arm_smmu_free_asid(&smmu_domain->cd);
 		mutex_unlock(&arm_smmu_asid_lock);
 	} else {
@@ -2082,14 +2078,13 @@ static void arm_smmu_domain_free(struct iommu_domain *domain)
 	kfree(smmu_domain);
 }
 
-static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
+static int arm_smmu_domain_finalise_cd(struct arm_smmu_domain *smmu_domain,
 				       struct arm_smmu_master *master,
 				       struct io_pgtable_cfg *pgtbl_cfg)
 {
 	int ret;
 	u32 asid;
 	struct arm_smmu_device *smmu = smmu_domain->smmu;
-	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
 	struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 
@@ -2102,10 +2097,6 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 	if (ret)
 		goto out_unlock;
 
-	ret = arm_smmu_init_s1_cfg(master, cfg);
-	if (ret)
-		goto out_free_asid;
-
 	cd->asid	= (u16)asid;
 	cd->ttbr	= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
 	cd->tcr		= FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
@@ -2117,19 +2108,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
 			  CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
 	cd->mair	= pgtbl_cfg->arm_lpae_s1_cfg.mair;
 
-	ret = arm_smmu_write_ctx_desc(smmu, cfg,
-				      NULL /*Not attached to a master yet */,
-				      0, cd);
-	if (ret)
-		goto out_free_cd_tables;
-
 	mutex_unlock(&arm_smmu_asid_lock);
 	return 0;
 
-out_free_cd_tables:
-	arm_smmu_free_cd_tables(smmu, &cfg->cdcfg);
-out_free_asid:
-	arm_smmu_free_asid(cd);
 out_unlock:
 	mutex_unlock(&arm_smmu_asid_lock);
 	return ret;
@@ -2192,7 +2173,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain,
 		ias = min_t(unsigned long, ias, VA_BITS);
 		oas = smmu->ias;
 		fmt = ARM_64_LPAE_S1;
-		finalise_stage_fn = arm_smmu_domain_finalise_s1;
+		finalise_stage_fn = arm_smmu_domain_finalise_cd;
 		break;
 	case ARM_SMMU_DOMAIN_NESTED:
 	case ARM_SMMU_DOMAIN_S2:
@@ -2439,20 +2420,20 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	} else if (smmu_domain->smmu != smmu) {
 		ret = -EINVAL;
 		goto out_unlock;
-	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
-		   master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
-		ret = -EINVAL;
-		goto out_unlock;
-	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
-		   smmu_domain->s1_cfg.stall_enabled !=
-			   master->stall_enabled) {
-		ret = -EINVAL;
-		goto out_unlock;
 	}
 
 	master->domain = smmu_domain;
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		master->s1_cfg = &smmu_domain->s1_cfg;
+		master->s1_cfg = &master->owned_s1_cfg;
+		ret = arm_smmu_write_ctx_desc(
+			smmu,
+			master->s1_cfg, NULL /*Not attached to a master yet */,
+			0, &smmu_domain->cd);
+		if (ret) {
+			master->s1_cfg = NULL;
+			master->domain = NULL;
+			goto out_unlock;
+		}
 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2 ||
 		   smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED) {
 		master->s2_cfg = &smmu_domain->s2_cfg;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +0 -3
index 00a493442d6f9..dff0fa8345462 100644
@@ -718,10 +718,7 @@ struct arm_smmu_domain {
 
 	enum arm_smmu_domain_stage		stage;
 	union {
-		struct {
 		struct arm_smmu_ctx_desc	cd;
-		struct arm_smmu_s1_cfg		s1_cfg;
-		};
 		struct arm_smmu_s2_cfg		s2_cfg;
 	};
 
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#6 lore raw

PATCH v2 6/18 iommu/arm-smmu-v3: Simplify arm_smmu_enable_ats

In reply to Michael Shavit

arm_smmu_enable_ats's call to inv_domain would trigger an invalidation for all masters that a domain is attached to everytime it's attached to another ATS-enabled master. It doesn't seem like those invalidations are necessary, and it's easier to reason about arm_smmu_enable_ats if it only issues invalidation commands for the current master.

---
v1->v2: Fix commit message wrapping

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +1 -1
index 08f440fe1da6d..dc7a59e87a2b4 100644
@@ -2286,7 +2286,7 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)
 	pdev = to_pci_dev(master->dev);
 
 	atomic_inc(&smmu_domain->nr_ats_masters);
-	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
+	arm_smmu_atc_inv_master(master);
 	if (pci_enable_ats(pdev, stu))
 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
 }
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#7 lore raw

PATCH v2 7/18 iommu/arm-smmu-v3: Keep track of attached ssids

In reply to Michael Shavit

The arm-smmu-v3 driver keeps track of all masters that a domain is attached to so that it can re-write their STEs when the domain's ASID is upated by SVA. This tracking is also used to invalidate ATCs on all masters that a domain is attached to.

This change introduces a new data structures to track all the CD entries that a domain is attached to. This change is a pre-requisite to allow domain attachment on non 0 SSIDs.

---
The arm_smmu_atc_inv_domain_ssid function is only temporarily introduced to make these changes atomic, but is eventually removed in latter SVA refactoring patches.

v1->v2: Fix arm_smmu_atc_inv_cmd_set_ssid and other cosmetic changes

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   | 53 +++++++----
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 88 ++++++++++++-------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   | 18 ++--
 3 files changed, 105 insertions(+), 54 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +35 -18
index 48fa8eb271a45..d07c08b53c5cf 100644
@@ -51,6 +51,7 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_domain *smmu_domain;
 	struct arm_smmu_master *master;
+	struct arm_smmu_attached_domain *attached_domain;
 
 	cd = xa_load(&arm_smmu_asid_xa, asid);
 	if (!cd)
@@ -82,11 +83,14 @@ arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
 	 * be some overlap between use of both ASIDs, until we invalidate the
 	 * TLB.
 	 */
-	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
-		arm_smmu_write_ctx_desc(smmu, master->s1_cfg, master, 0, cd);
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
+			    domain_head) {
+		master = attached_domain->master;
+		arm_smmu_write_ctx_desc(smmu, master->s1_cfg, master,
+					attached_domain->ssid, cd);
 	}
-	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 
 	/* Invalidate TLB entries previously associated with that context */
 	arm_smmu_tlb_inv_asid(smmu, asid);
@@ -210,7 +214,7 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
 		arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
 					    PAGE_SIZE, false, smmu_domain);
-	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size);
+	arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, start, size);
 }
 
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -218,6 +222,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
 	struct arm_smmu_master *master;
+	struct arm_smmu_attached_domain *attached_domain;
 	unsigned long flags;
 
 	mutex_lock(&sva_lock);
@@ -230,15 +235,21 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
 	 * but disable translation.
 	 */
-	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
-		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
-					mm->pasid, &quiet_cd);
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
+			    domain_head) {
+		master = attached_domain->master;
+		/*
+		 * SVA domains piggyback on the attached_domain with SSID 0.
+		 */
+		if (attached_domain->ssid == 0)
+			arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg,
+						master, mm->pasid, &quiet_cd);
 	}
-	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 
 	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
-	arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
+	arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
 
 	smmu_mn->cleared = true;
 	mutex_unlock(&sva_lock);
@@ -265,6 +276,7 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 	struct arm_smmu_ctx_desc *cd;
 	struct arm_smmu_mmu_notifier *smmu_mn;
 	struct arm_smmu_master *master;
+	struct arm_smmu_attached_domain *attached_domain;
 
 	list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
 		if (smmu_mn->mn.mm == mm) {
@@ -294,12 +306,14 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 		goto err_free_cd;
 	}
 
-	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
+			    domain_head) {
+		master = attached_domain->master;
 		ret = arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg,
 					      master, mm->pasid, cd);
 	}
-	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 	if (ret)
 		goto err_put_notifier;
 
@@ -319,6 +333,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 	unsigned long flags;
 	struct mm_struct *mm = smmu_mn->mn.mm;
 	struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
+	struct arm_smmu_attached_domain *attached_domain;
 	struct arm_smmu_master *master;
 	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
 
@@ -327,12 +342,14 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 
 	list_del(&smmu_mn->list);
 
-	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
-	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
+			    domain_head) {
+		master = attached_domain->master;
 		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
 					mm->pasid, NULL);
 	}
-	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 
 	/*
 	 * If we went through clear(), we've already invalidated, and no
@@ -340,7 +357,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 	 */
 	if (!smmu_mn->cleared) {
 		arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
-		arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
+		arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
 	}
 
 	/* Frees smmu_mn */
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +57 -31

206 lines not on this page · raw message

drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +13 -5
index dff0fa8345462..6929590530367 100644
@@ -682,11 +682,19 @@ struct arm_smmu_stream {
 	struct rb_node			node;
 };
 
+/* List of {masters, ssid} that a domain is attached to */
+struct arm_smmu_attached_domain {
+	struct list_head	domain_head;
+	struct arm_smmu_domain  *domain;
+	struct arm_smmu_master  *master;
+	int			ssid;
+};
+
 /* SMMU private data for each master */
 struct arm_smmu_master {
 	struct arm_smmu_device		*smmu;
 	struct device			*dev;
-	struct arm_smmu_domain		*domain;
+	struct arm_smmu_attached_domain	non_pasid_domain;
 	struct list_head		domain_head;
 	struct arm_smmu_stream		*streams;
 	struct arm_smmu_s1_cfg		owned_s1_cfg;
@@ -724,8 +732,8 @@ struct arm_smmu_domain {
 
 	struct iommu_domain			domain;
 
-	struct list_head			devices;
-	spinlock_t				devices_lock;
+	struct list_head			attached_domains;
+	spinlock_t				attached_domains_lock;
 
 	struct list_head			mmu_notifiers;
 };
@@ -748,8 +756,8 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 				 size_t granule, bool leaf,
 				 struct arm_smmu_domain *smmu_domain);
 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
-			    unsigned long iova, size_t size);
+int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
+				 unsigned long iova, size_t size);
 
 #ifdef CONFIG_ARM_SMMU_V3_SVA
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#8 lore raw

PATCH v2 8/18 iommu/arm-smmu-v3: Add helper for atc invalidation

In reply to Michael Shavit

This will be used to invalidate ATC entries made on an SSID for a master when detaching a domain with pasid.

---
v1->v2: Make use of arm_smmu_atc_inv_cmd_set_ssid

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +8 -1
index 70580ba7065dc..176013bb974b8 100644
@@ -1784,13 +1784,15 @@ arm_smmu_atc_inv_to_cmd(unsigned long iova, size_t size,
 	cmd->atc.size	= log2_span;
 }
 
-static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
+static int arm_smmu_atc_inv_master_ssid(struct arm_smmu_master *master,
+					int ssid)
 {
 	int i;
 	struct arm_smmu_cmdq_ent cmd;
 	struct arm_smmu_cmdq_batch cmds;
 
 	arm_smmu_atc_inv_to_cmd(0, 0, &cmd);
+	arm_smmu_atc_inv_cmd_set_ssid(ssid, &cmd);
 	cmds.num = 0;
 	for (i = 0; i < master->num_streams; i++) {
 		cmd.atc.sid = master->streams[i].id;
@@ -1800,6 +1802,11 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
 }
 
+static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
+{
+	return arm_smmu_atc_inv_master_ssid(master, 0);
+}
+
 /*
  * If ssid is non-zero, issue atc invalidations with the given ssid instead of
  * the one the domain is attached to. This is used by SVA since it's pasid
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#9 lore raw

PATCH v2 9/18 iommu/arm-smmu-v3: Implement set_dev_pasid

In reply to Michael Shavit

This change enables the use of the iommu_attach_dev_pasid API for UNMANAGED domains. The primary use-case is to allow in-kernel users of the iommu API to manage domains with PASID. This change also allows for future support of pasid in the DMA api.

---
v1->v2: Add missing atc invalidation when detaching with pasid

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 168 +++++++++++++++++---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |   1 +
 2 files changed, 149 insertions(+), 20 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +148 -20
index 176013bb974b8..a6fa56585c219 100644
@@ -2173,6 +2173,10 @@ static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
 	return 0;
 }
 
+/*
+ * master may be null for domain types that are finalized before being attached
+ * to a master.
+ */
 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
 				    struct arm_smmu_master *master)
 {
@@ -2369,6 +2373,11 @@ static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
 	return 0;
 }
 
+static bool arm_smmu_master_has_pasid_domains(struct arm_smmu_master *master)
+{
+	return master->nr_attached_pasid_domains > 0;
+}
+
 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
 {
 	struct pci_dev *pdev;
@@ -2411,6 +2420,28 @@ static void arm_smmu_detach_dev(struct arm_smmu_master *master)
 	arm_smmu_install_ste_for_dev(master);
 }
 
+/*
+ * Once attached for the first time, a domain can no longer be attached to any
+ * master with a distinct upstream SMMU.
+ */
+static int arm_smmu_prepare_domain_for_smmu(struct arm_smmu_device *smmu,
+					    struct arm_smmu_domain *smmu_domain)
+{
+	int ret = 0;
+
+	mutex_lock(&smmu_domain->init_mutex);
+	if (!smmu_domain->smmu) {
+		smmu_domain->smmu = smmu;
+		ret = arm_smmu_domain_finalise(&smmu_domain->domain, NULL);
+		if (ret)
+			smmu_domain->smmu = NULL;
+	} else if (smmu_domain->smmu != smmu) {
+		ret = -EINVAL;
+	}
+	mutex_unlock(&smmu_domain->init_mutex);
+	return ret;
+}
+
 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 {
 	int ret = 0;
@@ -2426,6 +2457,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 	master = dev_iommu_priv_get(dev);
 	smmu = master->smmu;
 
+	ret = arm_smmu_prepare_domain_for_smmu(smmu, smmu_domain);
+	if (ret)
+		return ret;
+
 	/*
 	 * Checking that SVA is disabled ensures that this device isn't bound to
 	 * any mm, and can be safely detached from its old domain. Bonds cannot
@@ -2436,22 +2471,18 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 		return -EBUSY;
 	}
 
-	arm_smmu_detach_dev(master);
-
-	mutex_lock(&smmu_domain->init_mutex);
-
-	if (!smmu_domain->smmu) {
-		smmu_domain->smmu = smmu;
-		ret = arm_smmu_domain_finalise(domain, master);
-		if (ret) {
-			smmu_domain->smmu = NULL;
-			goto out_unlock;
-		}
-	} else if (smmu_domain->smmu != smmu) {
-		ret = -EINVAL;
-		goto out_unlock;
+	/*
+	 * Attaching a bypass or stage 2 domain would break any domains attached
+	 * with pasid. Attaching an S1 domain should be feasible but requires
+	 * more complicated logic to handle.
+	 */
+	if (arm_smmu_master_has_pasid_domains(master)) {
+		dev_err(dev, "cannot attach - domain attached with pasid\n");
+		return -EBUSY;
 	}
 
+	arm_smmu_detach_dev(master);
+
 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
 		master->s1_cfg = &master->owned_s1_cfg;
 		ret = arm_smmu_write_ctx_desc(
@@ -2460,8 +2491,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 			0, &smmu_domain->cd);
 		if (ret) {
 			master->s1_cfg = NULL;
-			master->domain = NULL;
-			goto out_unlock;
+			return ret;
 		}
 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2 ||
 		   smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED) {
@@ -2490,11 +2520,75 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
 
 	arm_smmu_enable_ats(master, smmu_domain);
 
-out_unlock:
-	mutex_unlock(&smmu_domain->init_mutex);
 	return ret;
 }
 
+static int arm_smmu_set_dev_pasid(struct iommu_domain *domain,
+				  struct device *dev, ioasid_t pasid)
+{
+	int ret = 0;
+	unsigned long flags;
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct arm_smmu_device *smmu;
+	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct arm_smmu_attached_domain *attached_domain;
+	struct arm_smmu_master *master;
+
+	if (!fwspec)
+		return -ENOENT;
+
+	master = dev_iommu_priv_get(dev);
+	smmu = master->smmu;
+
+	ret = arm_smmu_prepare_domain_for_smmu(smmu, smmu_domain);
+	if (ret)
+		return ret;
+
+	if (pasid == 0) {
+		dev_err(dev, "pasid 0 is reserved for the device's primary domain\n");
+		return -ENODEV;
+	}
+
+	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) {
+		dev_err(dev, "set_dev_pasid only supports stage 1 domains\n");
+		return -EINVAL;
+	}
+
+	if (!master->s1_cfg || master->s2_cfg)
+		return -EBUSY;
+
+	attached_domain = kzalloc(sizeof(*attached_domain), GFP_KERNEL);
+	if (!attached_domain)
+		return -ENOMEM;
+
+	attached_domain->master = master;
+	attached_domain->domain = smmu_domain;
+	attached_domain->ssid = pasid;
+
+	master->nr_attached_pasid_domains += 1;
+	/*
+	 * arm_smmu_share_asid may update the cd's asid value and write the
+	 * ctx_desc for every attached_domains in the list. There's a potential
+	 * race here regardless of whether we first write the ctx_desc or
+	 * first insert into the domain's list. Grabbing the asic_lock prevents
+	 * SVA from changing the cd's ASID while the cd is being attached.
+	 */
+	mutex_lock(&arm_smmu_asid_lock);
+	ret = arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
+				      pasid, &smmu_domain->cd);
+	if (ret) {
+		mutex_unlock(&arm_smmu_asid_lock);
+		kfree(attached_domain);
+	}
+
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_add(&attached_domain->domain_head, &smmu_domain->attached_domains);
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
+	mutex_unlock(&arm_smmu_asid_lock);
+
+	return 0;
+}
+
 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
 			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
 			      int prot, gfp_t gfp, size_t *mapped)
@@ -2740,6 +2834,15 @@ static void arm_smmu_release_device(struct device *dev)
 
 	if (WARN_ON(arm_smmu_master_sva_enabled(master)))
 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
+	if (WARN_ON(master->nr_attached_pasid_domains != 0)) {
+		/*
+		 * TODO: Do we need to handle this case?
+		 * This requires a mechanism to obtain all the pasid domains
+		 * that this master is attached to so that we can clean up the
+		 * domain's attached_domain list.
+		 */
+	}
+
 	arm_smmu_detach_dev(master);
 	arm_smmu_free_cd_tables(master->smmu, &master->owned_s1_cfg.cdcfg);
 	arm_smmu_disable_pasid(master);
@@ -2875,12 +2978,36 @@ static int arm_smmu_def_domain_type(struct device *dev)
 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
 {
 	struct iommu_domain *domain;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	struct arm_smmu_domain *smmu_domain;
+	struct arm_smmu_attached_domain *attached_domain;
+	unsigned long flags;
 
-	domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
+	if (!master || pasid == 0)
+		return;
+
+	domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
 	if (WARN_ON(IS_ERR(domain)) || !domain)
 		return;
+	if (domain->type == IOMMU_DOMAIN_SVA)
+		return arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
 
-	arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
+	smmu_domain = to_smmu_domain(domain);
+	mutex_lock(&arm_smmu_asid_lock);
+	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains, domain_head) {
+		if (attached_domain->master != master ||
+		    attached_domain->ssid != pasid)
+			continue;
+		list_del(&attached_domain->domain_head);
+		break;
+	}
+	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
+	arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master, pasid,
+				NULL);
+	arm_smmu_atc_inv_master_ssid(master, pasid);
+	master->nr_attached_pasid_domains -= 1;
+	mutex_unlock(&arm_smmu_asid_lock);
 }
 
 static struct iommu_ops arm_smmu_ops = {
@@ -2900,6 +3027,7 @@ static struct iommu_ops arm_smmu_ops = {
 	.owner			= THIS_MODULE,
 	.default_domain_ops = &(const struct iommu_domain_ops) {
 		.attach_dev		= arm_smmu_attach_dev,
+		.set_dev_pasid		= arm_smmu_set_dev_pasid,
 		.map_pages		= arm_smmu_map_pages,
 		.unmap_pages		= arm_smmu_unmap_pages,
 		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +1 -0
index 6929590530367..48795a7287b69 100644
@@ -707,6 +707,7 @@ struct arm_smmu_master {
 	bool				iopf_enabled;
 	struct list_head		bonds;
 	unsigned int			ssid_bits;
+	unsigned int			nr_attached_pasid_domains;
 };
 
 /* SMMU private data for an IOMMU domain */
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#10 lore raw

PATCH v2 10/18 iommu/arm-smmu-v3-sva: Remove bond refcount

In reply to Michael Shavit

The iommu-sva framework checks if a bond between a device and mm already exists and handles refcounting at the iommu_domain level. __arm_smmu_sva_bind is therefore only called once for a device/mm pair.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +1 -11
index d07c08b53c5cf..20301d0a2c0b0 100644
@@ -29,7 +29,6 @@ struct arm_smmu_bond {
 	struct mm_struct		*mm;
 	struct arm_smmu_mmu_notifier	*smmu_mn;
 	struct list_head		list;
-	refcount_t			refs;
 };
 
 #define sva_to_bond(handle) \
@@ -377,21 +376,12 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
 	if (!master || !master->sva_enabled)
 		return ERR_PTR(-ENODEV);
 
-	/* If bind() was already called for this {dev, mm} pair, reuse it. */
-	list_for_each_entry(bond, &master->bonds, list) {
-		if (bond->mm == mm) {
-			refcount_inc(&bond->refs);
-			return &bond->sva;
-		}
-	}
-
 	bond = kzalloc(sizeof(*bond), GFP_KERNEL);
 	if (!bond)
 		return ERR_PTR(-ENOMEM);
 
 	bond->mm = mm;
 	bond->sva.dev = dev;
-	refcount_set(&bond->refs, 1);
 
 	bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
 	if (IS_ERR(bond->smmu_mn)) {
@@ -570,7 +560,7 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
 		}
 	}
 
-	if (!WARN_ON(!bond) && refcount_dec_and_test(&bond->refs)) {
+	if (!WARN_ON(!bond)) {
 		list_del(&bond->list);
 		arm_smmu_mmu_notifier_put(bond->smmu_mn);
 		kfree(bond);
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#11 lore raw

PATCH v2 11/18 iommu/arm-smmu-v3-sva: Clean unused iommu_sva

In reply to Michael Shavit

The __arm_smmu_sva_bind function returned an unused iommu_sva handle that can be removed.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c    | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +6 -12
index 20301d0a2c0b0..650c9c9ad52f1 100644
@@ -25,7 +25,6 @@ struct arm_smmu_mmu_notifier {
 #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
 
 struct arm_smmu_bond {
-	struct iommu_sva		sva;
 	struct mm_struct		*mm;
 	struct arm_smmu_mmu_notifier	*smmu_mn;
 	struct list_head		list;
@@ -364,8 +363,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 	arm_smmu_free_shared_cd(cd);
 }
 
-static struct iommu_sva *
-__arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
+static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
 {
 	int ret;
 	struct arm_smmu_bond *bond;
@@ -374,14 +372,13 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 
 	if (!master || !master->sva_enabled)
-		return ERR_PTR(-ENODEV);
+		return -ENODEV;
 
 	bond = kzalloc(sizeof(*bond), GFP_KERNEL);
 	if (!bond)
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 
 	bond->mm = mm;
-	bond->sva.dev = dev;
 
 	bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
 	if (IS_ERR(bond->smmu_mn)) {
@@ -390,11 +387,11 @@ __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
 	}
 
 	list_add(&bond->list, &master->bonds);
-	return &bond->sva;
+	return 0;
 
 err_free_bond:
 	kfree(bond);
-	return ERR_PTR(ret);
+	return ret;
 }
 
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
@@ -572,13 +569,10 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
 				      struct device *dev, ioasid_t id)
 {
 	int ret = 0;
-	struct iommu_sva *handle;
 	struct mm_struct *mm = domain->mm;
 
 	mutex_lock(&sva_lock);
-	handle = __arm_smmu_sva_bind(dev, mm);
-	if (IS_ERR(handle))
-		ret = PTR_ERR(handle);
+	ret = __arm_smmu_sva_bind(dev, mm);
 	mutex_unlock(&sva_lock);
 
 	return ret;
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#12 lore raw

PATCH v2 12/18 iommu/arm-smmu-v3-sva: Remove arm_smmu_bond

In reply to Michael Shavit

There's a 1:1 relationship between arm_smmu_bond and the iommu_domain used in set_dev_pasid/remove_dev_pasid. arm_smmu_bond has become an unnecessary complication. It's more natural to store any needed information at the iommu_domain container level.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   | 69 +++++++------------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   |  1 -
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |  2 +-
 3 files changed, 24 insertions(+), 48 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +23 -46
index 650c9c9ad52f1..b615a85e6a54e 100644
@@ -24,14 +24,13 @@ struct arm_smmu_mmu_notifier {
 
 #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
 
-struct arm_smmu_bond {
-	struct mm_struct		*mm;
+struct arm_smmu_sva_domain {
+	struct iommu_domain		iommu_domain;
 	struct arm_smmu_mmu_notifier	*smmu_mn;
-	struct list_head		list;
 };
 
-#define sva_to_bond(handle) \
-	container_of(handle, struct arm_smmu_bond, sva)
+#define to_sva_domain(domain) \
+	container_of(domain, struct arm_smmu_sva_domain, iommu_domain)
 
 static DEFINE_MUTEX(sva_lock);
 
@@ -363,10 +362,10 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 	arm_smmu_free_shared_cd(cd);
 }
 
-static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
+static int __arm_smmu_sva_bind(struct device *dev,
+			       struct arm_smmu_sva_domain *sva_domain,
+			       struct mm_struct *mm)
 {
-	int ret;
-	struct arm_smmu_bond *bond;
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -374,24 +373,14 @@ static int __arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm)
 	if (!master || !master->sva_enabled)
 		return -ENODEV;
 
-	bond = kzalloc(sizeof(*bond), GFP_KERNEL);
-	if (!bond)
-		return -ENOMEM;
-
-	bond->mm = mm;
-
-	bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
-	if (IS_ERR(bond->smmu_mn)) {
-		ret = PTR_ERR(bond->smmu_mn);
-		goto err_free_bond;
+	sva_domain->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain,
+							mm);
+	if (IS_ERR(sva_domain->smmu_mn)) {
+		sva_domain->smmu_mn = NULL;
+		return PTR_ERR(sva_domain->smmu_mn);
 	}
-
-	list_add(&bond->list, &master->bonds);
+	master->nr_attached_sva_domains += 1;
 	return 0;
-
-err_free_bond:
-	kfree(bond);
-	return ret;
 }
 
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
@@ -521,7 +510,7 @@ int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
 int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
 {
 	mutex_lock(&sva_lock);
-	if (!list_empty(&master->bonds)) {
+	if (master->nr_attached_sva_domains != 0) {
 		dev_err(master->dev, "cannot disable SVA, device is bound\n");
 		mutex_unlock(&sva_lock);
 		return -EBUSY;
@@ -545,23 +534,12 @@ void arm_smmu_sva_notifier_synchronize(void)
 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
 				   struct device *dev, ioasid_t id)
 {
-	struct mm_struct *mm = domain->mm;
-	struct arm_smmu_bond *bond = NULL, *t;
+	struct arm_smmu_sva_domain *sva_domain = to_sva_domain(domain);
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 
 	mutex_lock(&sva_lock);
-	list_for_each_entry(t, &master->bonds, list) {
-		if (t->mm == mm) {
-			bond = t;
-			break;
-		}
-	}
-
-	if (!WARN_ON(!bond)) {
-		list_del(&bond->list);
-		arm_smmu_mmu_notifier_put(bond->smmu_mn);
-		kfree(bond);
-	}
+	master->nr_attached_sva_domains -= 1;
+	arm_smmu_mmu_notifier_put(sva_domain->smmu_mn);
 	mutex_unlock(&sva_lock);
 }
 
@@ -572,7 +550,7 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
 	struct mm_struct *mm = domain->mm;
 
 	mutex_lock(&sva_lock);
-	ret = __arm_smmu_sva_bind(dev, mm);
+	ret = __arm_smmu_sva_bind(dev, to_sva_domain(domain), mm);
 	mutex_unlock(&sva_lock);
 
 	return ret;
@@ -590,12 +568,11 @@ static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
 
 struct iommu_domain *arm_smmu_sva_domain_alloc(void)
 {
-	struct iommu_domain *domain;
+	struct arm_smmu_sva_domain *sva_domain;
 
-	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
-	if (!domain)
+	sva_domain = kzalloc(sizeof(*sva_domain), GFP_KERNEL);
+	if (!sva_domain)
 		return NULL;
-	domain->ops = &arm_smmu_sva_domain_ops;
-
-	return domain;
+	sva_domain->iommu_domain.ops = &arm_smmu_sva_domain_ops;
+	return &sva_domain->iommu_domain;
 }
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +0 -1
index a6fa56585c219..b7f834dde85d1 100644
@@ -2784,7 +2784,6 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 
 	master->dev = dev;
 	master->smmu = smmu;
-	INIT_LIST_HEAD(&master->bonds);
 	dev_iommu_priv_set(dev, master);
 
 	ret = arm_smmu_insert_master(smmu, master);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +1 -1
index 48795a7287b69..3525d60668c23 100644
@@ -705,7 +705,7 @@ struct arm_smmu_master {
 	bool				stall_enabled;
 	bool				sva_enabled;
 	bool				iopf_enabled;
-	struct list_head		bonds;
+	unsigned int			nr_attached_sva_domains;
 	unsigned int			ssid_bits;
 	unsigned int			nr_attached_pasid_domains;
 };
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#13 lore raw

PATCH v2 13/18 iommu/arm-smmu-v3-sva: Add check when enabling sva

In reply to Michael Shavit

SVA domains can only be attached when the master's STEs have a stage 1 domain.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 6 ++++++
 1 file changed, 6 insertions(+)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +6 -0
index b615a85e6a54e..e2a91f20f0906 100644
@@ -499,9 +499,15 @@ int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
 	int ret;
 
 	mutex_lock(&sva_lock);
+
+	if (!master->s1_cfg) {
+		ret = -EBUSY;
+		goto unlock;
+	}
 	ret = arm_smmu_master_sva_enable_iopf(master);
 	if (!ret)
 		master->sva_enabled = true;
+unlock:
 	mutex_unlock(&sva_lock);
 
 	return ret;
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#14 lore raw

PATCH v2 14/18 iommu/arm-smmu-v3: Support domains with shared CDs

In reply to Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however. To work around this limitation, we propose an ARM_SMMU_DOMAIN_S1_SHARED domain type for domains that attach a CD shared across with arm_smmu_domains (each attached to a different upstream SMMU device).

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 26 ++++++++++++++++-----
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |  2 ++
 2 files changed, 22 insertions(+), 6 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +20 -6
index b7f834dde85d1..69b1d09fd0284 100644
@@ -965,6 +965,20 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
 }
 
+static struct arm_smmu_ctx_desc *arm_smmu_get_cd(struct arm_smmu_domain *domain)
+{
+	if (domain->stage == ARM_SMMU_DOMAIN_S1_SHARED_CD)
+		return domain->shared_cd;
+	else
+		return &domain->cd;
+}
+
+static bool arm_smmu_is_s1_domain(struct arm_smmu_domain *domain)
+{
+	return domain->stage == ARM_SMMU_DOMAIN_S1_SHARED_CD ||
+	       domain->stage == ARM_SMMU_DOMAIN_S1;
+}
+
 /* master may be null */
 static void arm_smmu_sync_cd(struct arm_smmu_master *master,
 			     int ssid, bool leaf)
@@ -1887,8 +1901,8 @@ static void arm_smmu_tlb_inv_context(void *cookie)
 	 * insertion to guarantee those are observed before the TLBI. Do be
 	 * careful, 007.
 	 */
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
-		arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
+	if (arm_smmu_is_s1_domain(smmu_domain)) {
+		arm_smmu_tlb_inv_asid(smmu, arm_smmu_get_cd(smmu_domain)->asid);
 	} else {
 		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
@@ -1968,10 +1982,10 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
 		},
 	};
 
-	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
+	if (arm_smmu_is_s1_domain(smmu_domain)) {
 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
 				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
-		cmd.tlbi.asid	= smmu_domain->cd.asid;
+		cmd.tlbi.asid	= arm_smmu_get_cd(smmu_domain)->asid;
 	} else {
 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
@@ -2549,7 +2563,7 @@ static int arm_smmu_set_dev_pasid(struct iommu_domain *domain,
 		return -ENODEV;
 	}
 
-	if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1) {
+	if (!arm_smmu_is_s1_domain(smmu_domain)) {
 		dev_err(dev, "set_dev_pasid only supports stage 1 domains\n");
 		return -EINVAL;
 	}
@@ -2575,7 +2589,7 @@ static int arm_smmu_set_dev_pasid(struct iommu_domain *domain,
 	 */
 	mutex_lock(&arm_smmu_asid_lock);
 	ret = arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
-				      pasid, &smmu_domain->cd);
+				      pasid, arm_smmu_get_cd(smmu_domain));
 	if (ret) {
 		mutex_unlock(&arm_smmu_asid_lock);
 		kfree(attached_domain);
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +2 -0
index 3525d60668c23..4ac69427abf1c 100644
@@ -713,6 +713,7 @@ struct arm_smmu_master {
 /* SMMU private data for an IOMMU domain */
 enum arm_smmu_domain_stage {
 	ARM_SMMU_DOMAIN_S1 = 0,
+	ARM_SMMU_DOMAIN_S1_SHARED_CD,
 	ARM_SMMU_DOMAIN_S2,
 	ARM_SMMU_DOMAIN_NESTED,
 	ARM_SMMU_DOMAIN_BYPASS,
@@ -728,6 +729,7 @@ struct arm_smmu_domain {
 	enum arm_smmu_domain_stage		stage;
 	union {
 		struct arm_smmu_ctx_desc	cd;
+		struct arm_smmu_ctx_desc	*shared_cd;
 		struct arm_smmu_s2_cfg		s2_cfg;
 	};
 
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#15 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 3 lines · Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however.

Isn't that pretty much because we don't support replicating invalidations to each of the different SMMU instances?

I don't really get why there would be anything like a "shared CD".

It looks kind of like the arm_smmu_ctx_desc has become a bit weird since its main purpose seems to be to refcount the ASID.

But our general design is that the iommu_domain should be 1:1 with the ASID, so it is really strange that we'd refcount the ASID..

I would expect different SMMU devices to be handled by allowing single domains to be attached to a list of masters where the masters can all be from different instances.

When an invalidation comes in we have to replicate it through all the instances for the IOTLB and all the masters for the ATC.

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from :)

You might try to stop your series here and get the first batch of patches done. This latter bit seems to be a seperate topic?

Jason

#16 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 8 lines · Jason Gunthorpe
Quoted 3 lines · Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however.

Isn't that pretty much because we don't support replicating invalidations to each of the different SMMU instances?

....

Quoted 3 lines

I would expect different SMMU devices to be handled by allowing single domains to be attached to a list of masters where the masters can all be from different instances.

Oh you're right! When I first looked at this, arm_smmu_domain still held an arm_smmu_s1_cfg which depends on the SMMU device. But attaching a single (stage 1) arm_smmu_domain to multiple SMMUs looks much more tractable after the first half of this patch series. We can add a handle to the smmu device in the new arm_smmu_attached_domain struct for this purpose.

Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from :)

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

Quoted 2 lines

You might try to stop your series here and get the first batch of patches done. This latter bit seems to be a seperate topic?

The main motivation for this part of the series is to reduce inconsistencies with the smmu_domain->attached_domains list and arm-smmu-v3 functions that rely on that list. Specifically to reach the last patch in the series: "iommu/arm-smmu-v3-sva: Remove atc_inv_domain_ssid".

Splitting this part into a follow-up patch series would definitely be easier and helpful if you're all ok with it :) .

#17 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 8 lines · Michael Shavit
Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from :)

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

I suppose we also don't really have a entirely clear picture what allocating multiple SVA domains should even do in the iommu driver.

The driver would like to share the ASID, but things are much cleaner for everything if the driver model has ASID 1:1 with the iommu_domain.

It suggests we are missing some core code in iommu_sva_bind_device() to try to re-use existing SVA iommu_domains. This would certainly be better than trying to teach every driver how to share and refcount its ASID concept...

Today we have this super hacky iommu_get_domain_for_dev_pasid() thing that allows SVA domain reuse for a single device.

Possibly what we should do is conver the u32 pasid in the mm_struct to a struct iommu_mm_data * and put alot more stuff in there. eg a linked list of all SVA domains.

Quoted 2 lines

Splitting this part into a follow-up patch series would definitely be easier and helpful if you're all ok with it :) .

I think splitting it into a series to re-organize the way ste/cd stuff works is a nice contained topic.

Adjusting the way the ASID works with SVA is another good topic

And so on

Jason

#18 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 8 lines · Jason Gunthorpe
Quoted 7 lines · Michael Shavit
Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from 😄

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

Quoted 5 lines

I suppose we also don't really have a entirely clear picture what allocating multiple SVA domains should even do in the iommu driver.

The driver would like to share the ASID, but things are much cleaner for everything if the driver model has ASID 1:1 with the iommu_domain.

This means that each ASID should be mapped to a single IOMMU domain. This is conceptually right as iommu_domain represents a hardware page table. For SVA, it's an mm_struct.

So in my mind, each sva_domain should have a 1:1 relationship with an mm_struct. Each sva_domain could have a 1:N relationship with ASID (or PCI PASID), but in current implementation, it's a 1:1 relationship due to the current global pasid policy for SVA.

Quoted 6 lines

It suggests we are missing some core code in iommu_sva_bind_device() to try to re-use existing SVA iommu_domains. This would certainly be better than trying to teach every driver how to share and refcount its ASID concept...

Quoted 6 lines

Today we have this super hacky iommu_get_domain_for_dev_pasid() thing that allows SVA domain reuse for a single device.

Possibly what we should do is conver the u32 pasid in the mm_struct to a struct iommu_mm_data * and put alot more stuff in there. eg a linked list of all SVA domains.

I don't quite follow "a linked list of all SVA domains". If my above understanding is correct, then there should be a single sva_domain for each mm_struct. The iommu_sva_domain_alloc() takes the responsibility to allocate/free and refcount the sva domain. The sva bind code could simply:

	domain = iommu_get_sva_domain(dev, mm);
	iommu_attach_device_pasid(domain, dev, pasid);

and the sva unbind code:

	iommu_detach_device_pasid(domain, dev, pasid);
	iommu_put_sva_domain(domain);

Perhaps, we can further drop struct iommu_sva and make the sva bind/unbind interfaces to return the sva domain instead?

Best regards,
baolu

#19 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 15 lines · Baolu Lu
Quoted 8 lines · Jason Gunthorpe
Quoted 7 lines · Michael Shavit
Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from 😄

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

Quoted 5 lines

I suppose we also don't really have a entirely clear picture what allocating multiple SVA domains should even do in the iommu driver.

The driver would like to share the ASID, but things are much cleaner for everything if the driver model has ASID 1:1 with the iommu_domain.

Quoted 6 lines

This means that each ASID should be mapped to a single IOMMU domain. This is conceptually right as iommu_domain represents a hardware page table. For SVA, it's an mm_struct.

So in my mind, each sva_domain should have a 1:1 relationship with an mm_struct.

If we want to support multiple iommu drivers then we should support multiple iommu_domains per mm_struct so that each driver can have its own. In this world if each instance wants its own iommu_domain it is not a big deal.

Drivers that can share iommu_domains across instances should probably also share sva iommu_domains across instances.

Most real systems have only one iommu driver and we'd like the good iommu drivers to be able to share domains across instances, so we'd expect only 1 iommu_domain per mm struct.

Jason

#20 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 26 lines · Jason Gunthorpe
Quoted 12 lines · Baolu Lu
Quoted 11 lines · Jason Gunthorpe
Quoted 7 lines · Michael Shavit
Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from 😄

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

I suppose we also don't really have a entirely clear picture what allocating multiple SVA domains should even do in the iommu driver.

Quoted 8 lines
Quoted 2 lines

The driver would like to share the ASID, but things are much cleaner for everything if the driver model has ASID 1:1 with the iommu_domain.

This means that each ASID should be mapped to a single IOMMU domain. This is conceptually right as iommu_domain represents a hardware page table. For SVA, it's an mm_struct.

So in my mind, each sva_domain should have a 1:1 relationship with an mm_struct.

If we want to support multiple iommu drivers then we should support multiple iommu_domains per mm_struct so that each driver can have its own. In this world if each instance wants its own iommu_domain it is not a big deal.

Quoted 6 lines

Drivers that can share iommu_domains across instances should probably also share sva iommu_domains across instances.

Most real systems have only one iommu driver and we'd like the good iommu drivers to be able to share domains across instances, so we'd expect only 1 iommu_domain per mm struct.

Yes. You are right. I overlooked the multiple-drivers case. So we stay on the same page now.

Best regards,
baolu

#21 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

In reply to Jason Gunthorpe
Quoted 8 lines · Jason Gunthorpe
Quoted 3 lines · Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however.

Isn't that pretty much because we don't support replicating invalidations to each of the different SMMU instances?

Looked into this some more, and supporting attach to multiple devices is still very hard:
1. When an arm_smmu_domain is first attached to a master, it initializes an io_pgtable_cfg object whose properties depend on the master's upstream SMMU device. This io_pgtable_cfg is then used to allocate an io_pgtable object, and the arm_smmu_ctx_desc's TTBR field points to that io_pgtable's TTBR (and ditto for the vttbr on stage 2 domains). So then arm_smmu_domain needs to be split into two, arm_smmu_domain and arm_smmu_device_domain with the latter containing a per-SMMU device io_pgtable, arm_smmu_ctx_desc and arm_smmu_s2_cfg. Each iommu_domain_ops operation now needs to loop over each arm_smmu_device_domain.
2. Some of the iommu_domain fields also depend on the per-SMMU io_pgtable_cfg; specifically pgsize_bitmap and geometry.aperture_end. These need to be restricted as the domain is attached to more devices.
3. Attaching a domain to a new SMMU device must be prohibited after any call to map_pages or if iommu_domain.pgsize_bitmap and iommu-domain.geometry.aperture_end have been consumed by any system. The first is something the arm-smmu-v3.c driver could feasibly enforce on its own, the second requires changes to the iommu framework.

The arm-smmu-v3-sva.c implementation avoids all these problems because it doesn't need to allocate an io_pgtable; the shared arm_smmu_ctx_desc's TTBR is set to the MM's TTBR. One possible solution would be to make the changes proposed in 1., but only allowing SVA domains to attach to multiple devices:
1. Domain functions only accessed by arm-smmu-v3.c (such as arm_smmu_iova_to_phys) can assume there's a single arm_smmu_device_domain per arm_smmu_domain.
2. Domain functions also accessed by arm-smmu-v3-sva.c (such as invalidations commands) must iterate over all arm_smmu_device_domains.

Quoted 3 lines · Jason Gunthorpe

I think splitting it into a series to re-organize the way ste/cd stuff works is a nice contained topic.

Should I explicitly resend this patch series as a v3 cutting off patches 14 and onwards?

#22 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 10 lines · Michael Shavit

The arm-smmu-v3-sva.c implementation avoids all these problems because it doesn't need to allocate an io_pgtable; the shared arm_smmu_ctx_desc's TTBR is set to the MM's TTBR. One possible solution would be to make the changes proposed in 1., but only allowing SVA domains to attach to multiple devices:
1. Domain functions only accessed by arm-smmu-v3.c (such as arm_smmu_iova_to_phys) can assume there's a single arm_smmu_device_domain per arm_smmu_domain.
2. Domain functions also accessed by arm-smmu-v3-sva.c (such as invalidations commands) must iterate over all arm_smmu_device_domains.

Or rather, arm_smmu_domain would hold a list of attached smmu devices, but only SVA domains would be allowed to have multiple entries in that list since the other arm_smmu_domain fields are tied to a single SMMU device for other domain types.

reviewer
#23 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

In reply to Michael Shavit
Quoted 22 lines · Michael Shavit
Quoted 8 lines · Jason Gunthorpe
Quoted 3 lines · Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however.

Isn't that pretty much because we don't support replicating invalidations to each of the different SMMU instances?

Looked into this some more, and supporting attach to multiple devices is still very hard:
1. When an arm_smmu_domain is first attached to a master, it initializes an io_pgtable_cfg object whose properties depend on the master's upstream SMMU device. This io_pgtable_cfg is then used to allocate an io_pgtable object, and the arm_smmu_ctx_desc's TTBR field points to that io_pgtable's TTBR (and ditto for the vttbr on stage 2 domains). So then arm_smmu_domain needs to be split into two, arm_smmu_domain and arm_smmu_device_domain with the latter containing a per-SMMU device io_pgtable, arm_smmu_ctx_desc and arm_smmu_s2_cfg. Each iommu_domain_ops operation now needs to loop over each arm_smmu_device_domain.

Quoted 8 lines

2. Some of the iommu_domain fields also depend on the per-SMMU io_pgtable_cfg; specifically pgsize_bitmap and geometry.aperture_end. These need to be restricted as the domain is attached to more devices.
3. Attaching a domain to a new SMMU device must be prohibited after any call to map_pages or if iommu_domain.pgsize_bitmap and iommu-domain.geometry.aperture_end have been consumed by any system. The first is something the arm-smmu-v3.c driver could feasibly enforce on its own, the second requires changes to the iommu framework.

In practice it would be entirely reasonable to only support cross-instance attach between instances with matching capabilities such that they *can* share the pagetable directly.

Quoted 2 lines

The arm-smmu-v3-sva.c implementation avoids all these problems because it doesn't need to allocate an io_pgtable;

SVA has all the same underlying problems - pagetable sharing is still pagetable sharing regardless of which code happens to allocate the physical pages - they're just avoided up-front by disallowing SVA at all if the relevant capabilities don't line up between SMMU and CPU.

Thanks,
Robin.

#24 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

In reply to Michael Shavit
Quoted 7 lines · Michael Shavit
Quoted 6 lines · Jason Gunthorpe
Quoted 3 lines · Michael Shavit

SVA may attach a CD to masters that have different upstream SMMU devices. The arm_smmu_domain structure can only be attached to a single upstream SMMU device however.

Quoted 8 lines
Quoted 2 lines

Isn't that pretty much because we don't support replicating invalidations to each of the different SMMU instances?

Looked into this some more, and supporting attach to multiple devices is still very hard:
1. When an arm_smmu_domain is first attached to a master, it initializes an io_pgtable_cfg object whose properties depend on the master's upstream SMMU device.

So, this is actually kind of wrong, Robin is working on fixing it.

The mental model you should have is when an iommu_domain is created it represents a single, definitive, IO page table format. The format is selected based on creation arguments and the 'master' it will be used with.

An iommu_domain has a single IO page table in a single format, and that format doesn't change while the iommu_domain exists.

Even single-instance cases have S1 and S2 IO page table formats co-existing.

When we talk about multi instance support, it means the iommu_domain - in whatever fixed IO page table format it uses - can be attached to any SMMU instance that supports it as a compatible page table format.

ARM doesn't quite reach this model, but once it passes the finalize step it does. The goal is to move finalize to allocate. So for your purposes you can ignore the difference.

Quoted 5 lines

domains). So then arm_smmu_domain needs to be split into two, arm_smmu_domain and arm_smmu_device_domain with the latter containing a per-SMMU device io_pgtable, arm_smmu_ctx_desc and arm_smmu_s2_cfg. Each iommu_domain_ops operation now needs to loop over each arm_smmu_device_domain.

No, if the instance is not compatible whith the domain then the attachment fails.

Quoted 4 lines

2. Some of the iommu_domain fields also depend on the per-SMMU io_pgtable_cfg; specifically pgsize_bitmap and geometry.aperture_end. These need to be restricted as the domain is attached to more devices.

These need to be an exact match then.

Quoted 3 lines

3. Attaching a domain to a new SMMU device must be prohibited after any call to map_pages or if iommu_domain.pgsize_bitmap and iommu-domain.geometry.aperture_end have been consumed by any system.

Attach needs to fail, we don't reconfigure the domain.

Quoted 3 lines

The arm-smmu-v3-sva.c implementation avoids all these problems because it doesn't need to allocate an io_pgtable; the shared arm_smmu_ctx_desc's

This isn't true, it has an IO page table (owned by the MM) and that page table has all the same properties you were concerned about above. They all still need to be checked for compatability.

ie from a SMMU HW instance perspective there is no difference between a S1 or MM owned IO page table. From a code perspective the only difference should be where the TTBR comes from. The SVA case still should have a cfg describing what kind of IO pagetable the mm has created, and that cfg should still technically be checked for compatability.

Have in mind that SVA is *exactly* the same as a S1 iommu_domain with PRI enabled except that the TTBR comes from the MM instead of being internally allocated. There should be no other differences between the two operating modes, that SVA became its owns special thing needs to be undone, not doubled down on.

Quoted 5 lines
Quoted 2 lines

I think splitting it into a series to re-organize the way ste/cd stuff works is a nice contained topic.

Should I explicitly resend this patch series as a v3 cutting off patches 14 and onwards?

I think it is good to make progress, it looked to me like the first part stood alone fairly well and was an improvement on its own.

Jason

#25 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 4 lines · Robin Murphy

In practice it would be entirely reasonable to only support cross-instance attach between instances with matching capabilities such that they *can* share the pagetable directly.

Quoted 7 lines · Jason Gunthorpe

When we talk about multi instance support, it means the iommu_domain - in whatever fixed IO page table format it uses - can be attached to any SMMU instance that supports it as a compatible page table format.

ARM doesn't quite reach this model, but once it passes the finalize step it does. The goal is to move finalize to allocate. So for your purposes you can ignore the difference.

Got you. Failing the atach when the page table format is incompatible with the smmu device is a lot easier to handle. I didn't notice that SVA was already checking this elsewhere.
I can give the multi-instance support a try (with the rest of these patches on top) and send it out as a follow-up series to this one.

Quoted 2 lines

I think it is good to make progress, it looked to me like the first part stood alone fairly well and was an improvement on its own.

Sorry for the noobie question; it's not 100% obvious to me what the next step is here. Is there anything I should do to progress that first part forward?

#26 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 6 lines · Michael Shavit
Quoted 2 lines

I think it is good to make progress, it looked to me like the first part stood alone fairly well and was an improvement on its own.

Sorry for the noobie question; it's not 100% obvious to me what the next step is here. Is there anything I should do to progress that first part forward?

Reword your cover letter to cover the new subset and repost the series as v3 with just the patches you want to go with

Jason

#27 lore raw

RE: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

In reply to Jason Gunthorpe

Hi,

Quoted 32 lines

-----Original Message-----
From: Jason Gunthorpe <[email protected]>
Sent: Wednesday, June 7, 2023 8:00 PM
To: Michael Shavit <[email protected]>
Cc: Will Deacon <[email protected]>; Robin Murphy <[email protected]>; Joerg Roedel <[email protected]>; [email protected]; [email protected]; [email protected]; linux-arm- [email protected]; [email protected]; linux- [email protected]
Subject: Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 8 lines · Michael Shavit
Quoted 3 lines

What we definately shouldn't do is try to have different SVA iommu_domain's pointing at the same ASID. That is again making SVA special, which we are trying to get away from :)

Fwiw, this change is preserving the status-quo in that regard; arm-smmu-v3-sva.c is already doing this. But yes, I agree that resolving the limitation is a better long term solution... and something I can try to look at further.

I suppose we also don't really have a entirely clear picture what allocating multiple SVA domains should even do in the iommu driver.

The driver would like to share the ASID, but things are much cleaner for everything if the driver model has ASID 1:1 with the iommu_domain.

It suggests we are missing some core code in iommu_sva_bind_device() to try to re-use existing SVA iommu_domains. This would certainly be better than trying to teach every driver how to share and refcount its ASID concept...

Quoted 6 lines

Today we have this super hacky iommu_get_domain_for_dev_pasid() thing that allows SVA domain reuse for a single device.

Possibly what we should do is conver the u32 pasid in the mm_struct to a struct iommu_mm_data * and put alot more stuff in there. eg a linked list of all SVA domains.

If we are going to have 1:1 between SVA domain and pasid, why we need a linked list of all SVA domains? Would a SVA domain pointer be enough?

I've got a patch-set which takes this suggestion to add an iommu_mm_data struct field to mm_struct. I'll send it out for review soon. The motivation of that patch-set is to let the invalidate_range() callback use the SVA domain referenced by mm->iommu_mm_data->sva_domain to do per-iommu IOTLB invalidation.

Regards,
-Tina

Quoted 12 lines
Quoted 3 lines

Splitting this part into a follow-up patch series would definitely be easier and helpful if you're all ok with it :) .

I think splitting it into a series to re-organize the way ste/cd stuff works is a nice contained topic.

Adjusting the way the ASID works with SVA is another good topic

And so on

Jason

#28 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 7 lines · Tina
Quoted 3 lines

Possibly what we should do is conver the u32 pasid in the mm_struct to a struct iommu_mm_data * and put alot more stuff in there. eg a linked list of all SVA domains.

If we are going to have 1:1 between SVA domain and pasid, why we need a linked list of all SVA domains? Would a SVA domain pointer be enough?

Kevin asked this, we can't assume that a single SVA domain is going to work in a multi-iommu-driver system, which we are trying to enable at the core level..

Quoted 5 lines

I've got a patch-set which takes this suggestion to add an iommu_mm_data struct field to mm_struct. I'll send it out for review soon. The motivation of that patch-set is to let the invalidate_range() callback use the SVA domain referenced by mm->iommu_mm_data->sva_domain to do per-iommu IOTLB invalidation.

Huh?

You are supposed to put the struct mmu_notifier inside the sva_domain struct and use container_of().

This is another reason why I'd prefer we de-duplicate SVA domains at the core code level as duplicitive notifiers are expensive..

Please don't add stuff to the mm just for this reason.

Jason

#29 lore raw

RE: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 33 lines

-----Original Message-----
From: Jason Gunthorpe <[email protected]>
Sent: Tuesday, July 11, 2023 12:55 AM
To: Zhang, Tina <[email protected]>
Cc: Michael Shavit <[email protected]>; Will Deacon <[email protected]>; Robin Murphy <[email protected]>; Joerg Roedel <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; linux- [email protected]
Subject: Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 7 lines · Tina
Quoted 3 lines

Possibly what we should do is conver the u32 pasid in the mm_struct to a struct iommu_mm_data * and put alot more stuff in there. eg a linked list of all SVA domains.

If we are going to have 1:1 between SVA domain and pasid, why we need a linked list of all SVA domains? Would a SVA domain pointer be enough?

Kevin asked this, we can't assume that a single SVA domain is going to work in a multi-iommu-driver system, which we are trying to enable at the core level..

Quoted 5 lines

I've got a patch-set which takes this suggestion to add an iommu_mm_data struct field to mm_struct. I'll send it out for review soon. The motivation of that patch-set is to let the invalidate_range() callback use the SVA domain referenced by mm->iommu_mm_data->sva_domain to do per-iommu IOTLB invalidation.

Quoted 4 lines

Huh?

You are supposed to put the struct mmu_notifier inside the sva_domain struct and use container_of().

No. Just want to use iommu_mm to replace the old pasid field.
https://lore.kernel.org/linux-iommu/[email protected]/

The motivation is mainly to for performance optimization for vIOMMU, though it could also benefit native world.

Currently, in invalidate_range() callback implemented by VT-d driver, due to lacking sva_domain knowledge (i.e., intel_invalidate_range() doesn't know which IOMMUs' IOTLB should be invalidated), intel_invalidate_range() just performs IOMMU IOTLB per device and that leads to superfluous IOTLB invalidations. For example, if there are two devices physically behind an IOMMU and those two devices are used by one user space application, it means in each invalidate_range() callback, two IOTLB and two device-IOTLB invalidations would be performed, instead of one IOTLB plus two device-TLB invalidations (i.e., one IOTLB invalidation is redundant). Things will become worse in a virtualized environment, especially with one vIOMMU exposed to guest, as invoking IOTLB is much more expensive than doing it in native and conceptually there could be more devices behind a vIOMMU that means more superfluous IOTLB invalidations.

So, we want to add sva_domain related info to mm to remove the superfluous IOTLB invalidations.

Thanks,
-Tina

Quoted 7 lines

This is another reason why I'd prefer we de-duplicate SVA domains at the core code level as duplicitive notifiers are expensive..

Please don't add stuff to the mm just for this reason.

Jason

#30 lore raw

Re: [PATCH v2 14/18] iommu/arm-smmu-v3: Support domains with shared CDs

Quoted 8 lines · Tina

The motivation is mainly to for performance optimization for vIOMMU, though it could also benefit native world.

Currently, in invalidate_range() callback implemented by VT-d driver, due to lacking sva_domain knowledge (i.e., intel_invalidate_range() doesn't know which IOMMUs' IOTLB should be invalidated), intel_invalidate_range() just performs IOMMU IOTLB per device and that leads to superfluous IOTLB invalidations.

You get the sva_domain from container_of on the notifier

The sva_domain has a list of all the devices and iommu's that it is connected to

The driver optimizes the invalidations based on the list.

The core code de-duplicates the sva domain so there is usually only one.

If someone creates two SVA domains then the driver is less optimal, oh well.

The driver should not look inside the mm_struct and certainly should never try to get a "SVA domain" from it.

Jason

#31 lore raw

PATCH v2 15/18 iommu/arm-smmu-v3: Allow more re-use for SVA

In reply to Michael Shavit

Now that arm-smmu-v3.c supports attaching domains with pasid, SVA can also re-use much of the same logic. This change allows SVA to allocate arm_smmu_domains with a shared CD and attach them using the arm-smmu-v3 set_dev_pasid implementation.

Because these domains aren't backed by an iommu_domain we must make sure that an arm_smmu_domain's backing iommu_domain isn't accessed on functions used by SVA. A good rule here would be to allow domain_finalize to access it.

---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 63 ++++++++++++++++-----
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 11 ++++
 2 files changed, 60 insertions(+), 14 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +49 -14
index 69b1d09fd0284..3c5ff4f58934a 100644
@@ -1926,7 +1926,7 @@ static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
 
 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
 		/* Get the leaf page size */
-		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
+		tg = __ffs(smmu_domain->smmu->pgsize_bitmap);
 
 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
 		cmd->tlbi.tg = (tg - 10) / 2;
@@ -2053,6 +2053,14 @@ static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
 	}
 }
 
+static void arm_smmu_init_smmu_domain(struct arm_smmu_domain *smmu_domain)
+{
+	mutex_init(&smmu_domain->init_mutex);
+	INIT_LIST_HEAD(&smmu_domain->attached_domains);
+	spin_lock_init(&smmu_domain->attached_domains_lock);
+	INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
+}
+
 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 {
 	struct arm_smmu_domain *smmu_domain;
@@ -2075,14 +2083,22 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 	if (!smmu_domain)
 		return NULL;
 
-	mutex_init(&smmu_domain->init_mutex);
-	INIT_LIST_HEAD(&smmu_domain->attached_domains);
-	spin_lock_init(&smmu_domain->attached_domains_lock);
-	INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
-
+	arm_smmu_init_smmu_domain(smmu_domain);
 	return &smmu_domain->domain;
 }
 
+struct arm_smmu_domain *
+arm_smmu_init_shared_cd_domain(struct arm_smmu_device *smmu,
+			       struct arm_smmu_domain *smmu_domain,
+			       struct arm_smmu_ctx_desc *cd)
+{
+	arm_smmu_init_smmu_domain(smmu_domain);
+	smmu_domain->smmu = smmu;
+	smmu_domain->stage = ARM_SMMU_DOMAIN_S1_SHARED_CD;
+	smmu_domain->shared_cd = cd;
+	return smmu_domain;
+}
+
 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
 {
 	int idx, size = 1 << span;
@@ -2541,11 +2557,9 @@ static int arm_smmu_set_dev_pasid(struct iommu_domain *domain,
 				  struct device *dev, ioasid_t pasid)
 {
 	int ret = 0;
-	unsigned long flags;
 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
 	struct arm_smmu_device *smmu;
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
-	struct arm_smmu_attached_domain *attached_domain;
 	struct arm_smmu_master *master;
 
 	if (!fwspec)
@@ -2558,6 +2572,18 @@ static int arm_smmu_set_dev_pasid(struct iommu_domain *domain,
 	if (ret)
 		return ret;
 
+	return arm_smmu_domain_set_dev_pasid(dev, master, smmu_domain, pasid);
+}
+
+int arm_smmu_domain_set_dev_pasid(struct device *dev,
+				  struct arm_smmu_master *master,
+				  struct arm_smmu_domain *smmu_domain,
+				  ioasid_t pasid)
+{
+	unsigned long flags;
+	struct arm_smmu_attached_domain *attached_domain;
+	int ret;
+
 	if (pasid == 0) {
 		dev_err(dev, "pasid 0 is reserved for the device's primary domain\n");
 		return -ENODEV;
@@ -2991,12 +3017,8 @@ static int arm_smmu_def_domain_type(struct device *dev)
 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
 {
 	struct iommu_domain *domain;
-	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
-	struct arm_smmu_domain *smmu_domain;
-	struct arm_smmu_attached_domain *attached_domain;
-	unsigned long flags;
 
-	if (!master || pasid == 0)
+	if (pasid == 0)
 		return;
 
 	domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
@@ -3005,7 +3027,20 @@ static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
 	if (domain->type == IOMMU_DOMAIN_SVA)
 		return arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
 
-	smmu_domain = to_smmu_domain(domain);
+	arm_smmu_domain_remove_dev_pasid(dev, to_smmu_domain(domain), pasid);
+}
+
+void arm_smmu_domain_remove_dev_pasid(struct device *dev,
+				      struct arm_smmu_domain *smmu_domain,
+				      ioasid_t pasid)
+{
+	struct arm_smmu_attached_domain *attached_domain;
+	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
+	unsigned long flags;
+
+	if (!master)
+		return;
+
 	mutex_lock(&arm_smmu_asid_lock);
 	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
 	list_for_each_entry(attached_domain, &smmu_domain->attached_domains, domain_head) {
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +11 -0
index 4ac69427abf1c..2c33c0461036d 100644
@@ -761,6 +761,17 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
 int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
 				 unsigned long iova, size_t size);
+int arm_smmu_domain_set_dev_pasid(struct device *dev,
+				  struct arm_smmu_master *master,
+				  struct arm_smmu_domain *smmu_domain,
+				  ioasid_t pasid);
+void arm_smmu_domain_remove_dev_pasid(struct device *dev,
+				      struct arm_smmu_domain *smmu_domain,
+				      ioasid_t pasid);
+struct arm_smmu_domain *
+arm_smmu_init_shared_cd_domain(struct arm_smmu_device *smmu,
+			       struct arm_smmu_domain *smmu_domain,
+			       struct arm_smmu_ctx_desc *cd);
 
 #ifdef CONFIG_ARM_SMMU_V3_SVA
 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#32 lore raw

PATCH v2 16/18 iommu/arm-smmu-v3-sva: Attach S1_SHARED_CD domain

In reply to Michael Shavit

Prepare an smmu domain of type S1_SHARED_CD per smmu_mmu_notifier. Attach that domain using the common arm_smmu_domain_set_dev_pasid implementation when attaching an SVA domain.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   | 67 ++++++-------------
 1 file changed, 22 insertions(+), 45 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +22 -45
index e2a91f20f0906..9a2da579c3563 100644
@@ -19,7 +19,7 @@ struct arm_smmu_mmu_notifier {
 	bool				cleared;
 	refcount_t			refs;
 	struct list_head		list;
-	struct arm_smmu_domain		*domain;
+	struct arm_smmu_domain		domain;
 };
 
 #define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
@@ -198,7 +198,7 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
 					 unsigned long start, unsigned long end)
 {
 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
-	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+	struct arm_smmu_domain *smmu_domain = &smmu_mn->domain;
 	size_t size;
 
 	/*
@@ -217,7 +217,7 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 {
 	struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
-	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+	struct arm_smmu_domain *smmu_domain = &smmu_mn->domain;
 	struct arm_smmu_master *master;
 	struct arm_smmu_attached_domain *attached_domain;
 	unsigned long flags;
@@ -233,15 +233,10 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	 * but disable translation.
 	 */
 	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
-	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
-			    domain_head) {
+	list_for_each_entry(attached_domain, &smmu_domain->attached_domains, domain_head) {
 		master = attached_domain->master;
-		/*
-		 * SVA domains piggyback on the attached_domain with SSID 0.
-		 */
-		if (attached_domain->ssid == 0)
-			arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg,
-						master, mm->pasid, &quiet_cd);
+		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
+					attached_domain->ssid, &quiet_cd);
 	}
 	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 
@@ -265,15 +260,13 @@ static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
 
 /* Allocate or get existing MMU notifier for this {domain, mm} pair */
 static struct arm_smmu_mmu_notifier *
-arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
+arm_smmu_mmu_notifier_get(struct arm_smmu_device *smmu,
+			  struct arm_smmu_domain *smmu_domain,
 			  struct mm_struct *mm)
 {
 	int ret;
-	unsigned long flags;
 	struct arm_smmu_ctx_desc *cd;
 	struct arm_smmu_mmu_notifier *smmu_mn;
-	struct arm_smmu_master *master;
-	struct arm_smmu_attached_domain *attached_domain;
 
 	list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
 		if (smmu_mn->mn.mm == mm) {
@@ -294,7 +287,6 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 
 	refcount_set(&smmu_mn->refs, 1);
 	smmu_mn->cd = cd;
-	smmu_mn->domain = smmu_domain;
 	smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops;
 
 	ret = mmu_notifier_register(&smmu_mn->mn, mm);
@@ -302,24 +294,11 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 		kfree(smmu_mn);
 		goto err_free_cd;
 	}
-
-	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
-	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
-			    domain_head) {
-		master = attached_domain->master;
-		ret = arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg,
-					      master, mm->pasid, cd);
-	}
-	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
-	if (ret)
-		goto err_put_notifier;
+	arm_smmu_init_shared_cd_domain(smmu, &smmu_mn->domain, cd);
 
 	list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers);
 	return smmu_mn;
 
-err_put_notifier:
-	/* Frees smmu_mn */
-	mmu_notifier_put(&smmu_mn->mn);
 err_free_cd:
 	arm_smmu_free_shared_cd(cd);
 	return ERR_PTR(ret);
@@ -327,27 +306,15 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
 
 static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 {
-	unsigned long flags;
 	struct mm_struct *mm = smmu_mn->mn.mm;
 	struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
-	struct arm_smmu_attached_domain *attached_domain;
-	struct arm_smmu_master *master;
-	struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
+	struct arm_smmu_domain *smmu_domain = &smmu_mn->domain;
 
 	if (!refcount_dec_and_test(&smmu_mn->refs))
 		return;
 
 	list_del(&smmu_mn->list);
 
-	spin_lock_irqsave(&smmu_domain->attached_domains_lock, flags);
-	list_for_each_entry(attached_domain, &smmu_domain->attached_domains,
-			    domain_head) {
-		master = attached_domain->master;
-		arm_smmu_write_ctx_desc(master->smmu, master->s1_cfg, master,
-					mm->pasid, NULL);
-	}
-	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
-
 	/*
 	 * If we went through clear(), we've already invalidated, and no
 	 * new TLB entry can have been formed.
@@ -369,17 +336,26 @@ static int __arm_smmu_sva_bind(struct device *dev,
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	int ret;
 
 	if (!master || !master->sva_enabled)
 		return -ENODEV;
 
-	sva_domain->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain,
+	sva_domain->smmu_mn = arm_smmu_mmu_notifier_get(master->smmu,
+							smmu_domain,
 							mm);
 	if (IS_ERR(sva_domain->smmu_mn)) {
 		sva_domain->smmu_mn = NULL;
 		return PTR_ERR(sva_domain->smmu_mn);
 	}
+
 	master->nr_attached_sva_domains += 1;
+	smmu_domain = &sva_domain->smmu_mn->domain;
+	ret = arm_smmu_domain_set_dev_pasid(dev, master, smmu_domain, mm->pasid);
+	if (ret) {
+		arm_smmu_mmu_notifier_put(sva_domain->smmu_mn);
+		return ret;
+	}
 	return 0;
 }
 
@@ -544,8 +520,9 @@ void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 
 	mutex_lock(&sva_lock);
-	master->nr_attached_sva_domains -= 1;
+	arm_smmu_domain_remove_dev_pasid(dev, &sva_domain->smmu_mn->domain, id);
 	arm_smmu_mmu_notifier_put(sva_domain->smmu_mn);
+	master->nr_attached_sva_domains -= 1;
 	mutex_unlock(&sva_lock);
 }
 
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#33 lore raw

PATCH v2 17/18 iommu/arm-smmu-v3-sva: Alloc notifier for {smmu,mn}

In reply to Michael Shavit

The arm_smmu_nofitier for an mn can be shared across all devices with the same upstream smmu. This breaks the last remaining explicit dependency on the device's primary domain in arm-smmu-v3-sva.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c    | 18 +++++++-----------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c    |  2 +-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h    |  4 ++--
 3 files changed, 10 insertions(+), 14 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +7 -11
index 9a2da579c3563..3e49838e4f55c 100644
@@ -258,17 +258,16 @@ static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
 	.free_notifier		= arm_smmu_mmu_notifier_free,
 };
 
-/* Allocate or get existing MMU notifier for this {domain, mm} pair */
+/* Allocate or get existing MMU notifier for this {smmu, mm} pair */
 static struct arm_smmu_mmu_notifier *
 arm_smmu_mmu_notifier_get(struct arm_smmu_device *smmu,
-			  struct arm_smmu_domain *smmu_domain,
 			  struct mm_struct *mm)
 {
 	int ret;
 	struct arm_smmu_ctx_desc *cd;
 	struct arm_smmu_mmu_notifier *smmu_mn;
 
-	list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
+	list_for_each_entry(smmu_mn, &smmu->mmu_notifiers, list) {
 		if (smmu_mn->mn.mm == mm) {
 			refcount_inc(&smmu_mn->refs);
 			return smmu_mn;
@@ -296,9 +295,8 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_device *smmu,
 	}
 	arm_smmu_init_shared_cd_domain(smmu, &smmu_mn->domain, cd);
 
-	list_add(&smmu_mn->list, &smmu_domain->mmu_notifiers);
+	list_add(&smmu_mn->list, &smmu->mmu_notifiers);
 	return smmu_mn;
-
 err_free_cd:
 	arm_smmu_free_shared_cd(cd);
 	return ERR_PTR(ret);
@@ -314,7 +312,6 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 		return;
 
 	list_del(&smmu_mn->list);
-
 	/*
 	 * If we went through clear(), we've already invalidated, and no
 	 * new TLB entry can have been formed.
@@ -331,18 +328,17 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 
 static int __arm_smmu_sva_bind(struct device *dev,
 			       struct arm_smmu_sva_domain *sva_domain,
-			       struct mm_struct *mm)
+			       struct mm_struct *mm,
+			       ioasid_t id)
 {
 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
-	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
-	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
+	struct arm_smmu_domain *smmu_domain;
 	int ret;
 
 	if (!master || !master->sva_enabled)
 		return -ENODEV;
 
 	sva_domain->smmu_mn = arm_smmu_mmu_notifier_get(master->smmu,
-							smmu_domain,
 							mm);
 	if (IS_ERR(sva_domain->smmu_mn)) {
 		sva_domain->smmu_mn = NULL;
@@ -533,7 +529,7 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
 	struct mm_struct *mm = domain->mm;
 
 	mutex_lock(&sva_lock);
-	ret = __arm_smmu_sva_bind(dev, to_sva_domain(domain), mm);
+	ret = __arm_smmu_sva_bind(dev, to_sva_domain(domain), mm, id);
 	mutex_unlock(&sva_lock);
 
 	return ret;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +1 -1
index 3c5ff4f58934a..e68c5264c6171 100644
@@ -2058,7 +2058,6 @@ static void arm_smmu_init_smmu_domain(struct arm_smmu_domain *smmu_domain)
 	mutex_init(&smmu_domain->init_mutex);
 	INIT_LIST_HEAD(&smmu_domain->attached_domains);
 	spin_lock_init(&smmu_domain->attached_domains_lock);
-	INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
 }
 
 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
@@ -2859,6 +2858,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev)
 		goto err_free_master;
 	}
 
+	INIT_LIST_HEAD(&smmu->mmu_notifiers);
 	return &smmu->iommu;
 
 err_free_master:
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +2 -2
index 2c33c0461036d..041b0e532ac3d 100644
@@ -674,6 +674,8 @@ struct arm_smmu_device {
 
 	struct rb_root			streams;
 	struct mutex			streams_mutex;
+
+	struct list_head		mmu_notifiers;
 };
 
 struct arm_smmu_stream {
@@ -737,8 +739,6 @@ struct arm_smmu_domain {
 
 	struct list_head			attached_domains;
 	spinlock_t				attached_domains_lock;
-
-	struct list_head			mmu_notifiers;
 };
 
 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog
#34 lore raw

PATCH v2 18/18 iommu/arm-smmu-v3-sva: Remove atc_inv_domain_ssid

In reply to Michael Shavit

arm_smmu_atc_inv_domain is sufficient in all cases now that arm_smmu_domain always tracks all the master/ssids that it is attached to. Also remove the last usage of mm->pasid in arm-smmu-v3-sva.

---
 .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c   |  9 ++++----
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   | 21 +++----------------
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h   |  4 ++--
 3 files changed, 9 insertions(+), 25 deletions(-)
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +4 -5
index 3e49838e4f55c..5a124281bbef6 100644
@@ -211,7 +211,7 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
 		arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
 					    PAGE_SIZE, false, smmu_domain);
-	arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, start, size);
+	arm_smmu_atc_inv_domain(smmu_domain, start, size);
 }
 
 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -241,7 +241,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 	spin_unlock_irqrestore(&smmu_domain->attached_domains_lock, flags);
 
 	arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
-	arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
+	arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
 
 	smmu_mn->cleared = true;
 	mutex_unlock(&sva_lock);
@@ -304,7 +304,6 @@ arm_smmu_mmu_notifier_get(struct arm_smmu_device *smmu,
 
 static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 {
-	struct mm_struct *mm = smmu_mn->mn.mm;
 	struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
 	struct arm_smmu_domain *smmu_domain = &smmu_mn->domain;
 
@@ -318,7 +317,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
 	 */
 	if (!smmu_mn->cleared) {
 		arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
-		arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
+		arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
 	}
 
 	/* Frees smmu_mn */
@@ -347,7 +346,7 @@ static int __arm_smmu_sva_bind(struct device *dev,
 
 	master->nr_attached_sva_domains += 1;
 	smmu_domain = &sva_domain->smmu_mn->domain;
-	ret = arm_smmu_domain_set_dev_pasid(dev, master, smmu_domain, mm->pasid);
+	ret = arm_smmu_domain_set_dev_pasid(dev, master, smmu_domain, id);
 	if (ret) {
 		arm_smmu_mmu_notifier_put(sva_domain->smmu_mn);
 		return ret;
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +3 -18
index e68c5264c6171..1e02e73e586f7 100644
@@ -1821,13 +1821,8 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
 	return arm_smmu_atc_inv_master_ssid(master, 0);
 }
 
-/*
- * If ssid is non-zero, issue atc invalidations with the given ssid instead of
- * the one the domain is attached to. This is used by SVA since it's pasid
- * attachments aren't recorded in smmu_domain yet.
- */
-int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
-				 unsigned long iova, size_t size)
+int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
+			    unsigned long iova, size_t size)
 {
 	int i;
 	unsigned long flags;
@@ -1866,11 +1861,7 @@ int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
 		master = attached_domain->master;
 		if (!master->ats_enabled)
 			continue;
-		if (ssid != 0)
-			arm_smmu_atc_inv_cmd_set_ssid(ssid, &cmd);
-		else
-			arm_smmu_atc_inv_cmd_set_ssid(attached_domain->ssid, &cmd);
-
+		arm_smmu_atc_inv_cmd_set_ssid(attached_domain->ssid, &cmd);
 		for (i = 0; i < master->num_streams; i++) {
 			cmd.atc.sid = master->streams[i].id;
 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
@@ -1881,12 +1872,6 @@ int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
 }
 
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
-			    unsigned long iova, size_t size)
-{
-	return arm_smmu_atc_inv_domain_ssid(smmu_domain, 0, iova, size);
-}
-
 /* IO_PGTABLE API */
 static void arm_smmu_tlb_inv_context(void *cookie)
 {
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +2 -2
index 041b0e532ac3d..9c382bc8c0549 100644
@@ -759,8 +759,8 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
 				 size_t granule, bool leaf,
 				 struct arm_smmu_domain *smmu_domain);
 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
-int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
-				 unsigned long iova, size_t size);
+int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
+			    unsigned long iova, size_t size);
 int arm_smmu_domain_set_dev_pasid(struct device *dev,
 				  struct arm_smmu_master *master,
 				  struct arm_smmu_domain *smmu_domain,
Signature · 1 line
2.41.0.rc0.172.g3f132b7071-goog