Skip to contentSkip to conversation

iommu/vt-d: Refactor PRI enable/disable steps

PATCH Superseded Lu Baolu 3 participants 10 messages Jun 27, 2024 – Jul 2, 2024

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Lu Baolu

author maintainer
#0 lore raw

v2 cover iommu/vt-d: Refactor PRI enable/disable steps

The page fault handling framework within the iommu core has defined the PRI enable and disable flows in the comments for the iopf_queue_remove_device() interface. This series aims to refactor the PRI enable/disable steps in the Intel iommu driver to align with these definitions.

Change log:

v2:
- The cache invalidation for a context entry change should not affect the devices not related to the entry. Fix this by always using device-selective cache invalidation.
v1:
- https://lore.kernel.org/linux-iommu/[email protected]/

Lu Baolu (2):
iommu/vt-d: Add helper to flush caches for context change iommu/vt-d: Refactor PCI PRI enabling/disabling callbacks

 drivers/iommu/intel/iommu.h |  13 +++++
 drivers/iommu/intel/iommu.c |  89 ++++++++++++++++++-------------
 drivers/iommu/intel/pasid.c | 103 ++++++++++++++++++++++++++++--------
 3 files changed, 148 insertions(+), 57 deletions(-)
Signature · 1 line
2.34.1
author maintainer
#1 lore raw

PATCH v2 1/2 iommu/vt-d: Add helper to flush caches for context change

This helper is used to flush the related caches following a change in a context table entry that was previously present. The VT-d specification provides guidance for such invalidations in section 6.5.3.3.

This helper replaces the existing open code in the code paths where a present context entry is being torn down.

---
 drivers/iommu/intel/iommu.h |   4 ++
 drivers/iommu/intel/iommu.c |  32 +-----------
 drivers/iommu/intel/pasid.c | 101 +++++++++++++++++++++++++++++-------
 3 files changed, 87 insertions(+), 50 deletions(-)
drivers/iommu/intel/iommu.h +4 -0
index 9a3b064126de..63eb3306c025 100644
@@ -1143,6 +1143,10 @@ void cache_tag_flush_all(struct dmar_domain *domain);
 void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start,
 			      unsigned long end);
 
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains);
+
 #ifdef CONFIG_INTEL_IOMMU_SVM
 void intel_svm_check(struct intel_iommu *iommu);
 int intel_svm_enable_prq(struct intel_iommu *iommu);
drivers/iommu/intel/iommu.c +1 -31
index 6c682c9defee..89f489372138 100644
@@ -1359,21 +1359,6 @@ static void iommu_disable_pci_caps(struct device_domain_info *info)
 	}
 }
 
-static void __iommu_flush_dev_iotlb(struct device_domain_info *info,
-				    u64 addr, unsigned int mask)
-{
-	u16 sid, qdep;
-
-	if (!info || !info->ats_enabled)
-		return;
-
-	sid = info->bus << 8 | info->devfn;
-	qdep = info->ats_qdep;
-	qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
-			   qdep, addr, mask);
-	quirk_extra_dev_tlb_flush(info, addr, mask, IOMMU_NO_PASID, qdep);
-}
-
 static void intel_flush_iotlb_all(struct iommu_domain *domain)
 {
 	cache_tag_flush_all(to_dmar_domain(domain));
@@ -1959,7 +1944,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
 {
 	struct intel_iommu *iommu = info->iommu;
 	struct context_entry *context;
-	u16 did_old;
 
 	spin_lock(&iommu->lock);
 	context = iommu_context_addr(iommu, bus, devfn, 0);
@@ -1968,24 +1952,10 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
 		return;
 	}
 
-	did_old = context_domain_id(context);
-
 	context_clear_entry(context);
 	__iommu_flush_cache(iommu, context, sizeof(*context));
+	intel_context_flush_present(info, context, true);
 	spin_unlock(&iommu->lock);
-	iommu->flush.flush_context(iommu,
-				   did_old,
-				   (((u16)bus) << 8) | devfn,
-				   DMA_CCMD_MASK_NOBIT,
-				   DMA_CCMD_DEVICE_INVL);
-
-	iommu->flush.flush_iotlb(iommu,
-				 did_old,
-				 0,
-				 0,
-				 DMA_TLB_DSI_FLUSH);
-
-	__iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
 }
 
 static int domain_setup_first_level(struct intel_iommu *iommu,
drivers/iommu/intel/pasid.c +82 -19

111 lines not on this page · raw message

Signature · 1 line
2.34.1
author maintainer
#9 lore raw

PATCH v2 2/2 iommu/vt-d: Refactor PCI PRI enabling/disabling callbacks

In reply to Lu Baolu

Commit 0095bf83554f8 ("iommu: Improve iopf_queue_remove_device()") specified the flow for disabling the PRI on a device. Refactor the PRI callbacks in the intel iommu driver to better manage PRI enabling and disabling and align it with the device queue interfaces in the iommu core.

---
 drivers/iommu/intel/iommu.h |  9 ++++++
 drivers/iommu/intel/iommu.c | 57 +++++++++++++++++++++++++++++++++----
 drivers/iommu/intel/pasid.c |  2 --
 3 files changed, 61 insertions(+), 7 deletions(-)
drivers/iommu/intel/iommu.h +9 -0
index 63eb3306c025..b67c14da1240 100644
@@ -1045,6 +1045,15 @@ static inline void context_set_sm_pre(struct context_entry *context)
 	context->lo |= BIT_ULL(4);
 }
 
+/*
+ * Clear the PRE(Page Request Enable) field of a scalable mode context
+ * entry.
+ */
+static inline void context_clear_sm_pre(struct context_entry *context)
+{
+	context->lo &= ~BIT_ULL(4);
+}
+
 /* Returns a number of VTD pages, but aligned to MM page size */
 static inline unsigned long aligned_nrpages(unsigned long host_addr, size_t size)
 {
drivers/iommu/intel/iommu.c +52 -5

86 lines not on this page · raw message

drivers/iommu/intel/pasid.c +0 -2
index aef4b4afb873..b1ec9660bc11 100644
@@ -749,8 +749,6 @@ static int context_entry_set_pasid_table(struct context_entry *context,
 
 	if (info->ats_supported)
 		context_set_sm_dte(context);
-	if (info->pri_supported)
-		context_set_sm_pre(context);
 	if (info->pasid_supported)
 		context_set_pasid(context);
 
Signature · 1 line
2.34.1
#2 lore raw

RE: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

In reply to Lu Baolu
Quoted 8 lines

From: Lu Baolu <[email protected]>
Sent: Thursday, June 27, 2024 10:31 AM

+/*
+ * Cache invalidations after change in a context table entry that was present
+ * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If
+ * IOMMU is in scalable mode and all PASID table entries of the device were
+ * non-present, set affect_domains to true. Otherwise, false.

if no PASID is present then the flag should be false.

s/affect_domains/flush_domains/

Quoted 60 lines

+ */

drivers/iommu/intel/pasid.c +15 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID field
+	 * of the Context-cache Invalidate Descriptor is ignored by hardware
+	 * when operating in scalable mode. Therefore the @did value

doesn't

drivers/iommu/intel/pasid.c +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info-
Quoted 1 line

devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL);

drivers/iommu/intel/pasid.c +8 -0
+
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +19 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected domains
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +5 -1
+		}
+	}
+
+	__context_flush_dev_iotlb(info);
+}
--
Quoted 2 lines

2.34.1

this change moves the entire cache invalidation flow inside iommu->lock. Though the directly-affected operations are not in critical path the indirect impact applies to all other paths acquiring iommu->lock given it'll be held unnecessarily longer after this change.

If the only requirement of holding iommu->lock is to walk the pasid table, probably we can collect a bitmap of DID's in the locked walk and then invalidate each in a loop outside of iommu->lock. Here we only care about DIDs associated with the old context entry at this point. New pasid attach will hit new context entry. Concurrent pasid detach then may just come with duplicated invalidations.

author maintainer
#3 lore raw

Re: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

Quoted 9 lines · Kevin
Quoted 8 lines

From: Lu Baolu <[email protected]>
Sent: Thursday, June 27, 2024 10:31 AM

+/*
+ * Cache invalidations after change in a context table entry that was present
+ * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If
+ * IOMMU is in scalable mode and all PASID table entries of the device were
+ * non-present, set affect_domains to true. Otherwise, false.

Quoted 3 lines

if no PASID is present then the flag should be false.

s/affect_domains/flush_domains/

Yes.

Quoted 70 lines
Quoted 61 lines

+ */

drivers/iommu/intel/pasid.c +15 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID field
+	 * of the Context-cache Invalidate Descriptor is ignored by hardware
+	 * when operating in scalable mode. Therefore the @did value

doesn't

drivers/iommu/intel/pasid.c +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info-
Quoted 1 line

devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL);

drivers/iommu/intel/pasid.c +8 -0
+
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +19 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected domains
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +5 -1
+		}
+	}
+
+	__context_flush_dev_iotlb(info);
+}
--
Quoted 2 lines

2.34.1

this change moves the entire cache invalidation flow inside iommu->lock. Though the directly-affected operations are not in critical path the indirect impact applies to all other paths acquiring iommu->lock given it'll be held unnecessarily longer after this change.

Quoted 6 lines

If the only requirement of holding iommu->lock is to walk the pasid table, probably we can collect a bitmap of DID's in the locked walk and then invalidate each in a loop outside of iommu->lock. Here we only care about DIDs associated with the old context entry at this point. New pasid attach will hit new context entry. Concurrent pasid detach then may just come with duplicated invalidations.

The iommu->lock is not only for the PASID table walk. The basic schematic here is that once a present context table entry is being changed, all PASID entries should not be altered until all the related caches have been flushed. This is because the configuration of the context entry might also impact PASID translation.

Previously, we did not apply this lock because all those cases involved changing the context entry from present to non-present, and we were certain that all PASID entries were empty. Now, as we are making it a generic helper that also serves scenarios where the entry remains present after the change, we need this lock to ensure that no PASID entry changes occur at the same time.

Best regards,
baolu

#4 lore raw

RE: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

Quoted 83 lines

From: Baolu Lu <[email protected]>
Sent: Thursday, June 27, 2024 4:22 PM

Quoted 72 lines · Kevin
Quoted 63 lines

From: Lu Baolu <[email protected]>
Sent: Thursday, June 27, 2024 10:31 AM

+ */

drivers/iommu/intel/pasid.c +15 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID field
+	 * of the Context-cache Invalidate Descriptor is ignored by hardware
+	 * when operating in scalable mode. Therefore the @did value

doesn't

drivers/iommu/intel/pasid.c +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info-
Quoted 1 line

devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL);

drivers/iommu/intel/pasid.c +8 -0
+
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +19 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected domains
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +5 -1
+		}
+	}
+
+	__context_flush_dev_iotlb(info);
+}
--
Quoted 2 lines

2.34.1

this change moves the entire cache invalidation flow inside iommu->lock. Though the directly-affected operations are not in critical path the indirect impact applies to all other paths acquiring iommu->lock given it'll be held unnecessarily longer after this change.

Quoted 6 lines

If the only requirement of holding iommu->lock is to walk the pasid table, probably we can collect a bitmap of DID's in the locked walk and then invalidate each in a loop outside of iommu->lock. Here we only care about DIDs associated with the old context entry at this point. New pasid attach will hit new context entry. Concurrent pasid detach then may just come with duplicated invalidations.

Quoted 5 lines

The iommu->lock is not only for the PASID table walk. The basic schematic here is that once a present context table entry is being changed, all PASID entries should not be altered until all the related caches have been flushed. This is because the configuration of the context entry might also impact PASID translation.

Is it what the spec explicitly requires? My impression was that we need to invalidate any cache which may be associated with the old context entry, which is not equal to prohibiting PASID entries from being changed at the same time (as long as those changes won't cause a stale cache entry being active).

e.g. let's say this helper collects valid pasid entries (2, 3, 4) and associated DIDs (x, y, z) in a locked walk of the pasid table and then follows the spec sequence to invalidate the pasid cache for (2, 3, 4) and the iotlb for (x, y, z) outside of the lock.

there could be several concurrent scenarios if iommu->lock is only guarded on the pasid table walking:

1) pasid entry #1 is torn down before the locked walk. The teardown path will invalidate its pasid cache and iotlb properly.
2) pasid entry #4 is torn down after the locked walk. Then we may see duplicated invalidations both in this helper and in the teardown path.
3) new pasid attach before locked walk may be associated with either old or new context entry, depending on whether it's passed the context cache invalidation. In any case it will be captured by locked walk and then have related cache invalidated in the helper.
4) new pasid attach after locked walk is always safe as related cache will only be associated with the new context entry.

Quoted 8 lines

Previously, we did not apply this lock because all those cases involved changing the context entry from present to non-present, and we were certain that all PASID entries were empty. Now, as we are making it a generic helper that also serves scenarios where the entry remains present after the change, we need this lock to ensure that no PASID entry changes occur at the same time.

Even if we want to do a conservative locking I prefer to not applying it to existing paths which clearly have no need for extended lock.

Then probably making a specific helper for pri flip makes more sense than generalizing code to incur unnecessary lock overhead on existing paths...

author maintainer
#5 lore raw

Re: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

Quoted 89 lines · Kevin
Quoted 83 lines

From: Baolu Lu <[email protected]>
Sent: Thursday, June 27, 2024 4:22 PM

Quoted 72 lines · Kevin
Quoted 63 lines

From: Lu Baolu <[email protected]>
Sent: Thursday, June 27, 2024 10:31 AM

+ */

(quoted diff) +15 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID field
+	 * of the Context-cache Invalidate Descriptor is ignored by hardware
+	 * when operating in scalable mode. Therefore the @did value

doesn't

(quoted diff) +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info-
Quoted 1 line

devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL);

(quoted diff) +8 -0
+
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

(quoted diff) +19 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected domains
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

(quoted diff) +5 -1
+		}
+	}
+
+	__context_flush_dev_iotlb(info);
+}
--
Quoted 2 lines

2.34.1

this change moves the entire cache invalidation flow inside iommu->lock. Though the directly-affected operations are not in critical path the indirect impact applies to all other paths acquiring iommu->lock given it'll be held unnecessarily longer after this change.

Quoted 6 lines

If the only requirement of holding iommu->lock is to walk the pasid table, probably we can collect a bitmap of DID's in the locked walk and then invalidate each in a loop outside of iommu->lock. Here we only care about DIDs associated with the old context entry at this point. New pasid attach will hit new context entry. Concurrent pasid detach then may just come with duplicated invalidations.

Quoted 5 lines

The iommu->lock is not only for the PASID table walk. The basic schematic here is that once a present context table entry is being changed, all PASID entries should not be altered until all the related caches have been flushed. This is because the configuration of the context entry might also impact PASID translation.

Quoted 5 lines

Is it what the spec explicitly requires? My impression was that we need to invalidate any cache which may be associated with the old context entry, which is not equal to prohibiting PASID entries from being changed at the same time (as long as those changes won't cause a stale cache entry being active).

No, I didn't find that the VT-d spec explicitly requires this. But conceptually, the context entry controls features of the whole device, while pasid entries control the translation on a pasid. The change in context entry potentially impacts the pasid translation ...

Quoted 9 lines

e.g. let's say this helper collects valid pasid entries (2, 3, 4) and associated DIDs (x, y, z) in a locked walk of the pasid table and then follows the spec sequence to invalidate the pasid cache for (2, 3, 4) and the iotlb for (x, y, z) outside of the lock.

there could be several concurrent scenarios if iommu->lock is only guarded on the pasid table walking:

Quoted 11 lines

1) pasid entry #1 is torn down before the locked walk. The teardown path will invalidate its pasid cache and iotlb properly.
2) pasid entry #4 is torn down after the locked walk. Then we may see duplicated invalidations both in this helper and in the teardown path.
3) new pasid attach before locked walk may be associated with either old or new context entry, depending on whether it's passed the context cache invalidation. In any case it will be captured by locked walk and then have related cache invalidated in the helper.
4) new pasid attach after locked walk is always safe as related cache will only be associated with the new context entry.

... hence, from the driver's point of view, let's start from simplicity unless there's any real use case.

Quoted 9 lines
Quoted 8 lines

Previously, we did not apply this lock because all those cases involved changing the context entry from present to non-present, and we were certain that all PASID entries were empty. Now, as we are making it a generic helper that also serves scenarios where the entry remains present after the change, we need this lock to ensure that no PASID entry changes occur at the same time.

Quoted 6 lines

Even if we want to do a conservative locking I prefer to not applying it to existing paths which clearly have no need for extended lock.

Then probably making a specific helper for pri flip makes more sense than generalizing code to incur unnecessary lock overhead on existing paths...

Yes, agreed with you. We don't need to extend this lock mechanism change to the existing cases where it's unnecessary.

How about below additional changes?

drivers/iommu/intel/iommu.c +1 -1
index c5d9c2283977..523407f6f6b2 100644
@@ -1954,8 +1954,8 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
 
 	context_clear_entry(context);
 	__iommu_flush_cache(iommu, context, sizeof(*context));
-	intel_context_flush_present(info, context, true);
 	spin_unlock(&iommu->lock);
+	intel_context_flush_present(info, context, true);
 }
 
 static int domain_setup_first_level(struct intel_iommu *iommu,
drivers/iommu/intel/pasid.c +1 -1
index 01156135e60f..9a7b5668c723 100644
@@ -690,8 +690,8 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn)
 
 	context_clear_entry(context);
 	__iommu_flush_cache(iommu, context, sizeof(*context));
+	spin_unlock(&iommu->lock);
 	intel_context_flush_present(info, context, false);
-	spin_unlock(&iommu->lock);
 }
 
 static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias, 

void *data)
@@ -889,8 +889,6 @@ void intel_context_flush_present(struct device_domain_info *info,
struct pasid_entry *pte;
int i;

- assert_spin_locked(&iommu->lock);
-
/*
* Device-selective context-cache invalidation. The Domain-ID field
* of the Context-cache Invalidate Descriptor is ignored by hardware
@@ -919,6 +917,13 @@ void intel_context_flush_present(struct device_domain_info *info,
* - Global Device-TLB invalidation to affected functions
*/
if (flush_domains) {
+ /*
+ * If the IOMMU is running in scalable mode and there might
+ * be potential PASID translations, the caller should hold
+ * the lock to ensure that context changes and cache flushes
+ * are atomic.
+ */
+ assert_spin_locked(&iommu->lock);
for (i = 0; i < info->pasid_table->max_pasid; i++) {
pte = intel_pasid_get_entry(info->dev, i);
if (!pte || !pasid_pte_is_present(pte))

Best regards,
baolu

#6 lore raw

RE: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

Quoted 140 lines

From: Baolu Lu <[email protected]>
Sent: Friday, June 28, 2024 7:25 PM

Quoted 89 lines · Kevin
Quoted 83 lines

From: Baolu Lu <[email protected]>
Sent: Thursday, June 27, 2024 4:22 PM

Quoted 72 lines · Kevin
Quoted 63 lines

From: Lu Baolu <[email protected]>
Sent: Thursday, June 27, 2024 10:31 AM

+ */

(quoted diff) +15 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID field
+	 * of the Context-cache Invalidate Descriptor is ignored by hardware
+	 * when operating in scalable mode. Therefore the @did value

doesn't

(quoted diff) +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info-
Quoted 1 line

devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL);

(quoted diff) +8 -0
+
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

(quoted diff) +19 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected domains
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

(quoted diff) +5 -1
+		}
+	}
+
+	__context_flush_dev_iotlb(info);
+}
--
Quoted 2 lines

2.34.1

this change moves the entire cache invalidation flow inside iommu->lock. Though the directly-affected operations are not in critical path the indirect impact applies to all other paths acquiring iommu->lock given it'll be held unnecessarily longer after this change.

Quoted 6 lines

If the only requirement of holding iommu->lock is to walk the pasid table, probably we can collect a bitmap of DID's in the locked walk and then invalidate each in a loop outside of iommu->lock. Here we only care about DIDs associated with the old context entry at this point. New pasid attach will hit new context entry. Concurrent pasid detach then may just come with duplicated invalidations.

Quoted 5 lines

The iommu->lock is not only for the PASID table walk. The basic schematic here is that once a present context table entry is being changed, all PASID entries should not be altered until all the related caches have been flushed. This is because the configuration of the context entry might also impact PASID translation.

Quoted 5 lines

Is it what the spec explicitly requires? My impression was that we need to invalidate any cache which may be associated with the old context entry, which is not equal to prohibiting PASID entries from being changed at the same time (as long as those changes won't cause a stale cache entry being active).

No, I didn't find that the VT-d spec explicitly requires this. But conceptually, the context entry controls features of the whole device, while pasid entries control the translation on a pasid. The change in context entry potentially impacts the pasid translation ...

Quoted 9 lines

e.g. let's say this helper collects valid pasid entries (2, 3, 4) and associated DIDs (x, y, z) in a locked walk of the pasid table and then follows the spec sequence to invalidate the pasid cache for (2, 3, 4) and the iotlb for (x, y, z) outside of the lock.

there could be several concurrent scenarios if iommu->lock is only guarded on the pasid table walking:

Quoted 11 lines

1) pasid entry #1 is torn down before the locked walk. The teardown path will invalidate its pasid cache and iotlb properly.
2) pasid entry #4 is torn down after the locked walk. Then we may see duplicated invalidations both in this helper and in the teardown path.
3) new pasid attach before locked walk may be associated with either old or new context entry, depending on whether it's passed the context cache invalidation. In any case it will be captured by locked walk and then have related cache invalidated in the helper.
4) new pasid attach after locked walk is always safe as related cache will only be associated with the new context entry.

... hence, from the driver's point of view, let's start from simplicity unless there's any real use case.

Quoted 12 lines
Quoted 8 lines

Previously, we did not apply this lock because all those cases involved changing the context entry from present to non-present, and we were certain that all PASID entries were empty. Now, as we are making it a generic helper that also serves scenarios where the entry remains present after the change, we need this lock to ensure that no PASID entry changes occur at the same time.

Even if we want to do a conservative locking I prefer to not applying it to existing paths which clearly have no need for extended lock.

Quoted 8 lines
Quoted 3 lines

Then probably making a specific helper for pri flip makes more sense than generalizing code to incur unnecessary lock overhead on existing paths...

Yes, agreed with you. We don't need to extend this lock mechanism change to the existing cases where it's unnecessary.

How about below additional changes?

it's better

Quoted 60 lines
drivers/iommu/intel/iommu.c +0 -0
index c5d9c2283977..523407f6f6b2 100644
@@ -1954,8 +1954,8 @@ static void domain_context_clear_one(struct

device_domain_info *info, u8 bus, u8

  	context_clear_entry(context);
  	__iommu_flush_cache(iommu, context, sizeof(*context));
drivers/iommu/intel/iommu.c +1 -1
-	intel_context_flush_present(info, context, true);
  	spin_unlock(&iommu->lock);
+	intel_context_flush_present(info, context, true);
  }
 
  static int domain_setup_first_level(struct intel_iommu *iommu,
drivers/iommu/intel/pasid.c +1 -1
index 01156135e60f..9a7b5668c723 100644
@@ -690,8 +690,8 @@ static void device_pasid_table_teardown(struct
 device *dev, u8 bus, u8 devfn)
 
  	context_clear_entry(context);
  	__iommu_flush_cache(iommu, context, sizeof(*context));
+	spin_unlock(&iommu->lock);
  	intel_context_flush_present(info, context, false);
-	spin_unlock(&iommu->lock);
  }
 

static int pci_pasid_table_teardown(struct pci_dev *pdev, u16 alias,
void *data)

drivers/iommu/intel/pasid.c +0 -2
@@ -889,8 +889,6 @@ void intel_context_flush_present(struct
 device_domain_info *info,
  	struct pasid_entry *pte;
  	int i;
 
-	assert_spin_locked(&iommu->lock);
-
  	/*
  	 * Device-selective context-cache invalidation. The Domain-ID field

* of the Context-cache Invalidate Descriptor is ignored by hardware

drivers/iommu/intel/pasid.c +7 -0
@@ -919,6 +917,13 @@ void intel_context_flush_present(struct
 device_domain_info *info,
  	 * - Global Device-TLB invalidation to affected functions
  	 */
  	if (flush_domains) {
+		/*
+		 * If the IOMMU is running in scalable mode and there might
+		 * be potential PASID translations, the caller should hold
+		 * the lock to ensure that context changes and cache flushes
+		 * are atomic.
+		 */
+		assert_spin_locked(&iommu->lock);
  		for (i = 0; i < info->pasid_table->max_pasid; i++) {
  			pte = intel_pasid_get_entry(info->dev, i);
  			if (!pte || !pasid_pte_is_present(pte))

Best regards,
baolu

#7 lore raw

Re: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

In reply to Lu Baolu
Quoted 9 lines · Lu Baolu

+/*
+ * Cache invalidations after change in a context table entry that was present
+ * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If
+ * IOMMU is in scalable mode and all PASID table entries of the device were
+ * non-present, set affect_domains to true. Otherwise, false.
+ */

The spec says:
"Domain-selective PASID-cache invalidation to affected domains (can be skipped if all PASID entries were not-present and CM=0)"

So we should skip PASID cache invalidation if affect_domain is true according to this comment.

Quoted 52 lines
drivers/iommu/intel/pasid.c +13 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID

field

(quoted diff) +1 -0
+	 * of the Context-cache Invalidate Descriptor is ignored by

hardware

(quoted diff) +1 -0
+	 * when operating in scalable mode. Therefore the @did value

doesn't

drivers/iommu/intel/pasid.c +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus,

info->devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL); +

drivers/iommu/intel/pasid.c +7 -0
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +8 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected

domains

drivers/iommu/intel/pasid.c +11 -0
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,
Quoted 1 line

QI_PC_ALL_PASIDS, 0);

This is conflicting with the comments above where PASID cache flush can be skipped if affect_domain==true, no?

Review

Quoted 4 lines
(quoted diff) +1 -0
+			iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

(quoted diff) +2 -0
+		}
+	}

Thanks,

Jacob

author maintainer
#8 lore raw

Re: [PATCH v2 1/2] iommu/vt-d: Add helper to flush caches for context change

Quoted 66 lines · Jacob Pan
Quoted 9 lines · Lu Baolu

+/*
+ * Cache invalidations after change in a context table entry that was present
+ * according to the Spec 6.5.3.3 (Guidance to Software for Invalidations). If
+ * IOMMU is in scalable mode and all PASID table entries of the device were
+ * non-present, set affect_domains to true. Otherwise, false.
+ */

The spec says:
"Domain-selective PASID-cache invalidation to affected domains (can be skipped if all PASID entries were not-present and CM=0)"

So we should skip PASID cache invalidation if affect_domain is true according to this comment.

Quoted 47 lines
drivers/iommu/intel/pasid.c +13 -0
+void intel_context_flush_present(struct device_domain_info *info,
+				 struct context_entry *context,
+				 bool affect_domains)
+{
+	struct intel_iommu *iommu = info->iommu;
+	u16 did = context_domain_id(context);
+	struct pasid_entry *pte;
+	int i;
+
+	assert_spin_locked(&iommu->lock);
+
+	/*
+	 * Device-selective context-cache invalidation. The Domain-ID

field

(quoted diff) +1 -0
+	 * of the Context-cache Invalidate Descriptor is ignored by

hardware

(quoted diff) +1 -0
+	 * when operating in scalable mode. Therefore the @did value

doesn't

drivers/iommu/intel/pasid.c +3 -0
+	 * matter in scalable mode.
+	 */
+	iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus,

info->devfn),

(quoted diff) +1 -0
+				   DMA_CCMD_MASK_NOBIT,

DMA_CCMD_DEVICE_INVL); +

drivers/iommu/intel/pasid.c +7 -0
+	/*
+	 * For legacy mode:
+	 * - Domain-selective IOTLB invalidation
+	 * - Global Device-TLB invalidation to all affected functions
+	 */
+	if (!sm_supported(iommu)) {
+		iommu->flush.flush_iotlb(iommu, did, 0, 0,

DMA_TLB_DSI_FLUSH);

drivers/iommu/intel/pasid.c +8 -0
+		__context_flush_dev_iotlb(info);
+
+		return;
+	}
+
+	/*
+	 * For scalable mode:
+	 * - Domain-selective PASID-cache invalidation to affected

domains

drivers/iommu/intel/pasid.c +6 -0
+	 * - Domain-selective IOTLB invalidation to affected domains
+	 * - Global Device-TLB invalidation to affected functions
+	 */
+	if (affect_domains) {
+		for (i = 0; i < info->pasid_table->max_pasid; i++) {
+			pte = intel_pasid_get_entry(info->dev, i);
Quoted 8 lines
Quoted 6 lines
drivers/iommu/intel/pasid.c +5 -0
+			if (!pte || !pasid_pte_is_present(pte))
+				continue;
+
+			did = pasid_get_domain_id(pte);
+			qi_flush_pasid_cache(iommu, did,

QI_PC_ALL_PASIDS, 0);

This is conflicting with the comments above where PASID cache flush can be skipped if affect_domain==true, no?

Yes. I have fixed it in the v3.

Best regards,
baolu