Enable PASID for DMA API users

drivers/iommu/intel/iommu.c +9 -0

+static int intel_iommu_attach_dev_pasid(struct iommu_domain *domain,
+					struct device *dev,
+					ioasid_t pasid)
+{
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+	struct intel_iommu *iommu = info->iommu;
+	unsigned long flags;
+	int ret = 0;

Quoted 15 lines · Jacob Pan

drivers/iommu/intel/iommu.c +15 -0

+static int intel_iommu_attach_dev_pasid(struct iommu_domain *domain,
+					struct device *dev,
+					ioasid_t pasid)
+{
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+	struct intel_iommu *iommu = info->iommu;
+	unsigned long flags;
+	int ret = 0;
+
+	if (!sm_supported(iommu) || !info)
+		return -ENODEV;
+
+	spin_lock_irqsave(&device_domain_lock, flags);
+	/*

drivers/iommu/intel/iommu.c static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · line 4944

Quoted 5 lines

(quoted diff) +1 -0

+	 * If the same device already has a PASID attached, just

return.

drivers/iommu/intel/iommu.c +3 -0

+	 * DMA layer will return the PASID value to the caller.
+	 */
+	if (pasid != PASID_RID2PASID && info->pasid) {  

Why check for PASID == 0 like this? Shouldn't pasid == 0 be rejected as an invalid argument?

drivers/iommu/intel/iommu.c static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain, · line 4946

Quoted 3 lines

drivers/iommu/intel/iommu.c +2 -0

+		if (info->pasid == pasid)
+			ret = 0;  

Doesn't this need to check that the current domain is the requested domain as well? How can this happen anyhow - isn't it an error to double attach?

include/linux/intel-iommu.h struct device_domain_info { · line 630

Quoted 8 lines

include/linux/intel-iommu.h +2 -0

index 5af24befc9f1..55845a8c4f4d 100644
+++ b/include/linux/intel-iommu.h

@@ -627,6 +627,7 @@ struct device_domain_info {
 	struct intel_iommu *iommu; /* IOMMU used by this device */
 	struct dmar_domain *domain; /* pointer to domain */
 	struct pasid_table *pasid_table; /* pasid table */
+	ioasid_t pasid; /* DMA request with PASID */  

And this seems wrong - the DMA API is not the only user of attach_dev_pasid, so there should not be any global pasid for the device.

I suspect this should be a counter of # of pasid domains attached so that the special flush logic triggers

And rely on the core code to worry about assigning only one domain per pasid - this should really be a 'set' function.

Quoted 4 lines

Quoted 4 lines

include/linux/intel-iommu.h +1 -0

index 5af24befc9f1..55845a8c4f4d 100644
+++ b/include/linux/intel-iommu.h

@@ -627,6 +627,7 @@ struct device_domain_info {

Quoted 9 lines

include/linux/intel-iommu.h struct device_domain_info { · line 630

Quoted 4 lines

include/linux/intel-iommu.h +1 -0

 	struct intel_iommu *iommu; /* IOMMU used by this device */
 	struct dmar_domain *domain; /* pointer to domain */
 	struct pasid_table *pasid_table; /* pasid table */
+	ioasid_t pasid; /* DMA request with PASID */  

And this seems wrong - the DMA API is not the only user of attach_dev_pasid, so there should not be any global pasid for the device.

True but the attach_dev_pasid() op is domain type specific. i.e. DMA API has its own attach_dev_pasid which is different than sva domain attach_dev_pasid().

device_domain_info is only used by DMA API.

Quoted 3 lines

I suspect this should be a counter of # of pasid domains attached so that the special flush logic triggers

This field is only used for devTLB, so it is per domain-device. struct device_domain_info is allocated per device-domain as well. Sorry, I might have totally missed your point.

Quoted 2 lines

And rely on the core code to worry about assigning only one domain per pasid - this should really be a 'set' function.

Yes, in this set the core code (in dma-iommu.c) only assign one PASID per DMA domain type.

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

Quoted 13 lines · Jacob Pan

Quoted 13 lines

include/linux/intel-iommu.h struct device_domain_info { · line 630

Quoted 8 lines

(quoted diff) +0 -0

b/include/linux/intel-iommu.h index 5af24befc9f1..55845a8c4f4d 100644 +++ b/include/linux/intel-iommu.h

include/linux/intel-iommu.h +1 -0

@@ -627,6 +627,7 @@ struct device_domain_info {
 	struct intel_iommu *iommu; /* IOMMU used by this device */
 	struct dmar_domain *domain; /* pointer to domain */
 	struct pasid_table *pasid_table; /* pasid table */
+	ioasid_t pasid; /* DMA request with PASID */    

And this seems wrong - the DMA API is not the only user of attach_dev_pasid, so there should not be any global pasid for the device.

Quoted 3 lines

True but the attach_dev_pasid() op is domain type specific. i.e. DMA API has its own attach_dev_pasid which is different than sva domain attach_dev_pasid().

Huh? The intel driver shares the same ops between UNMANAGED and DMA - and in general I do not think we should be putting special knowledge about the DMA domains in the drivers. Drivers should continue to treat them identically to UNMANAGED.

Quoted 1 line

device_domain_info is only used by DMA API.

Huh?

Quoted 4 lines

Quoted 4 lines

I suspect this should be a counter of # of pasid domains attached so that the special flush logic triggers

Quoted 3 lines

This field is only used for devTLB, so it is per domain-device. struct device_domain_info is allocated per device-domain as well. Sorry, I might have totally missed your point.

You can't store a single pasid in the driver like this, since the only thing it does is trigger the flush logic just count how many pasids are used by the device-domain and trigger pasid flush if any pasids are attached

Quoted 8 lines

Quoted 2 lines

And rely on the core code to worry about assigning only one domain per pasid - this should really be a 'set' function.

Yes, in this set the core code (in dma-iommu.c) only assign one PASID per DMA domain type.

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Quoted 5 lines

Huh? The intel driver shares the same ops between UNMANAGED and DMA - and in general I do not think we should be putting special knowledge about the DMA domains in the drivers. Drivers should continue to treat them identically to UNMANAGED.

OK, other than SVA domain, the rest domain types share the same default ops. I agree that the default ops should be the same for UNMANAGED, IDENTITY, and DMA domain types. Minor detail is that we need to treat IDENTITY domain slightly different when it comes down to PASID entry programming.

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Quoted 5 lines

Quoted 2 lines

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Sounds good, this operation has little in common with domain_ops.dev_attach_pasid() used by SVA domain. So I will add a new domain_ops.dev_set_pasid()

Quoted 5 lines · Jacob Pan

Quoted 5 lines

Huh? The intel driver shares the same ops between UNMANAGED and DMA - and in general I do not think we should be putting special knowledge about the DMA domains in the drivers. Drivers should continue to treat them identically to UNMANAGED.

Quoted 5 lines

OK, other than SVA domain, the rest domain types share the same default ops. I agree that the default ops should be the same for UNMANAGED, IDENTITY, and DMA domain types. Minor detail is that we need to treat IDENTITY domain slightly different when it comes down to PASID entry programming.

I would be happy if IDENTITY had its own ops, if that makes sense

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

Quoted 8 lines

Quoted 5 lines

Quoted 2 lines

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Sounds good, this operation has little in common with domain_ops.dev_attach_pasid() used by SVA domain. So I will add a new domain_ops.dev_set_pasid()

What? No, their should only be one operation, 'dev_set_pasid' and it is exactly the same as the SVA operation. It configures things so that any existing translation on the PASID is removed and the PASID translates according to the given domain.

SVA given domain or UNMANAGED given domain doesn't matter to the higher level code. The driver should implement per-domain ops as required to get the different behaviors.

Quoted 9 lines

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

I am not following, here are the PASIDs for devTLB flush which is per device. Why group?

We could retrieve PASIDs from the device PASID table but xa would be more efficient.

Quoted 14 lines

Quoted 8 lines

Quoted 5 lines

Quoted 2 lines

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Sounds good, this operation has little in common with domain_ops.dev_attach_pasid() used by SVA domain. So I will add a new domain_ops.dev_set_pasid()

What? No, their should only be one operation, 'dev_set_pasid' and it is exactly the same as the SVA operation. It configures things so that any existing translation on the PASID is removed and the PASID translates according to the given domain.

Quoted 3 lines

SVA given domain or UNMANAGED given domain doesn't matter to the higher level code. The driver should implement per-domain ops as required to get the different behaviors.

Perhaps some code to clarify, we have sva_domain_ops.dev_attach_pasid() = intel_svm_attach_dev_pasid; default_domain_ops.dev_attach_pasid() = intel_iommu_attach_dev_pasid;

Consolidate pasid programming into dev_set_pasid() then called by both intel_svm_attach_dev_pasid() and intel_iommu_attach_dev_pasid(), right?

Quoted 5 lines · Jacob Pan

Quoted 5 lines

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Quoted 6 lines

Quoted 4 lines

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

I am not following, here are the PASIDs for devTLB flush which is per device. Why group?

Because group is where the core code stores it.

Quoted 18 lines

We could retrieve PASIDs from the device PASID table but xa would be more efficient.

Quoted 14 lines

Quoted 8 lines

Quoted 5 lines

Quoted 2 lines

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Sounds good, this operation has little in common with domain_ops.dev_attach_pasid() used by SVA domain. So I will add a new domain_ops.dev_set_pasid()

What? No, their should only be one operation, 'dev_set_pasid' and it is exactly the same as the SVA operation. It configures things so that any existing translation on the PASID is removed and the PASID translates according to the given domain.

Quoted 6 lines

Quoted 3 lines

SVA given domain or UNMANAGED given domain doesn't matter to the higher level code. The driver should implement per-domain ops as required to get the different behaviors.

Perhaps some code to clarify, we have sva_domain_ops.dev_attach_pasid() = intel_svm_attach_dev_pasid; default_domain_ops.dev_attach_pasid() = intel_iommu_attach_dev_pasid;

Yes, keep that structure

Quoted 3 lines

Consolidate pasid programming into dev_set_pasid() then called by both intel_svm_attach_dev_pasid() and intel_iommu_attach_dev_pasid(), right?

I was only suggesting that really dev_attach_pasid() op is misnamed, it should be called set_dev_pasid() and act like a set, not a paired attach/detach - same as the non-PASID ops.

Hi Jason,

Quoted 13 lines · Jason Gunthorpe

Quoted 11 lines · Jacob Pan

Quoted 9 lines

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

I am not following, here are the PASIDs for devTLB flush which is per device. Why group?

Quoted 1 line

Because group is where the core code stores it.

I see, with singleton group. I guess I can let dma-iommu code call

iommu_attach_dma_pasid {
	iommu_attach_device_pasid();

Then the PASID will be stored in the group xa.

The flush code can retrieve PASIDs from device_domain_info.device -> group -> pasid_array. Thanks for pointing it out, I missed the new pasid_array.

Quoted 17 lines · Jacob Pan

Hi Jason,

Quoted 13 lines · Jason Gunthorpe

Quoted 11 lines · Jacob Pan

Quoted 9 lines

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

I am not following, here are the PASIDs for devTLB flush which is per device. Why group?

Quoted 6 lines

Quoted 1 line

Because group is where the core code stores it.

I see, with singleton group. I guess I can let dma-iommu code call

iommu_attach_dma_pasid {
	iommu_attach_device_pasid();

Then the PASID will be stored in the group xa.

Yes, again, the dma-iommu should not be any different from the normal unmanaged path. At this point there is no longer any difference, we should not invent new ones.

Quoted 3 lines

The flush code can retrieve PASIDs from device_domain_info.device -> group -> pasid_array. Thanks for pointing it out, I missed the new pasid_array.

Yes.. It seems inefficient to iterate over that xarray multiple times on the flush hot path, but maybe there is little choice. Try to use use the xas iterators under the xa_lock spinlock..

The challenge will be accessing the group xa in the first place, but maybe the core code can gain a function call to return a pointer to that XA or something..

Quoted 4 lines

Yes.. It seems inefficient to iterate over that xarray multiple times on the flush hot path, but maybe there is little choice. Try to use use the xas iterators under the xa_lock spinlock..

xas_for_each takes a max range, here we don't really have one. So I posted v4 w/o using the xas advanced API. Please let me know if you have suggestions.

xa_for_each takes RCU read lock, it should be fast for tlb flush, right? The worst case maybe over flush when we have stale data but should be very rare.

Quoted 3 lines

The challenge will be accessing the group xa in the first place, but maybe the core code can gain a function call to return a pointer to that XA or something..

I added a helper function to find the matching DMA API PASID in v4.

Quoted 4 lines · Jacob Pan

Quoted 4 lines

Yes.. It seems inefficient to iterate over that xarray multiple times on the flush hot path, but maybe there is little choice. Try to use use the xas iterators under the xa_lock spinlock..

Quoted 3 lines

xas_for_each takes a max range, here we don't really have one. So I posted v4 w/o using the xas advanced API. Please let me know if you have suggestions.

You are supposed to use ULONG_MAX in cases like that.

Quoted 3 lines

xa_for_each takes RCU read lock, it should be fast for tlb flush, right? The worst case maybe over flush when we have stale data but should be very rare.

Not really, xa_for_each walks the tree for every iteration, it is slower than a linked list walk in any cases where the xarray is multi-node. xas_for_each is able to retain a pointer where it is in the tree so each iteration is usually just a pointer increment.

The downside is you cannot sleep while doing xas_for_each

Quoted 5 lines

Quoted 3 lines

The challenge will be accessing the group xa in the first place, but maybe the core code can gain a function call to return a pointer to that XA or something..

I added a helper function to find the matching DMA API PASID in v4.

Again, why are we focused on DMA API? Nothing you build here should be DMA API beyond the fact that the iommu_domain being attached is the default domain.

Hi Jason,

Quoted 6 lines · Jason Gunthorpe

Quoted 5 lines · Jacob Pan

Quoted 5 lines

Quoted 4 lines

If not global, perhaps we could have a list of pasids (e.g. xarray) attached to the device_domain_info. The TLB flush logic would just go through the list w/o caring what the PASIDs are for. Does it make sense to you?

Quoted 8 lines

Quoted 6 lines

Quoted 4 lines

Sort of, but we shouldn't duplicate xarrays - the group already has this xarray - need to find some way to allow access to it from the driver.

I am not following, here are the PASIDs for devTLB flush which is per device. Why group?

Because group is where the core code stores it.

I see, with singleton group. I guess I can let dma-iommu code call

iommu_attach_dma_pasid {
	iommu_attach_device_pasid();

Then the PASID will be stored in the group xa.
The flush code can retrieve PASIDs from device_domain_info.device -> group -> pasid_array.
Thanks for pointing it out, I missed the new pasid_array.

Quoted 31 lines

Quoted 18 lines

We could retrieve PASIDs from the device PASID table but xa would be more efficient.

Quoted 14 lines

Quoted 8 lines

Quoted 5 lines

Quoted 2 lines

Are you suggesting the dma-iommu API should be called iommu_set_dma_pasid instead of iommu_attach_dma_pasid?

No that API is Ok - the driver ops API should be 'set' not attach/detach

Sounds good, this operation has little in common with domain_ops.dev_attach_pasid() used by SVA domain. So I will add a new domain_ops.dev_set_pasid()

What? No, their should only be one operation, 'dev_set_pasid' and it is exactly the same as the SVA operation. It configures things so that any existing translation on the PASID is removed and the PASID translates according to the given domain.

Quoted 6 lines

Quoted 3 lines

SVA given domain or UNMANAGED given domain doesn't matter to the higher level code. The driver should implement per-domain ops as required to get the different behaviors.

Perhaps some code to clarify, we have sva_domain_ops.dev_attach_pasid() = intel_svm_attach_dev_pasid; default_domain_ops.dev_attach_pasid() = intel_iommu_attach_dev_pasid;

Yes, keep that structure

Quoted 3 lines

Consolidate pasid programming into dev_set_pasid() then called by both intel_svm_attach_dev_pasid() and intel_iommu_attach_dev_pasid(), right?

Quoted 4 lines

I was only suggesting that really dev_attach_pasid() op is misnamed, it should be called set_dev_pasid() and act like a set, not a paired attach/detach - same as the non-PASID ops.

Got it. Perhaps another patch to rename, Baolu?

Quoted 2 lines

Consolidate pasid programming into dev_set_pasid() then called by both intel_svm_attach_dev_pasid() and intel_iommu_attach_dev_pasid(), right?

I was only suggesting that really dev_attach_pasid() op is misnamed, it should be called set_dev_pasid() and act like a set, not a paired attach/detach - same as the non-PASID ops.

Hi Jason,

Quoted 5 lines · Jason Gunthorpe

Quoted 2 lines

Consolidate pasid programming into dev_set_pasid() then called by both intel_svm_attach_dev_pasid() and intel_iommu_attach_dev_pasid(), right?

I was only suggesting that really dev_attach_pasid() op is misnamed, it should be called set_dev_pasid() and act like a set, not a paired attach/detach - same as the non-PASID ops.

So,

"set_dev_pasid(domain, device, pasid)" equals to dev_attach_pasid()

and

"set_dev_pasid(NULL, device, pasid)" equals to dev_detach_pasid()?

do I understand it right?

DMA mapping API is the de facto standard for in-kernel DMA. It operates on a per device/RID basis which is not PASID-aware.

Some modern devices such as Intel Data Streaming Accelerator, PASID is required for certain work submissions. To allow such devices use DMA mapping API, we need the following functionalities:
1. Provide device a way to retrieve a PASID for work submission within the kernel
2. Enable the kernel PASID on the IOMMU for the device
3. Attach the kernel PASID to the device's default DMA domain, let it be IOVA or physical address in case of pass-through.

This patch introduces a driver facing API that enables DMA API PASID usage. Once enabled, device drivers can continue to use DMA APIs as is. There is no difference in dma_handle between without PASID and with PASID.

Signed-off-by Jacob Pan <[email protected]>

 drivers/iommu/dma-iommu.c | 107 ++++++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h |   3 ++
 include/linux/iommu.h     |   2 +
 3 files changed, 112 insertions(+)

drivers/iommu/dma-iommu.c +19 -0

index 1ca85d37eeab..5984f3129fa2 100644
+++ b/drivers/iommu/dma-iommu.c

@@ -34,6 +34,8 @@ struct iommu_dma_msi_page {
 	phys_addr_t		phys;
 };
 
+static DECLARE_IOASID_SET(iommu_dma_pasid);
+
 enum iommu_dma_cookie_type {
 	IOMMU_DMA_IOVA_COOKIE,
 	IOMMU_DMA_MSI_COOKIE,

@@ -370,6 +372,111 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
 	domain->iova_cookie = NULL;
 }
 
+/**
+ * iommu_attach_dma_pasid --Attach a PASID for in-kernel DMA. Use the device's
+ * DMA domain.
+ * @dev: Device to be enabled
+ * @pasid: The returned kernel PASID to be used for DMA
+ *
+ * DMA request with PASID will be mapped the same way as the legacy DMA.
+ * If the device is in pass-through, PASID will also pass-through. If the
+ * device is in IOVA, the PASID will point to the same IOVA page table.
+ *
+ * @return err code or 0 on success
+ */
+int iommu_attach_dma_pasid(struct device *dev, ioasid_t *pasid)
+{
+	struct iommu_domain *dom;
+	ioasid_t id, max;

drivers/iommu/dma-iommu.c +10 -0

+		return -EPERM;
+	}
+
+	id = dom->pasid;
+	if (!id) {
+		/*
+		 * First device to use PASID in its DMA domain, allocate
+		 * a single PASID per DMA domain is all we need, it is also
+		 * good for performance when it comes down to IOTLB flush.
+		 */

drivers/iommu/dma-iommu.c +40 -0

+	}
+
+	ret = dom->ops->attach_dev_pasid(dom, dev, id);
+	if (!ret) {
+		*pasid = id;
+		atomic_inc(&dom->pasid_users);
+		return 0;
+	}
+
+	if (atomic_dec_and_test(&dom->pasid_users)) {
+		ioasid_free(id);
+		dom->pasid = 0;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(iommu_attach_dma_pasid);
+
+/**
+ * iommu_detach_dma_pasid --Disable in-kernel DMA request with PASID
+ * @dev:	Device's PASID DMA to be disabled
+ *
+ * It is the device driver's responsibility to ensure no more incoming DMA
+ * requests with the kernel PASID before calling this function. IOMMU driver
+ * ensures PASID cache, IOTLBs related to the kernel PASID are cleared and
+ * drained.
+ *
+ */
+void iommu_detach_dma_pasid(struct device *dev)
+{
+	struct iommu_domain *dom;
+	ioasid_t pasid;
+
+	dom = iommu_get_domain_for_dev(dev);
+	if (!dom || !dom->ops || !dom->ops->detach_dev_pasid) {
+		dev_warn(dev, "No ops for detaching PASID %u", pasid);
+		return;
+	}
+	/* Only support DMA API managed domain type */
+	if (dom->type == IOMMU_DOMAIN_UNMANAGED ||

drivers/iommu/dma-iommu.c +8 -0

+	dom->ops->detach_dev_pasid(dom, dev, pasid);
+	if (atomic_dec_and_test(&dom->pasid_users)) {
+		ioasid_free(pasid);
+		dom->pasid = 0;
+	}
+}
+EXPORT_SYMBOL(iommu_detach_dma_pasid);
+
 /**
  * iommu_dma_get_resv_regions - Reserved region driver helper
  * @dev: Device from iommu_get_resv_regions()

include/linux/dma-iommu.h +4 -0

index 24607dc3c2ac..538650b9cb75 100644
+++ b/include/linux/dma-iommu.h

@@ -18,6 +18,9 @@ int iommu_get_dma_cookie(struct iommu_domain *domain);
 int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
 void iommu_put_dma_cookie(struct iommu_domain *domain);
 
+int iommu_attach_dma_pasid(struct device *dev, ioasid_t *pasid);
+void iommu_detach_dma_pasid(struct device *dev);
+
 /* Setup call for arch DMA mapping code */
 void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit);
 int iommu_dma_init_fq(struct iommu_domain *domain);

include/linux/iommu.h +1 -0

index 1164524814cb..281a87fdce77 100644
+++ b/include/linux/iommu.h

Quoted 62 lines · Jacob Pan

DMA mapping API is the de facto standard for in-kernel DMA. It operates on a per device/RID basis which is not PASID-aware.

Some modern devices such as Intel Data Streaming Accelerator, PASID is required for certain work submissions. To allow such devices use DMA mapping API, we need the following functionalities:
1. Provide device a way to retrieve a PASID for work submission within the kernel
2. Enable the kernel PASID on the IOMMU for the device
3. Attach the kernel PASID to the device's default DMA domain, let it be IOVA or physical address in case of pass-through.

This patch introduces a driver facing API that enables DMA API PASID usage. Once enabled, device drivers can continue to use DMA APIs as is. There is no difference in dma_handle between without PASID and with PASID.

Signed-off-by Jacob Pan <[email protected]>

 drivers/iommu/dma-iommu.c | 107 ++++++++++++++++++++++++++++++++++++++
 include/linux/dma-iommu.h |   3 ++
 include/linux/iommu.h     |   2 +
 3 files changed, 112 insertions(+)

drivers/iommu/dma-iommu.c +5 -0

index 1ca85d37eeab..5984f3129fa2 100644
+++ b/drivers/iommu/dma-iommu.c

@@ -34,6 +34,8 @@ struct iommu_dma_msi_page {
 	phys_addr_t		phys;
 };
 
+static DECLARE_IOASID_SET(iommu_dma_pasid);
+
 enum iommu_dma_cookie_type {
 	IOMMU_DMA_IOVA_COOKIE,
 	IOMMU_DMA_MSI_COOKIE,

@@ -370,6 +372,111 @@ void iommu_put_dma_cookie(struct iommu_domain
 *domain) domain->iova_cookie = NULL;
 }
 
+/**
+ * iommu_attach_dma_pasid --Attach a PASID for in-kernel DMA. Use the

device's
+ * DMA domain.
+ * @dev: Device to be enabled
+ * @pasid: The returned kernel PASID to be used for DMA
+ *
+ * DMA request with PASID will be mapped the same way as the legacy DMA.
+ * If the device is in pass-through, PASID will also pass-through. If the
+ * device is in IOVA, the PASID will point to the same IOVA page table.
+ *
+ * @return err code or 0 on success
+ */

drivers/iommu/dma-iommu.c +8 -0

+int iommu_attach_dma_pasid(struct device *dev, ioasid_t *pasid)
+{
+	struct iommu_domain *dom;
+	ioasid_t id, max;
+	int ret = 0;
+
+	dom = iommu_get_domain_for_dev(dev);
+	if (!dom || !dom->ops || !dom->ops->attach_dev_pasid)

drivers/iommu/dma-iommu.c void iommu_put_dma_cookie(struct iommu_domain *domain) · line 400

Quoted 6 lines

drivers/iommu/dma-iommu.c +6 -0

+		return -ENODEV;
+
+	/* Only support domain types that DMA API can be used */
+	if (dom->type == IOMMU_DOMAIN_UNMANAGED ||
+	    dom->type == IOMMU_DOMAIN_BLOCKED) {
+		dev_warn(dev, "Invalid domain type %d", dom->type);  

This should be a WARN_ON

Quoted 17 lines

drivers/iommu/dma-iommu.c +7 -0

+		return -EPERM;
+	}
+
+	id = dom->pasid;
+	if (!id) {
+		/*
+		 * First device to use PASID in its DMA domain,

allocate

drivers/iommu/dma-iommu.c +1 -0

+		 * a single PASID per DMA domain is all we need, it is

also

drivers/iommu/dma-iommu.c +1 -0

+		 * good for performance when it comes down to IOTLB

flush.

drivers/iommu/dma-iommu.c +5 -0

+		 */
+		max = 1U << dev->iommu->pasid_bits;
+		if (!max)
+			return -EINVAL;
+

drivers/iommu/dma-iommu.c void iommu_put_dma_cookie(struct iommu_domain *domain) · line 420

Quoted 6 lines

drivers/iommu/dma-iommu.c +6 -0

+		id = ioasid_alloc(&iommu_dma_pasid, 1, max, dev);
+		if (id == INVALID_IOASID)
+			return -ENOMEM;
+
+		dom->pasid = id;
+		atomic_set(&dom->pasid_users, 1);  

All of this needs proper locking.

Quoted 48 lines

drivers/iommu/dma-iommu.c +23 -0

+	}
+
+	ret = dom->ops->attach_dev_pasid(dom, dev, id);
+	if (!ret) {
+		*pasid = id;
+		atomic_inc(&dom->pasid_users);
+		return 0;
+	}
+
+	if (atomic_dec_and_test(&dom->pasid_users)) {
+		ioasid_free(id);
+		dom->pasid = 0;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(iommu_attach_dma_pasid);
+
+/**
+ * iommu_detach_dma_pasid --Disable in-kernel DMA request with PASID
+ * @dev:	Device's PASID DMA to be disabled
+ *
+ * It is the device driver's responsibility to ensure no more incoming

DMA
+ * requests with the kernel PASID before calling this function. IOMMU driver
+ * ensures PASID cache, IOTLBs related to the kernel PASID are cleared and
+ * drained.
+ *
+ */

drivers/iommu/dma-iommu.c +14 -0

+void iommu_detach_dma_pasid(struct device *dev)
+{
+	struct iommu_domain *dom;
+	ioasid_t pasid;
+
+	dom = iommu_get_domain_for_dev(dev);
+	if (!dom || !dom->ops || !dom->ops->detach_dev_pasid) {
+		dev_warn(dev, "No ops for detaching PASID %u", pasid);
+		return;
+	}
+	/* Only support DMA API managed domain type */
+	if (dom->type == IOMMU_DOMAIN_UNMANAGED ||
+	    dom->type == IOMMU_DOMAIN_BLOCKED) {
+		dev_err(dev, "Invalid domain type %d to detach DMA

PASID %u\n",

drivers/iommu/dma-iommu.c +2 -0

+			 dom->type, pasid);
+		return;

drivers/iommu/dma-iommu.c void iommu_put_dma_cookie(struct iommu_domain *domain) · line 471

Quoted 6 lines

drivers/iommu/dma-iommu.c +6 -0

+	}
+	pasid = dom->pasid;
+	if (!pasid) {
+		dev_err(dev, "No DMA PASID attached\n");
+		return;
+	}  

All WARN_ON's too

Quoted 29 lines

drivers/iommu/dma-iommu.c +8 -0

+	dom->ops->detach_dev_pasid(dom, dev, pasid);
+	if (atomic_dec_and_test(&dom->pasid_users)) {
+		ioasid_free(pasid);
+		dom->pasid = 0;
+	}
+}
+EXPORT_SYMBOL(iommu_detach_dma_pasid);
+
 /**
  * iommu_dma_get_resv_regions - Reserved region driver helper
  * @dev: Device from iommu_get_resv_regions()

include/linux/dma-iommu.h +4 -0

index 24607dc3c2ac..538650b9cb75 100644
+++ b/include/linux/dma-iommu.h

@@ -18,6 +18,9 @@ int iommu_get_dma_cookie(struct iommu_domain *domain);
 int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
 void iommu_put_dma_cookie(struct iommu_domain *domain);
 
+int iommu_attach_dma_pasid(struct device *dev, ioasid_t *pasid);
+void iommu_detach_dma_pasid(struct device *dev);
+
 /* Setup call for arch DMA mapping code */
 void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64
 dma_limit); int iommu_dma_init_fq(struct iommu_domain *domain);

include/linux/iommu.h +1 -0

index 1164524814cb..281a87fdce77 100644
+++ b/include/linux/iommu.h

@@ -105,6 +105,8 @@ struct iommu_domain {
 	enum iommu_page_response_code (*iopf_handler)(struct

include/linux/iommu.h struct iommu_domain { · line 109

Quoted 5 lines

include/linux/iommu.h +1 -0

 iommu_fault *fault, void *data);
 	void *fault_data;
+	ioasid_t pasid;		/* Used for DMA requests with

PASID */

include/linux/iommu.h +1 -0

+	atomic_t pasid_users;  

These are poorly named, this is really the DMA API global PASID and shouldn't be used for other things.

Jason