Skip to contentSkip to conversation

generic DMA bypass flag v2

PATCH In review Christoph Hellwig 5 participants 30 messages Mar 20, 2020 – Apr 14, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

Hi all,

I've recently beeing chatting with Lu about using dma-iommu and per-device DMA ops in the intel IOMMU driver, and one missing feature in dma-iommu is a bypass mode where the direct mapping is used even when an iommu is attached to improve performance. The powerpc code already has a similar mode, so I'd like to move it to the core DMA mapping code. As part of that I noticed that the current powerpc code has a little bug in that it used the wrong check in the dma_sync_* routines to see if the direct mapping code is used.

These two patches just add the generic code and move powerpc over, the intel IOMMU bits will require a separate discussion.

The x86 AMD Gart code also has a bypass mode, but it is a lot strange, so I'm not going to touch it for now.

Changes since v1:
- rebased to the current dma-mapping-for-next tree
_______________________________________________
iommu mailing list [email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

#1 lore raw

PATCH 1/2 dma-mapping: add a dma_ops_bypass flag to struct device

Several IOMMU drivers have a bypass mode where they can use a direct mapping if the devices DMA mask is large enough. Add generic support to the core dma-mapping code to do that to switch those drivers to a common solution.

---
 include/linux/device.h      |  6 ++++++
 include/linux/dma-mapping.h | 30 ++++++++++++++++++------------
 kernel/dma/mapping.c        | 36 +++++++++++++++++++++++++++---------
 3 files changed, 51 insertions(+), 21 deletions(-)
include/linux/device.h +6 -0
index 0cd7c647c16c..09be8bb2c4a6 100644
@@ -525,6 +525,11 @@ struct dev_links_info {
  *		  sync_state() callback.
  * @dma_coherent: this particular device is dma coherent, even if the
  *		architecture supports non-coherent devices.
+ * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
+ *		streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
+ *		and optionall (if the coherent mask is large enough) also
+ *		for dma allocations.  This flag is managed by the dma ops
+ *		instance from ->dma_supported.
  *
  * At the lowest level, every device in a Linux system is represented by an
  * instance of struct device. The device structure contains the information
@@ -625,6 +630,7 @@ struct device {
     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 	bool			dma_coherent:1;
 #endif
+	bool			dma_ops_bypass : 1;
 };
 
 static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-mapping.h +18 -12
index 330ad58fbf4d..c3af0cf5e435 100644
@@ -188,9 +188,15 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
 }
 #endif /* CONFIG_DMA_DECLARE_COHERENT */
 
-static inline bool dma_is_direct(const struct dma_map_ops *ops)
+/*
+ * Check if the devices uses a direct mapping for streaming DMA operations.
+ * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
+ * enough.
+ */
+static inline bool dma_map_direct(struct device *dev,
+		const struct dma_map_ops *ops)
 {
-	return likely(!ops);
+	return likely(!ops) || dev->dma_ops_bypass;
 }
 
 /*
@@ -279,7 +285,7 @@ static inline dma_addr_t dma_map_page_attrs(struct device *dev,
 	dma_addr_t addr;
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
 	else
 		addr = ops->map_page(dev, page, offset, size, dir, attrs);
@@ -294,7 +300,7 @@ static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_unmap_page(dev, addr, size, dir, attrs);
 	else if (ops->unmap_page)
 		ops->unmap_page(dev, addr, size, dir, attrs);
@@ -313,7 +319,7 @@ static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 	int ents;
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
 	else
 		ents = ops->map_sg(dev, sg, nents, dir, attrs);
@@ -331,7 +337,7 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 
 	BUG_ON(!valid_dma_direction(dir));
 	debug_dma_unmap_sg(dev, sg, nents, dir);
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
 	else if (ops->unmap_sg)
 		ops->unmap_sg(dev, sg, nents, dir, attrs);
@@ -352,7 +358,7 @@ static inline dma_addr_t dma_map_resource(struct device *dev,
 	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
 		return DMA_MAPPING_ERROR;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
 	else if (ops->map_resource)
 		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
@@ -368,7 +374,7 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (!dma_is_direct(ops) && ops->unmap_resource)
+	if (!dma_map_direct(dev, ops) && ops->unmap_resource)
 		ops->unmap_resource(dev, addr, size, dir, attrs);
 	debug_dma_unmap_resource(dev, addr, size, dir);
 }
@@ -380,7 +386,7 @@ static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
 	else if (ops->sync_single_for_cpu)
 		ops->sync_single_for_cpu(dev, addr, size, dir);
@@ -394,7 +400,7 @@ static inline void dma_sync_single_for_device(struct device *dev,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_single_for_device(dev, addr, size, dir);
 	else if (ops->sync_single_for_device)
 		ops->sync_single_for_device(dev, addr, size, dir);
@@ -408,7 +414,7 @@ dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
 	else if (ops->sync_sg_for_cpu)
 		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
@@ -422,7 +428,7 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
 	else if (ops->sync_sg_for_device)
 		ops->sync_sg_for_device(dev, sg, nelems, dir);
kernel/dma/mapping.c +27 -9
index 12ff766ec1fa..fdea45574345 100644
@@ -105,6 +105,24 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 }
 EXPORT_SYMBOL(dmam_alloc_attrs);
 
+static bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops)
+{
+	if (!ops)
+		return true;
+
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}
+
+	return false;
+}
+
 /*
  * Create scatter-list for the already allocated DMA buffer.
  */
@@ -138,7 +156,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
 				size, attrs);
 	if (!ops->get_sgtable)
@@ -206,7 +224,7 @@ bool dma_can_mmap(struct device *dev)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_can_mmap(dev);
 	return ops->mmap != NULL;
 }
@@ -231,7 +249,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
 				attrs);
 	if (!ops->mmap)
@@ -244,7 +262,7 @@ u64 dma_get_required_mask(struct device *dev)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		return dma_direct_get_required_mask(dev);
 	if (ops->get_required_mask)
 		return ops->get_required_mask(dev);
@@ -275,7 +293,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 	/* let the implementation decide on the zone to allocate from: */
 	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
 	else if (ops->alloc)
 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
@@ -307,7 +325,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
 		return;
 
 	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
 	else if (ops->free)
 		ops->free(dev, size, cpu_addr, dma_handle, attrs);
@@ -318,7 +336,7 @@ int dma_supported(struct device *dev, u64 mask)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (!ops)
 		return dma_direct_supported(dev, mask);
 	if (!ops->dma_supported)
 		return 1;
@@ -374,7 +392,7 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 
 	BUG_ON(!valid_dma_direction(dir));
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		arch_dma_cache_sync(dev, vaddr, size, dir);
 	else if (ops->cache_sync)
 		ops->cache_sync(dev, vaddr, size, dir);
@@ -386,7 +404,7 @@ size_t dma_max_mapping_size(struct device *dev)
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	size_t size = SIZE_MAX;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		size = dma_direct_max_mapping_size(dev);
 	else if (ops && ops->max_mapping_size)
 		size = ops->max_mapping_size(dev);
Signature · 6 lines
2.25.1

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
maintainer Reviewed-by on v1 1/2
#2 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 11 lines · Christoph Hellwig

Several IOMMU drivers have a bypass mode where they can use a direct mapping if the devices DMA mask is large enough. Add generic support to the core dma-mapping code to do that to switch those drivers to a common solution.

---
 include/linux/device.h      |  6 ++++++
 include/linux/dma-mapping.h | 30 ++++++++++++++++++------------
 kernel/dma/mapping.c        | 36 +++++++++++++++++++++++++++---------
 3 files changed, 51 insertions(+), 21 deletions(-)
#3 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

In reply to Christoph Hellwig
Quoted 160 lines · Christoph Hellwig

Several IOMMU drivers have a bypass mode where they can use a direct mapping if the devices DMA mask is large enough. Add generic support to the core dma-mapping code to do that to switch those drivers to a common solution.

---
 include/linux/device.h      |  6 ++++++
 include/linux/dma-mapping.h | 30 ++++++++++++++++++------------
 kernel/dma/mapping.c        | 36 +++++++++++++++++++++++++++---------
 3 files changed, 51 insertions(+), 21 deletions(-)
include/linux/device.h +6 -0
index 0cd7c647c16c..09be8bb2c4a6 100644
@@ -525,6 +525,11 @@ struct dev_links_info {
  *		  sync_state() callback.
  * @dma_coherent: this particular device is dma coherent, even if the
  *		architecture supports non-coherent devices.
+ * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
+ *		streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
+ *		and optionall (if the coherent mask is large enough) also
+ *		for dma allocations.  This flag is managed by the dma ops
+ *		instance from ->dma_supported.
  *
  * At the lowest level, every device in a Linux system is represented by an
  * instance of struct device. The device structure contains the information
@@ -625,6 +630,7 @@ struct device {
     defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
 	bool			dma_coherent:1;
 #endif
+	bool			dma_ops_bypass : 1;
 };
 
 static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-mapping.h +18 -12
index 330ad58fbf4d..c3af0cf5e435 100644
@@ -188,9 +188,15 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
 }
 #endif /* CONFIG_DMA_DECLARE_COHERENT */
 
-static inline bool dma_is_direct(const struct dma_map_ops *ops)
+/*
+ * Check if the devices uses a direct mapping for streaming DMA operations.
+ * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
+ * enough.
+ */
+static inline bool dma_map_direct(struct device *dev,
+		const struct dma_map_ops *ops)
 {
-	return likely(!ops);
+	return likely(!ops) || dev->dma_ops_bypass;
 }
 
 /*
@@ -279,7 +285,7 @@ static inline dma_addr_t dma_map_page_attrs(struct device *dev,
 	dma_addr_t addr;
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
 	else
 		addr = ops->map_page(dev, page, offset, size, dir, attrs);
@@ -294,7 +300,7 @@ static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_unmap_page(dev, addr, size, dir, attrs);
 	else if (ops->unmap_page)
 		ops->unmap_page(dev, addr, size, dir, attrs);
@@ -313,7 +319,7 @@ static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 	int ents;
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
 	else
 		ents = ops->map_sg(dev, sg, nents, dir, attrs);
@@ -331,7 +337,7 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 
 	BUG_ON(!valid_dma_direction(dir));
 	debug_dma_unmap_sg(dev, sg, nents, dir);
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
 	else if (ops->unmap_sg)
 		ops->unmap_sg(dev, sg, nents, dir, attrs);
@@ -352,7 +358,7 @@ static inline dma_addr_t dma_map_resource(struct device *dev,
 	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
 		return DMA_MAPPING_ERROR;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
 	else if (ops->map_resource)
 		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
@@ -368,7 +374,7 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (!dma_is_direct(ops) && ops->unmap_resource)
+	if (!dma_map_direct(dev, ops) && ops->unmap_resource)
 		ops->unmap_resource(dev, addr, size, dir, attrs);
 	debug_dma_unmap_resource(dev, addr, size, dir);
 }
@@ -380,7 +386,7 @@ static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
 	else if (ops->sync_single_for_cpu)
 		ops->sync_single_for_cpu(dev, addr, size, dir);
@@ -394,7 +400,7 @@ static inline void dma_sync_single_for_device(struct device *dev,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_single_for_device(dev, addr, size, dir);
 	else if (ops->sync_single_for_device)
 		ops->sync_single_for_device(dev, addr, size, dir);
@@ -408,7 +414,7 @@ dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
 	else if (ops->sync_sg_for_cpu)
 		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
@@ -422,7 +428,7 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
 	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
 	else if (ops->sync_sg_for_device)
 		ops->sync_sg_for_device(dev, sg, nelems, dir);
kernel/dma/mapping.c +4 -0
index 12ff766ec1fa..fdea45574345 100644
@@ -105,6 +105,24 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 }
 EXPORT_SYMBOL(dmam_alloc_attrs);
 
+static bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops)
+{
+	if (!ops)
+		return true;

kernel/dma/mapping.c void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · line 121

Quoted 10 lines
kernel/dma/mapping.c +10 -0
+
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}

Why not do this in dma_map_direct() as well?
Or simply have just one dma_map_direct()?

And one more general question - we need a way to use non-direct IOMMU for RAM above certain limit.

Let's say we have a system with:
0 .. 0x1.0000.0000
0x100.0000.0000 .. 0x101.0000.0000

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

Quoted 89 lines
kernel/dma/mapping.c +13 -9
+
+	return false;
+}
+
 /*
  * Create scatter-list for the already allocated DMA buffer.
  */
@@ -138,7 +156,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
 				size, attrs);
 	if (!ops->get_sgtable)
@@ -206,7 +224,7 @@ bool dma_can_mmap(struct device *dev)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_can_mmap(dev);
 	return ops->mmap != NULL;
 }
@@ -231,7 +249,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
 				attrs);
 	if (!ops->mmap)
@@ -244,7 +262,7 @@ u64 dma_get_required_mask(struct device *dev)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		return dma_direct_get_required_mask(dev);
 	if (ops->get_required_mask)
 		return ops->get_required_mask(dev);
@@ -275,7 +293,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 	/* let the implementation decide on the zone to allocate from: */
 	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
 	else if (ops->alloc)
 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
@@ -307,7 +325,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
 		return;
 
 	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
 	else if (ops->free)
 		ops->free(dev, size, cpu_addr, dma_handle, attrs);
@@ -318,7 +336,7 @@ int dma_supported(struct device *dev, u64 mask)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (!ops)
 		return dma_direct_supported(dev, mask);
 	if (!ops->dma_supported)
 		return 1;
@@ -374,7 +392,7 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 
 	BUG_ON(!valid_dma_direction(dir));
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
 		arch_dma_cache_sync(dev, vaddr, size, dir);
 	else if (ops->cache_sync)
 		ops->cache_sync(dev, vaddr, size, dir);
@@ -386,7 +404,7 @@ size_t dma_max_mapping_size(struct device *dev)
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	size_t size = SIZE_MAX;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
 		size = dma_direct_max_mapping_size(dev);
 	else if (ops && ops->max_mapping_size)
 		size = ops->max_mapping_size(dev);
Signature · 1 line
Alexey
#4 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

On Mon, Mar 23, 2020 at 12:28:34PM +1100, Alexey Kardashevskiy wrote:

[full quote deleted, please follow proper quoting rules]

Quoted 9 lines
Quoted 9 lines
kernel/dma/mapping.c +9 -0
+static bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops)
+{
+	if (!ops)
+		return true;
+
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
Quoted 8 lines

kernel/dma/mapping.c void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · line 121

Quoted 5 lines
kernel/dma/mapping.c +5 -0
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}

Why not do this in dma_map_direct() as well?

Mostly beacuse it is a relatively expensive operation, including a fls64.

Quoted 1 line

Or simply have just one dma_map_direct()?

What do you mean with that?

Quoted 7 lines

And one more general question - we need a way to use non-direct IOMMU for RAM above certain limit.

Let's say we have a system with:
0 .. 0x1.0000.0000
0x100.0000.0000 .. 0x101.0000.0000

Quoted 8 lines

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

bus_dma_limit is just another limiting factor applied on top of coherent_dma_mask or dma_mask respectively.

#5 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 13 lines · Christoph Hellwig
Quoted 4 lines
Quoted 4 lines
kernel/dma/mapping.c +4 -0
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
Quoted 8 lines

kernel/dma/mapping.c void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · line 121

Quoted 5 lines
kernel/dma/mapping.c +5 -0
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}

Why not do this in dma_map_direct() as well?

Quoted 2 lines

Mostly beacuse it is a relatively expensive operation, including a fls64.

Which I guess isn't too bad compared to a dynamic IOMMU mapping. Can you just send a draft patch for what you'd like to see for ppc?

#6 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 14 lines · Christoph Hellwig
Quoted 13 lines · Christoph Hellwig
Quoted 12 lines

Review

Quoted 9 lines
(quoted diff) +9 -0
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}

Why not do this in dma_map_direct() as well?

Quoted 5 lines
Quoted 2 lines

Mostly beacuse it is a relatively expensive operation, including a fls64.

Which I guess isn't too bad compared to a dynamic IOMMU mapping. Can you just send a draft patch for what you'd like to see for ppc?

This is what I was trying, but considering I am new to DMA subsystem, I am not sure I got all the details correct. The idea is to look at the cpu addr and see if that can be used in direct map fashion(is bus_dma_limit the right restriction here?) if not fallback to dynamic IOMMU mapping.

arch/powerpc/kernel/dma-iommu.c +85 -4
index e486d1d78de2..bc7e6a8b2caa 100644
@@ -31,6 +31,87 @@ static inline bool dma_iommu_map_bypass(struct device *dev,
 		(!iommu_fixed_is_weak || (attrs & DMA_ATTR_WEAK_ORDERING));
 }
 
+static inline bool __dma_direct_map_capable(struct device *dev, struct page *page,
+					    unsigned long offset, size_t size)
+{
+	phys_addr_t phys = page_to_phys(page) + offset;
+	dma_addr_t dma_addr = phys_to_dma(dev, phys);
+	dma_addr_t end = dma_addr + size - 1;
+
+	return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
+}
+
+static inline bool dma_direct_map_capable(struct device *dev, struct page *page,
+					  unsigned long offset, size_t size,
+					  unsigned long attrs)
+{
+	if (!dma_iommu_map_bypass(dev, attrs))
+		return false;
+
+	if (!dev->dma_mask)
+		return false;
+
+	return __dma_direct_map_capable(dev, page, offset, size);
+}
+
+
+static inline bool dma_direct_unmap_capable(struct device *dev, dma_addr_t addr, size_t size,
+					    unsigned long attrs)
+{
+	dma_addr_t end = addr + size - 1;
+
+	if (!dma_iommu_map_bypass(dev, attrs))
+		return false;
+
+	if (!dev->dma_mask)
+		return false;
+
+	return end <= min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
+}
+
+static inline bool dma_direct_sg_map_capable(struct device *dev, struct scatterlist *sglist,
+					     int nelems, unsigned long attrs)
+{
+	int i;
+	struct scatterlist *sg;
+
+	if (!dma_iommu_map_bypass(dev, attrs))
+		return false;
+
+	if (!dev->dma_mask)
+		return false;
+
+	for_each_sg(sglist, sg, nelems, i) {
+		if (!__dma_direct_map_capable(dev, sg_page(sg),
+					      sg->offset, sg->length))
+			return false;
+	}
+	return true;
+}
+
+static inline bool dma_direct_sg_unmap_capable(struct device *dev, struct scatterlist *sglist,
+					       int nelems, unsigned long attrs)
+{
+	int i;
+	dma_addr_t end;
+	struct scatterlist *sg;
+
+	if (!dma_iommu_map_bypass(dev, attrs))
+		return false;
+
+	if (!dev->dma_mask)
+		return false;
+
+	for_each_sg(sglist, sg, nelems, i) {
+		end = sg->dma_address + sg_dma_len(sg);
+
+		if (end > min_not_zero(*dev->dma_mask, dev->bus_dma_limit))
+			return false;
+	}
+	return true;
+}
+
+
 /* Allocates a contiguous real buffer and creates mappings over it.
  * Returns the virtual address of the buffer and sets dma_handle
  * to the dma address (mapping) of the first page.
@@ -67,7 +148,7 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page,
 				     enum dma_data_direction direction,
 				     unsigned long attrs)
 {
-	if (dma_iommu_map_bypass(dev, attrs))
+	if (dma_direct_map_capable(dev, page, offset, size, attrs))
 		return dma_direct_map_page(dev, page, offset, size, direction,
 				attrs);
 	return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
@@ -79,7 +160,7 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle,
 				 size_t size, enum dma_data_direction direction,
 				 unsigned long attrs)
 {
-	if (!dma_iommu_map_bypass(dev, attrs))
+	if (!dma_direct_unmap_capable(dev, dma_handle, size, attrs))
 		iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size,
 				direction,  attrs);
 	else
@@ -91,7 +172,7 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
 			    int nelems, enum dma_data_direction direction,
 			    unsigned long attrs)
 {
-	if (dma_iommu_map_bypass(dev, attrs))
+	if (dma_direct_sg_map_capable(dev, sglist, nelems, attrs))
 		return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
 	return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
 				dma_get_mask(dev), direction, attrs);
@@ -101,7 +182,7 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist,
 		int nelems, enum dma_data_direction direction,
 		unsigned long attrs)
 {
-	if (!dma_iommu_map_bypass(dev, attrs))
+	if (!dma_direct_sg_unmap_capable(dev, sglist, nelems, attrs))
 		ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
 			   direction, attrs);
 	else
arch/powerpc/platforms/pseries/iommu.c +1 -0
index 99f72162dd85..702a680f5766 100644
@@ -1119,6 +1119,7 @@ static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
 	spin_unlock(&direct_window_list_lock);
 
 	dma_addr = be64_to_cpu(ddwprop->dma_base);
+	dev->dev.bus_dma_limit = dma_addr + query.largest_available_block;
 	goto out_unlock;
 
 out_free_window:
#7 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 6 lines · Aneesh Kumar K.V

This is what I was trying, but considering I am new to DMA subsystem, I am not sure I got all the details correct. The idea is to look at the cpu addr and see if that can be used in direct map fashion(is bus_dma_limit the right restriction here?) if not fallback to dynamic IOMMU mapping.

I don't think we can throw all these complications into the dma mapping code. At some point I also wonder what the point is, especially for scatterlist mappings, where the iommu can coalesce.

#8 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 11 lines · Christoph Hellwig
Quoted 6 lines · Aneesh Kumar K.V

This is what I was trying, but considering I am new to DMA subsystem, I am not sure I got all the details correct. The idea is to look at the cpu addr and see if that can be used in direct map fashion(is bus_dma_limit the right restriction here?) if not fallback to dynamic IOMMU mapping.

I don't think we can throw all these complications into the dma mapping code. At some point I also wonder what the point is, especially for scatterlist mappings, where the iommu can coalesce.

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

Signature · 1 line
Alexey
#9 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 13 lines · Alexey Kardashevskiy
Quoted 11 lines · Christoph Hellwig
Quoted 6 lines · Aneesh Kumar K.V

This is what I was trying, but considering I am new to DMA subsystem, I am not sure I got all the details correct. The idea is to look at the cpu addr and see if that can be used in direct map fashion(is bus_dma_limit the right restriction here?) if not fallback to dynamic IOMMU mapping.

I don't think we can throw all these complications into the dma mapping code. At some point I also wonder what the point is, especially for scatterlist mappings, where the iommu can coalesce.

Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

This is O_DIRECT with a user buffer that is actually mmap from a dax mounted file system.

What we really need is something that will falback to iommu_map_page based on dma_addr. ie. Something equivalent to current dma_direct_map_page(), but instead of fallback to swiotlb_map page we should fallback to iommu_map_page().

Something like?

dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs)

{
	phys_addr_t phys = page_to_phys(page) + offset;
	dma_addr_t dma_addr = phys_to_dma(dev, phys);

	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
			return iommu_map(dev, phys, size, dir, attrs);

		return DMA_MAPPING_ERROR;
	}

....
...

-aneesh
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

#10 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 11 lines · Aneesh Kumar K.V

dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs)

{
	phys_addr_t phys = page_to_phys(page) + offset;
	dma_addr_t dma_addr = phys_to_dma(dev, phys);

	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
			return iommu_map(dev, phys, size, dir, attrs);

		return DMA_MAPPING_ERROR;

If powerpc hardware / firmware people really come up with crap that stupid you'll have to handle it yourself and will always pay the indirect call penality.

#11 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

In reply to Alexey Kardashevskiy
Quoted 5 lines · Alexey Kardashevskiy

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

#12 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 10 lines · Christoph Hellwig
Quoted 5 lines · Alexey Kardashevskiy

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

Signature · 1 line
Alexey
#13 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 11 lines · Alexey Kardashevskiy
Quoted 9 lines
Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

Quoted 4 lines

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

How can it be mapped too high for a direct mapping with a 64-bit DMA mask?

#14 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 12 lines · Christoph Hellwig
Quoted 11 lines · Alexey Kardashevskiy
Quoted 9 lines
Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

Quoted 7 lines
Quoted 4 lines

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

How can it be mapped too high for a direct mapping with a 64-bit DMA mask?

The window size is limited and often it is not even sparse. It requires an 8 byte entry per an IOMMU page (which is most commonly is 64k max) so 1TB limit (a guest RAM size) is a quite real thing. MMIO is mapped to guest physical address space outside of this 1TB (on PPC).

Signature · 1 line
Alexey
#15 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 20 lines · Alexey Kardashevskiy
Quoted 17 lines · Christoph Hellwig
Quoted 15 lines · Alexey Kardashevskiy
Quoted 9 lines
Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

Quoted 9 lines
Quoted 2 lines

How can it be mapped too high for a direct mapping with a 64-bit DMA mask?

The window size is limited and often it is not even sparse. It requires an 8 byte entry per an IOMMU page (which is most commonly is 64k max) so 1TB limit (a guest RAM size) is a quite real thing. MMIO is mapped to guest physical address space outside of this 1TB (on PPC).

I am trying now this approach on top of yours "dma-bypass.3" (it is "wip", needs an upper boundary check):

https://github.com/aik/linux/commit/49d73c7771e3f6054804f6cfa80b4e320111662d

Do you see any serious problem with this approach? Thanks!

Signature · 1 line
Alexey
#16 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 33 lines · Alexey Kardashevskiy
Quoted 20 lines · Alexey Kardashevskiy
Quoted 17 lines · Christoph Hellwig
Quoted 15 lines · Alexey Kardashevskiy
Quoted 9 lines
Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

Quoted 9 lines
Quoted 2 lines

How can it be mapped too high for a direct mapping with a 64-bit DMA mask?

The window size is limited and often it is not even sparse. It requires an 8 byte entry per an IOMMU page (which is most commonly is 64k max) so 1TB limit (a guest RAM size) is a quite real thing. MMIO is mapped to guest physical address space outside of this 1TB (on PPC).

Quoted 6 lines

I am trying now this approach on top of yours "dma-bypass.3" (it is "wip", needs an upper boundary check):

https://github.com/aik/linux/commit/49d73c7771e3f6054804f6cfa80b4e320111662d

Do you see any serious problem with this approach? Thanks!

Do you have a link to the whole branch? The github UI is unfortunately unusable for that (or I'm missing something).

#17 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 37 lines · Christoph Hellwig
Quoted 36 lines · Alexey Kardashevskiy
Quoted 20 lines · Alexey Kardashevskiy
Quoted 17 lines · Christoph Hellwig
Quoted 15 lines · Alexey Kardashevskiy
Quoted 9 lines
Quoted 5 lines

This is for persistent memory which you can DMA to/from but yet it does not appear in the system as a normal memory and therefore requires special handling anyway (O_DIRECT or DAX, I do not know the exact mechanics). All other devices in the system should just run as usual, i.e. use 1:1 mapping if possible.

On other systems (x86 and arm) pmem as long as it is page backed does not require any special handling. This must be some weird way powerpc fucked up again, and I suspect you'll have to suffer from it.

It does not matter if it is backed by pages or not, the problem may also appear if we wanted for example p2p PCI via IOMMU (between PHBs) and MMIO might be mapped way too high in the system address space and make 1:1 impossible.

Quoted 9 lines
Quoted 2 lines

How can it be mapped too high for a direct mapping with a 64-bit DMA mask?

The window size is limited and often it is not even sparse. It requires an 8 byte entry per an IOMMU page (which is most commonly is 64k max) so 1TB limit (a guest RAM size) is a quite real thing. MMIO is mapped to guest physical address space outside of this 1TB (on PPC).

I am trying now this approach on top of yours "dma-bypass.3" (it is "wip", needs an upper boundary check):

Quoted 6 lines
Quoted 3 lines

https://github.com/aik/linux/commit/49d73c7771e3f6054804f6cfa80b4e320111662d

Do you see any serious problem with this approach? Thanks!

Do you have a link to the whole branch? The github UI is unfortunately unusable for that (or I'm missing something).

The UI shows the branch but since I rebased and forcepushed it, it does not. Here is the current one with:

https://github.com/aik/linux/commits/dma-bypass.3

Thanks,

Signature · 1 line
Alexey
#18 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 9 lines · Alexey Kardashevskiy
Quoted 4 lines
Quoted 1 line

Do you see any serious problem with this approach? Thanks!

Do you have a link to the whole branch? The github UI is unfortunately unusable for that (or I'm missing something).

The UI shows the branch but since I rebased and forcepushed it, it does not. Here is the current one with:

https://github.com/aik/linux/commits/dma-bypass.3

Ok, so we use the core bypass without persistent memory, and then have another bypass mode on top. Not great, but I can't think of anything better. Note that your checks for the map_sg case aren't very efficient - for one it would make sense to calculate the limit only once, but also it would make sense to reuse the calculted diecect mapping addresses instead of doing another pass later on in the dma-direct code.

#19 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 9 lines · Christoph Hellwig
Quoted 8 lines · Alexey Kardashevskiy
Quoted 4 lines
Quoted 1 line

Do you see any serious problem with this approach? Thanks!

Do you have a link to the whole branch? The github UI is unfortunately unusable for that (or I'm missing something).

The UI shows the branch but since I rebased and forcepushed it, it does not. Here is the current one with:

Quoted 7 lines
Quoted 1 line

https://github.com/aik/linux/commits/dma-bypass.3

Ok, so we use the core bypass without persistent memory, and then have another bypass mode on top. Not great, but I can't think of anything better. Note that your checks for the map_sg case aren't very efficient - for one it would make sense to calculate the limit only once,

Good points, I'll post revised version when you post your v3 of this.

Quoted 3 lines

but also it would make sense to reuse the
calculted diecect mapping addresses instead of doing another pass later on in the dma-direct code.

Probably but I wonder what kind of hardware we need to see the difference. I might try, just need to ride to the office to plug the cable in my 100GBit eth machines :) Thanks,

Signature · 5 lines
Alexey
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
#20 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 14 lines · Alexey Kardashevskiy
Quoted 11 lines · Christoph Hellwig
Quoted 9 lines · Alexey Kardashevskiy
Quoted 4 lines
Quoted 1 line

Do you see any serious problem with this approach? Thanks!

Do you have a link to the whole branch? The github UI is unfortunately unusable for that (or I'm missing something).

The UI shows the branch but since I rebased and forcepushed it, it does not. Here is the current one with:

https://github.com/aik/linux/commits/dma-bypass.3

Quoted 7 lines
Quoted 5 lines

Ok, so we use the core bypass without persistent memory, and then have another bypass mode on top. Not great, but I can't think of anything better. Note that your checks for the map_sg case aren't very efficient - for one it would make sense to calculate the limit only once,

Good points, I'll post revised version when you post your v3 of this.

Any plans on posting v3 of this? Thanks,

Quoted 10 lines
Quoted 4 lines

but also it would make sense to reuse the
calculted diecect mapping addresses instead of doing another pass later on in the dma-direct code.

Probably but I wonder what kind of hardware we need to see the difference. I might try, just need to ride to the office to plug the cable in my 100GBit eth machines :) Thanks,

Signature · 1 line
Alexey
#21 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 5 lines · Alexey Kardashevskiy
Quoted 1 line

Good points, I'll post revised version when you post your v3 of this.

Any plans on posting v3 of this? Thanks,

Just back from a long weekend. I'll take a stab at it soon.

#22 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

In reply to Christoph Hellwig
Quoted 22 lines · Christoph Hellwig

On Mon, Mar 23, 2020 at 12:28:34PM +1100, Alexey Kardashevskiy wrote:

[full quote deleted, please follow proper quoting rules]

Quoted 9 lines
Quoted 9 lines
kernel/dma/mapping.c +9 -0
+static bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops)
+{
+	if (!ops)
+		return true;
+
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
Quoted 8 lines

kernel/dma/mapping.c void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, · line 121

Quoted 5 lines
kernel/dma/mapping.c +5 -0
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}

Why not do this in dma_map_direct() as well?

Quoted 2 lines

Mostly beacuse it is a relatively expensive operation, including a fls64.

Ah, ok.

Quoted 3 lines
Quoted 1 line

Or simply have just one dma_map_direct()?

What do you mean with that?

I mean use dma_alloc_direct() instead of dma_map_direct() everywhere, you explained just above.

Quoted 12 lines
Quoted 12 lines

And one more general question - we need a way to use non-direct IOMMU for RAM above certain limit.

Let's say we have a system with:
0 .. 0x1.0000.0000
0x100.0000.0000 .. 0x101.0000.0000

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

Quoted 7 lines
Quoted 4 lines

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

bus_dma_limit is just another limiting factor applied on top of coherent_dma_mask or dma_mask respectively.

This is not enough for the task: in my example, I'd set bus limit to 0x0800.0001.0000.0000 but this would disable bypass for all RAM addresses - the first and the second 4GB blocks.

Signature · 1 line
Alexey
#23 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 11 lines · Alexey Kardashevskiy
Quoted 11 lines
Quoted 10 lines

0x100.0000.0000 .. 0x101.0000.0000

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

Quoted 6 lines
Quoted 2 lines

bus_dma_limit is just another limiting factor applied on top of coherent_dma_mask or dma_mask respectively.

This is not enough for the task: in my example, I'd set bus limit to 0x0800.0001.0000.0000 but this would disable bypass for all RAM addresses - the first and the second 4GB blocks.

So what about something like the version here:

http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/dma-bypass.3

#24 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 15 lines · Christoph Hellwig
Quoted 14 lines · Alexey Kardashevskiy
Quoted 13 lines
Quoted 10 lines

0x100.0000.0000 .. 0x101.0000.0000

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

bus_dma_limit is just another limiting factor applied on top of coherent_dma_mask or dma_mask respectively.

Quoted 7 lines
Quoted 3 lines

This is not enough for the task: in my example, I'd set bus limit to 0x0800.0001.0000.0000 but this would disable bypass for all RAM addresses - the first and the second 4GB blocks.

So what about something like the version here:

http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/dma-bypass.3

dma_alloc_direct() and dma_map_direct() do the same thing now which is good, did I miss anything else?

This lets us disable bypass automatically if this weird memory appears in the system but does not let us have 1:1 after that even for normal RAM. Thanks,

Signature · 1 line
Alexey
#25 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 27 lines · Alexey Kardashevskiy
Quoted 15 lines · Christoph Hellwig
Quoted 14 lines · Alexey Kardashevskiy
Quoted 13 lines
Quoted 10 lines

0x100.0000.0000 .. 0x101.0000.0000

2x4G, each is 1TB aligned. And we can map directly only the first 4GB (because of the maximum IOMMU table size) but not the other. And 1:1 on that "pseries" is done with offset=0x0800.0000.0000.0000.

So we want to check every bus address against dev->bus_dma_limit, not dev->coherent_dma_mask. In the example above I'd set bus_dma_limit to 0x0800.0001.0000.0000 and 1:1 mapping for the second 4GB would not be tried. Does this sound reasonable? Thanks,

bus_dma_limit is just another limiting factor applied on top of coherent_dma_mask or dma_mask respectively.

Quoted 7 lines
Quoted 3 lines

This is not enough for the task: in my example, I'd set bus limit to 0x0800.0001.0000.0000 but this would disable bypass for all RAM addresses - the first and the second 4GB blocks.

So what about something like the version here:

http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/dma-bypass.3

Quoted 6 lines

dma_alloc_direct() and dma_map_direct() do the same thing now which is good, did I miss anything else?

This lets us disable bypass automatically if this weird memory appears in the system but does not let us have 1:1 after that even for normal RAM. Thanks,

Ah no, does not help much, simple setting dma_ops_bypass will though.

But eventually, in this function:

static inline bool dma_map_direct(struct device *dev,
const struct dma_map_ops *ops)

{
       if (likely(!ops))
               return true;
       if (!dev->dma_ops_bypass)
               return false;

       return min_not_zero(*dev->dma_mask, dev->bus_dma_limit) >=
                           dma_direct_get_required_mask(dev);
}

we rather want it to take a dma handle and a size, and add

if (dev->bus_dma_limit)

	return dev->bus_dma_limit > dma_handle + size;

where dma_handle=phys_to_dma(dev, phys) (I am not doing it here as unmap needs the same test and it does not receive phys as a parameter).

Signature · 1 line
Alexey
#26 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

In reply to Alexey Kardashevskiy
Quoted 2 lines · Alexey Kardashevskiy

dma_alloc_direct() and dma_map_direct() do the same thing now which is good, did I miss anything else?

dma_alloc_direct looks at coherent_dma_mask, dma_map_direct looks at dma_mask.

reviewer
#27 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

In reply to Christoph Hellwig
Quoted 4 lines · Christoph Hellwig

Several IOMMU drivers have a bypass mode where they can use a direct mapping if the devices DMA mask is large enough. Add generic support to the core dma-mapping code to do that to switch those drivers to a common solution.

Hmm, this is _almost_, but not quite the same as the case where drivers are managing their own IOMMU mappings, but still need to use streaming DMA for cache maintenance on the underlying pages. For that we need the ops bypass to be a "true" bypass and also avoid SWIOTLB regardless of the device's DMA mask. That behaviour should in fact be fine for the opportunistic bypass case here as well, since the mask being "big enough" implies by definition that this should never need to bounce either.

For the (hopefully less common) third case where, due to groups or user overrides, we end up giving an identity DMA domain to a device with limited DMA masks which _does_ need SWIOTLB, I'd like to think we can solve that by not giving the device IOMMU DMA ops in the first place, such that it never needs to engage the bypass mechanism at all.

Thoughts?

Robin.

Quoted 254 lines
---
  include/linux/device.h      |  6 ++++++
  include/linux/dma-mapping.h | 30 ++++++++++++++++++------------
  kernel/dma/mapping.c        | 36 +++++++++++++++++++++++++++---------
  3 files changed, 51 insertions(+), 21 deletions(-)
include/linux/device.h +6 -0
index 0cd7c647c16c..09be8bb2c4a6 100644
@@ -525,6 +525,11 @@ struct dev_links_info {
   *		  sync_state() callback.
   * @dma_coherent: this particular device is dma coherent, even if the
   *		architecture supports non-coherent devices.
+ * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
+ *		streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
+ *		and optionall (if the coherent mask is large enough) also
+ *		for dma allocations.  This flag is managed by the dma ops
+ *		instance from ->dma_supported.
   *
   * At the lowest level, every device in a Linux system is represented by an
   * instance of struct device. The device structure contains the information
@@ -625,6 +630,7 @@ struct device {
      defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
  	bool			dma_coherent:1;
  #endif
+	bool			dma_ops_bypass : 1;
  };
 
  static inline struct device *kobj_to_dev(struct kobject *kobj)
include/linux/dma-mapping.h +18 -12
index 330ad58fbf4d..c3af0cf5e435 100644
@@ -188,9 +188,15 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma,
  }
  #endif /* CONFIG_DMA_DECLARE_COHERENT */
 
-static inline bool dma_is_direct(const struct dma_map_ops *ops)
+/*
+ * Check if the devices uses a direct mapping for streaming DMA operations.
+ * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
+ * enough.
+ */
+static inline bool dma_map_direct(struct device *dev,
+		const struct dma_map_ops *ops)
  {
-	return likely(!ops);
+	return likely(!ops) || dev->dma_ops_bypass;
  }
 
  /*
@@ -279,7 +285,7 @@ static inline dma_addr_t dma_map_page_attrs(struct device *dev,
  	dma_addr_t addr;
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
  	else
  		addr = ops->map_page(dev, page, offset, size, dir, attrs);
@@ -294,7 +300,7 @@ static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_unmap_page(dev, addr, size, dir, attrs);
  	else if (ops->unmap_page)
  		ops->unmap_page(dev, addr, size, dir, attrs);
@@ -313,7 +319,7 @@ static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
  	int ents;
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
  	else
  		ents = ops->map_sg(dev, sg, nents, dir, attrs);
@@ -331,7 +337,7 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg
 
  	BUG_ON(!valid_dma_direction(dir));
  	debug_dma_unmap_sg(dev, sg, nents, dir);
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
  	else if (ops->unmap_sg)
  		ops->unmap_sg(dev, sg, nents, dir, attrs);
@@ -352,7 +358,7 @@ static inline dma_addr_t dma_map_resource(struct device *dev,
  	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
  		return DMA_MAPPING_ERROR;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
  	else if (ops->map_resource)
  		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
@@ -368,7 +374,7 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (!dma_is_direct(ops) && ops->unmap_resource)
+	if (!dma_map_direct(dev, ops) && ops->unmap_resource)
  		ops->unmap_resource(dev, addr, size, dir, attrs);
  	debug_dma_unmap_resource(dev, addr, size, dir);
  }
@@ -380,7 +386,7 @@ static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
  	else if (ops->sync_single_for_cpu)
  		ops->sync_single_for_cpu(dev, addr, size, dir);
@@ -394,7 +400,7 @@ static inline void dma_sync_single_for_device(struct device *dev,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_sync_single_for_device(dev, addr, size, dir);
  	else if (ops->sync_single_for_device)
  		ops->sync_single_for_device(dev, addr, size, dir);
@@ -408,7 +414,7 @@ dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
  	else if (ops->sync_sg_for_cpu)
  		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
@@ -422,7 +428,7 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
  	BUG_ON(!valid_dma_direction(dir));
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
  	else if (ops->sync_sg_for_device)
  		ops->sync_sg_for_device(dev, sg, nelems, dir);
kernel/dma/mapping.c +27 -9
index 12ff766ec1fa..fdea45574345 100644
@@ -105,6 +105,24 @@ void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
  }
  EXPORT_SYMBOL(dmam_alloc_attrs);
 
+static bool dma_alloc_direct(struct device *dev, const struct dma_map_ops *ops)
+{
+	if (!ops)
+		return true;
+
+	/*
+	 * Allows IOMMU drivers to bypass dynamic translations if the DMA mask
+	 * is large enough.
+	 */
+	if (dev->dma_ops_bypass) {
+		if (min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit) >=
+				dma_direct_get_required_mask(dev))
+			return true;
+	}
+
+	return false;
+}
+
  /*
   * Create scatter-list for the already allocated DMA buffer.
   */
@@ -138,7 +156,7 @@ int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
  				size, attrs);
  	if (!ops->get_sgtable)
@@ -206,7 +224,7 @@ bool dma_can_mmap(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		return dma_direct_can_mmap(dev);
  	return ops->mmap != NULL;
  }
@@ -231,7 +249,7 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
  				attrs);
  	if (!ops->mmap)
@@ -244,7 +262,7 @@ u64 dma_get_required_mask(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		return dma_direct_get_required_mask(dev);
  	if (ops->get_required_mask)
  		return ops->get_required_mask(dev);
@@ -275,7 +293,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
  	/* let the implementation decide on the zone to allocate from: */
  	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
  	else if (ops->alloc)
  		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
@@ -307,7 +325,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
  		return;
 
  	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
  	else if (ops->free)
  		ops->free(dev, size, cpu_addr, dma_handle, attrs);
@@ -318,7 +336,7 @@ int dma_supported(struct device *dev, u64 mask)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
 
-	if (dma_is_direct(ops))
+	if (!ops)
  		return dma_direct_supported(dev, mask);
  	if (!ops->dma_supported)
  		return 1;
@@ -374,7 +392,7 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 
  	BUG_ON(!valid_dma_direction(dir));
 
-	if (dma_is_direct(ops))
+	if (dma_alloc_direct(dev, ops))
  		arch_dma_cache_sync(dev, vaddr, size, dir);
  	else if (ops->cache_sync)
  		ops->cache_sync(dev, vaddr, size, dir);
@@ -386,7 +404,7 @@ size_t dma_max_mapping_size(struct device *dev)
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	size_t size = SIZE_MAX;
 
-	if (dma_is_direct(ops))
+	if (dma_map_direct(dev, ops))
  		size = dma_direct_max_mapping_size(dev);
  	else if (ops && ops->max_mapping_size)
  		size = ops->max_mapping_size(dev);
#28 lore raw

Re: [PATCH 1/2] dma-mapping: add a dma_ops_bypass flag to struct device

Quoted 9 lines · Robin Murphy
Quoted 4 lines · Christoph Hellwig

Several IOMMU drivers have a bypass mode where they can use a direct mapping if the devices DMA mask is large enough. Add generic support to the core dma-mapping code to do that to switch those drivers to a common solution.

Hmm, this is _almost_, but not quite the same as the case where drivers are managing their own IOMMU mappings, but still need to use streaming DMA for cache maintenance on the underlying pages.

In that case they should simply not call the DMA API at all. We'll just need versions of the cache maintainance APIs that tie in with the raw IOMMU API.

Quoted 5 lines

For that we need the ops bypass
to be a "true" bypass and also avoid SWIOTLB regardless of the device's DMA mask. That behaviour should in fact be fine for the opportunistic bypass case here as well, since the mask being "big enough" implies by definition that this should never need to bounce either.

In practice it does. But that means adding yet another code path vs the simple direct call to dma_direct_* vs calling the DMA ops which I'd rather avoid.

#29 lore raw

PATCH 2/2 powerpc: use the generic dma_ops_bypass mode

In reply to Christoph Hellwig

Use the DMA API bypass mechanism for direct window mappings. This uses common code and speed up the direct mapping case by avoiding indirect calls just when not using dma ops at all. It also fixes a problem where the sync_* methods were using the bypass check for DMA allocations, but those are part of the streaming ops.

Note that this patch loses the DMA_ATTR_WEAK_ORDERING override, which has never been well defined, as is only used by a few drivers, which IIRC never showed up in the typical Cell blade setups that are affected by the ordering workaround.

Fixes efd176a04bef ("powerpc/pseries/dma: Allow SWIOTLB")
---
 arch/powerpc/include/asm/device.h |  5 --
 arch/powerpc/kernel/dma-iommu.c   | 90 ++++---------------------------
 2 files changed, 9 insertions(+), 86 deletions(-)
arch/powerpc/include/asm/device.h +0 -5
index 266542769e4b..452402215e12 100644
@@ -18,11 +18,6 @@ struct iommu_table;
  * drivers/macintosh/macio_asic.c
  */
 struct dev_archdata {
-	/*
-	 * Set to %true if the dma_iommu_ops are requested to use a direct
-	 * window instead of dynamically mapping memory.
-	 */
-	bool			iommu_bypass : 1;
 	/*
 	 * These two used to be a union. However, with the hybrid ops we need
 	 * both so here we store both a DMA offset for direct mappings and
arch/powerpc/kernel/dma-iommu.c +9 -81

169 lines not on this page · raw message

Signature · 6 lines
2.25.1

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu