DMA, powerpc/dma: Fallback to dma_ops when persistent memory present
Subsystems (2)
v2 cover DMA, powerpc/dma: Fallback to dma_ops when persistent memory present
This allows mixing direct DMA (to/from RAM) and IOMMU (to/from apersistent memory) on the PPC64/pseries platform.
This replaces this: https://lkml.org/lkml/2020/10/20/1085
A lesser evil this is :)
This is based on sha14525c8781ec0 Linus Torvalds "scsi: qla2xxx: remove incorrect sparse #ifdef".
Please comment. Thanks.
Alexey Kardashevskiy (2):
dma: Allow mixing bypass and normal IOMMU operation powerpc/dma: Fallback to dma_ops when persistent memory present
arch/powerpc/kernel/dma-iommu.c | 12 ++++- arch/powerpc/platforms/pseries/iommu.c | 44 ++++++++++++++----- kernel/dma/mapping.c | 61 +++++++++++++++++++++++++- arch/powerpc/Kconfig | 1 + kernel/dma/Kconfig | 4 ++ 5 files changed, 108 insertions(+), 14 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/2 dma: Allow mixing bypass and normal IOMMU operation
At the moment we allow bypassing DMA ops only when we can do this for the entire RAM. However there are configs with mixed type memory where we could still allow bypassing IOMMU in most cases; POWERPC with persistent memory is one example.
This adds another check for the bus limit to determine where bypass can still work and we invoke direct DMA API; when DMA handle is outside that limit, we fall back to DMA ops.
This adds a CONFIG_DMA_OPS_BYPASS_BUS_LIMIT config option which is off by default and will be enable for PPC_PSERIES in the following patch.
--- kernel/dma/mapping.c | 61 ++++++++++++++++++++++++++++++++++++++++++-- kernel/dma/Kconfig | 4 +++ 2 files changed, 63 insertions(+), 2 deletions(-)
kernel/dma/mapping.c +59 -2
@@ -137,6 +137,18 @@ static inline bool dma_map_direct(struct device *dev,return dma_go_direct (dev, *dev->dma_mask, ops); } +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT +static inline bool can_map_direct (struct device *dev,phys_addr_t addr) +{ +return dev->bus_dma_limit >=phys_to_dma (dev, addr); +} + +static inline bool dma_handle_direct (struct device *dev,dma_addr_t dma_handle) +{ +return dma_handle >= dev->archdata.dma_offset; +} +#endif +dma_addr_t dma_map_page_attrs (struct device *dev,struct page *page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)
@@ -151,6 +163,11 @@ dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,if (dma_map_direct (dev, ops)) addr =dma_direct_map_page (dev, page, offset, size, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && + can_map_direct(dev, (phys_addr_t) page_to_phys(page) + offset + size)) + addr =dma_direct_map_page (dev, page, offset, size, dir, attrs); +#endif else addr = ops->map_page (dev, page, offset, size, dir, attrs);debug_dma_map_page (dev, page, offset, size, dir, addr);
@@ -167,6 +184,10 @@ void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,BUG_ON (!valid_dma_direction (dir));if (dma_map_direct (dev, ops))dma_direct_unmap_page (dev, addr, size, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && dma_handle_direct(dev, addr + size)) + dma_direct_unmap_page(dev, addr, size, dir, attrs); +#endif else if (ops->unmap_page) ops->unmap_page(dev, addr, size, dir, attrs);debug_dma_unmap_page (dev ,addr ,size ,dir );
@@ -190,6 +211,23 @@ int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,if (dma_map_direct (dev, ops)) ents =dma_direct_map_sg (dev, sg, nents, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) { + direct =can_map_direct (dev,sg_phys (s) + s->offset + s->length); +if (!direct) +break ; + } +if (direct) + ents =dma_direct_map_sg (dev, sg, nents, dir, attrs); +else + ents = ops->map_sg (dev, sg, nents, dir, attrs); + } +#endif else ents = ops->map_sg (dev, sg, nents, dir, attrs);BUG_ON (ents <0 );
@@ -207,9 +245,28 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,BUG_ON (!valid_dma_direction (dir));debug_dma_unmap_sg (dev, sg, nents, dir); -if (dma_map_direct (dev, ops)) +if (dma_map_direct (dev, ops)) {dma_direct_unmap_sg (dev, sg, nents, dir, attrs); -else if (ops->unmap_sg) +return ; + } +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT +if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) { + direct =dma_handle_direct (dev, s->dma_address + s->length); +if (!direct) +break ; + } +if (direct) { +dma_direct_unmap_sg (dev, sg, nents, dir, attrs); +return ; + } + } +#endif +if (ops->unmap_sg) ops->unmap_sg (dev, sg, nents, dir, attrs); }EXPORT_SYMBOL (dma_unmap_sg_attrs);
kernel/dma/Kconfig +4 -0
@@ -20,6 +20,10 @@ config DMA_OPS config DMA_OPS_BYPASS bool +# IOMMU driver limited by a DMA window size may switch between bypass and window +config DMA_OPS_BYPASS_BUS_LIMIT + bool + config NEED_SG_DMA_LENGTH bool
Signature · 1 line
2.17.1
Re: [PATCH kernel v2 1/2] dma: Allow mixing bypass and normal IOMMU operation
kernel/dma/mapping.c static inline bool dma_map_direct(struct device *dev, · line 149
Quoted 4 lines
kernel/dma/mapping.c +4
-0+static inline bool dma_handle_direct (struct device *dev,dma_addr_t dma_handle) +{ +return dma_handle >= dev->archdata.dma_offset; +}
This won't compile except for powerpc, and directly accesing arch members in common code is a bad idea. Maybe both your helpers need to be supplied by arch code to better abstract this out.
kernel/dma/mapping.c dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, · line 170
Quoted 7 lines
kernel/dma/mapping.c +5
-0if (dma_map_direct (dev, ops)) addr =dma_direct_map_page (dev, page, offset, size, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && + can_map_direct(dev, (phys_addr_t) page_to_phys(page) + offset + size)) + addr =dma_direct_map_page (dev, page, offset, size, dir, attrs); +#endif
I don't think page_to_phys needs a phys_addr_t on the return value. I'd also much prefer if we make this a little more beautiful, here are a few suggestions:
- hide the bus_dma_limit check inside can_map_direct, and provide a stub so that we can avoid the ifdef
- use a better name for can_map_direct, and maybe also a better calling convention by passing the page (the sg code also has the page), and maybe even hide the dma_map_direct inside it.
if (dma_map_direct(dev, ops) || arch_dma_map_page_direct(dev, page, offset, size)) addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
kernel/dma/mapping.c void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, · line 190
Quoted 7 lines
kernel/dma/mapping.c +4
-0BUG_ON (!valid_dma_direction (dir));if (dma_map_direct (dev, ops))dma_direct_unmap_page (dev, addr, size, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && dma_handle_direct(dev, addr + size)) + dma_direct_unmap_page(dev, addr, size, dir, attrs); +#endif
Same here.
Quoted 9 lines
kernel/dma/mapping.c +7
-0if (dma_map_direct (dev, ops)) ents =dma_direct_map_sg (dev, sg, nents, dir, attrs); +#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) {
kernel/dma/mapping.c int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, · line 230
Quoted 10 lines
kernel/dma/mapping.c +10
-0+ direct =can_map_direct (dev,sg_phys (s) + s->offset + s->length); +if (!direct) + break; + } +if (direct) + ents =dma_direct_map_sg (dev, sg, nents, dir, attrs); +else + ents = ops->map_sg (dev, sg, nents, dir, attrs); + } +#endif
This needs to go into a helper as well. I think the same style as above would work pretty nicely as well:
if (dma_map_direct(dev, ops) ||
arch_dma_map_sg_direct(dev, sg, nents))
ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
else
ents = ops->map_sg(dev, sg, nents, dir, attrs);
Quoted 7 lines
kernel/dma/mapping.c +7
-0+#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT +if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) {
kernel/dma/mapping.c void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, · line 268
Quoted 10 lines
kernel/dma/mapping.c +10
-0+ direct =dma_handle_direct (dev, s->dma_address + s->length); +if (!direct) + break; + } +if (direct) { +dma_direct_unmap_sg (dev, sg, nents, dir, attrs); +return ; + } + } +#endif
One more time here..
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH kernel v2 1/2] dma: Allow mixing bypass and normal IOMMU operation
Quoted 8 lines · Christoph Hellwig
kernel/dma/mapping.c static inline bool dma_map_direct(struct device *dev, · line 149
Quoted 4 lines
kernel/dma/mapping.c +4
-0+static inline bool dma_handle_direct (struct device *dev,dma_addr_t dma_handle) +{ +return dma_handle >= dev->archdata.dma_offset; +}This won't compile except for powerpc, and directly accesing arch members in common code is a bad idea. Maybe both your helpers need to be supplied by arch code to better abstract this out.
rats, overlooked it :( bus_dma_limit is generic but dma_offset is in archdata :-/
Quoted 13 lines
kernel/dma/mapping.c dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, · line 170
Quoted 8 lines
if (
dma_map_direct(dev, ops))
addr =dma_direct_map_page(dev, page, offset, size, dir, attrs);kernel/dma/mapping.c +5
-0+#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && + can_map_direct(dev, (phys_addr_t) page_to_phys(page) + offset + size)) + addr =dma_direct_map_page (dev, page, offset, size, dir, attrs); +#endif I don't think
page_to_physneeds aphys_addr_ton the return value. I'd also much prefer if we make this a little more beautiful, here are a few suggestions:
Quoted 4 lines
- hide the
bus_dma_limitcheck insidecan_map_direct, and provide a stub so that we can avoid the ifdef
- use a better name forcan_map_direct, and maybe also a better calling convention by passing the page (the sg code also has the page),
It is passing an address of the end of the mapped area so passing a page struct means passing page and offset which is an extra parameter and we do not want to do anything with the page in those hooks anyway so I'd keep it as is.
Quoted 2 lines
and
maybe even hide thedma_map_directinside it.
Call dma_map_direct() from arch_dma_map_page_direct() if arch_dma_map_page_direct() is defined? Seems suboptimal as it is going to be bypass=true in most cases and we save one call by avoiding calling arch_dma_map_page_direct(). Unless I missed something?
Quoted 29 lines
if (dma_map_direct(dev, ops) || arch_dma_map_page_direct(dev, page, offset, size)) addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);kernel/dma/mapping.c void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, · line 190
Quoted 7 lines
BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) dma_direct_unmap_page(dev, addr, size, dir, attrs);kernel/dma/mapping.c +4
-0+#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit && dma_handle_direct(dev, addr + size)) + dma_direct_unmap_page(dev, addr, size, dir, attrs); +#endif Same here.
Quoted 14 lines
if (
dma_map_direct(dev, ops))
ents =dma_direct_map_sg(dev, sg, nents, dir, attrs);kernel/dma/mapping.c +12
-0+#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT + else if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) { + direct =can_map_direct (dev,sg_phys (s) + s->offset + s->length); +if (!direct) +break ; + } +if (direct)
Quoted 8 lines
kernel/dma/mapping.c int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, · line 230
Quoted 5 lines
kernel/dma/mapping.c +5
-0+ ents =dma_direct_map_sg (dev, sg, nents, dir, attrs); + else + ents = ops->map_sg(dev, sg, nents, dir, attrs); + } +#endif This needs to go into a helper as well. I think the same style as above would work pretty nicely as well:
Yup. I'll repost v3 soon with this change. Thanks for the review.
Quoted 27 lines
if (
dma_map_direct(dev, ops)||
arch_dma_map_sg_direct(dev, sg, nents))
ents =dma_direct_map_sg(dev, sg, nents, dir, attrs);
else
ents =ops->map_sg(dev, sg, nents, dir, attrs);Quoted 7 lines
kernel/dma/mapping.c +7
-0+#ifdef CONFIG_DMA_OPS_BYPASS_BUS_LIMIT +if (dev->bus_dma_limit) { +struct scatterlist *s; +bool direct = true; +int i; + +for_each_sg (sg, s, nents, i) {kernel/dma/mapping.c void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, · line 268
Quoted 10 lines
kernel/dma/mapping.c +10
-0+ direct =dma_handle_direct (dev, s->dma_address + s->length); +if (!direct) + break; + } +if (direct) { +dma_direct_unmap_sg (dev, sg, nents, dir, attrs); +return ; + } + } +#endif One more time here..
Signature · 1 line
Alexey
Re: [PATCH kernel v2 1/2] dma: Allow mixing bypass and normal IOMMU operation
Quoted 7 lines · Alexey Kardashevskiy
It is passing an address of the end of the mapped area so passing a page struct means passing page and offset which is an extra parameter and we do not want to do anything with the page in those hooks anyway so I'd keep it as is.
Quoted 7 lines
Quoted 2 lines
and
maybe even hide thedma_map_directinside it.Call
dma_map_direct()fromarch_dma_map_page_direct()ifarch_dma_map_page_direct()is defined? Seems suboptimal as it is going to be bypass=true in most cases and we save one call by avoiding callingarch_dma_map_page_direct(). Unless I missed something?
C does not even evaluate the right hand side of a || expression if the left hand evaluates to true.
_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu
Re: [PATCH kernel v2 1/2] dma: Allow mixing bypass and normal IOMMU operation
Quoted 11 lines · Christoph Hellwig
Quoted 10 lines · Alexey Kardashevskiy
It is passing an address of the end of the mapped area so passing a page struct means passing page and offset which is an extra parameter and we do not want to do anything with the page in those hooks anyway so I'd keep it as is.
Quoted 2 lines
and
maybe even hide thedma_map_directinside it.
Quoted 7 lines
Quoted 4 lines
Call
dma_map_direct()fromarch_dma_map_page_direct()ifarch_dma_map_page_direct()is defined? Seems suboptimal as it is going to be bypass=true in most cases and we save one call by avoiding callingarch_dma_map_page_direct(). Unless I missed something?C does not even evaluate the right hand side of a || expression if the left hand evaluates to true.
Right, this is what I meant. dma_map_direct() is inline and fast so I did not want it inside the arch hook which is not inline.
Signature · 1 line
Alexey
PATCH v2 2/2 powerpc/dma: Fallback to dma_ops when persistent memory present
In reply to Alexey KardashevskiySo far we have been using huge DMA windows to map all the RAM available. The RAM is normally mapped to the VM address space contiguously, and there is always a reasonable upper limit for possible future hot plugged RAM which makes it easy to map all RAM via IOMMU.
Now there is persistent memory ("ibm,pmemory" in the FDT) which (unlike normal RAM) can map anywhere in the VM space beyond the maximum RAM size and since it can be used for DMA, it requires extending the huge window up to MAX_PHYSMEM_BITS which requires hypervisor support for:
1. huge TCE tables;
2. multilevel TCE tables;
3. huge IOMMU pages.
Certain hypervisors cannot do either so the only option left is restricting the huge DMA window to include only RAM and fallback to the default DMA window for persistent memory.
This checks if the system has persistent memory. If it does not, the DMA bypass mode is selected, i.e.
* dev->bus_dma_limit = 0
* dev->dma_ops_bypass = true <- this avoid calling dma_ops for mapping.
If there is such memory, this creates identity mapping only for RAM and sets the dev->bus_dma_limit to let the generic code decide whether to call into the direct DMA or the indirect DMA ops.
This should not change the existing behaviour when no persistent memory as dev->dma_ops_bypass is expected to be set.
--- arch/powerpc/kernel/dma-iommu.c | 12 +++++-- arch/powerpc/platforms/pseries/iommu.c | 44 ++++++++++++++++++++------ arch/powerpc/Kconfig | 1 + 3 files changed, 45 insertions(+), 12 deletions(-)
arch/powerpc/kernel/dma-iommu.c +10 -2
@@ -90,8 +90,16 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask)struct iommu_table *tbl =get_iommu_table_base (dev);if (dev_is_pci (dev) &&dma_iommu_bypass_supported (dev, mask)) { - dev->dma_ops_bypass = true; -dev_dbg (dev,"iommu: 64-bit OK, using fixed ops\n" ); +/* +* dma_iommu_bypass_supported() sets dma_max when there is +* 1:1 mapping but it is somehow limited. +* ibm,pmemory is one example. +*/ + dev->dma_ops_bypass = dev->bus_dma_limit ==0 ; +if (!dev->dma_ops_bypass) +dev_warn (dev,"iommu: 64-bit OK but using default ops\n" ); +else +dev_dbg (dev,"iommu: 64-bit OK, using fixed ops\n" );return 1 ; }
arch/powerpc/platforms/pseries/iommu.c +34 -10
85 lines not on this page · raw message
arch/powerpc/Kconfig +1 -0
@@ -159,6 +159,7 @@ config PPC select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN select DMA_OPS if PPC64 select DMA_OPS_BYPASS if PPC64 + select DMA_OPS_BYPASS_BUS_LIMIT if PPC64 && PPC_PSERIES select DYNAMIC_FTRACE if FUNCTION_TRACER select EDAC_ATOMIC_SCRUB select EDAC_SUPPORT
Signature · 1 line
2.17.1