extend support for CMA CMA on x86
Subsystems (5)
cover extend support for CMA CMA on x86
This patch set extends support for the DMA Contiguous Memory Allocator on x86. Currently it is only supported on pci-nommu. So this aims at enabling it on swiotlb and intel-iommu, too.
Regardless of which dma mapping implementation is actually used in the system, I would like to allocate big contiguous memory with dma_alloc_coherent() and tell the base address to the device that requires it. This is why I need this change.
Akinobu Mita (2):
x86: enable DMA CMA with swiotlb
intel-iommu: integrate DMA CMA
arch/x86/Kconfig | 2 +- arch/x86/include/asm/swiotlb.h | 7 +++++++ arch/x86/kernel/amd_gart_64.c | 2 +- arch/x86/kernel/pci-swiotlb.c | 9 ++++++--- arch/x86/pci/sta2x11-fixup.c | 6 ++---- drivers/iommu/intel-iommu.c | 32 ++++++++++++++++++++++++-------- include/linux/swiotlb.h | 2 ++ lib/swiotlb.c | 2 +- 8 files changed, 44 insertions(+), 18 deletions(-)
Signature · 1 line
1.8.3.2
PATCH 1/2 x86: enable DMA CMA with swiotlb
The DMA Contiguous Memory Allocator support on x86 is disabled when swiotlb config option is enabled. So DMA CMA is always disabled on x86_64 because swiotlb is always enabled. This attempts to support for DMA CMA with enabling swiotlb config option.
The contiguous memory allocator on x86 is integrated in the function dma_generic_alloc_coherent() which is .alloc callback in nommu_dma_ops for dma_alloc_coherent().
x86_swiotlb_alloc_coherent() which is .alloc callback in swiotlb_dma_ops tries to allocate with dma_generic_alloc_coherent() firstly and then swiotlb_alloc_coherent() is called as a fallback.
The main part of supporting DMA CMA with swiotlb is that changing x86_swiotlb_free_coherent() which is .free callback in swiotlb_dma_ops for dma_free_coherent() so that it can distinguish memory allocated by dma_generic_alloc_coherent() from one allocated by swiotlb_alloc_coherent() and release it with dma_generic_free_coherent() which can handle contiguous memory. This change requires making is_swiotlb_buffer() global function.
This also needs to change .free callback in the dma_map_ops for amd_gart and sta2x11, because these dma_ops are also using dma_generic_alloc_coherent().
---
No change from the previous version
arch/x86/Kconfig | 2 +- arch/x86/include/asm/swiotlb.h | 7 +++++++ arch/x86/kernel/amd_gart_64.c | 2 +- arch/x86/kernel/pci-swiotlb.c | 9 ++++++--- arch/x86/pci/sta2x11-fixup.c | 6 ++---- include/linux/swiotlb.h | 2 ++ lib/swiotlb.c | 2 +- 7 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/Kconfig +1 -1
@@ -39,7 +39,7 @@ config X86 select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_WANT_FRAME_POINTERS select HAVE_DMA_ATTRS - select HAVE_DMA_CONTIGUOUS if !SWIOTLB + select HAVE_DMA_CONTIGUOUS select HAVE_KRETPROBES select HAVE_OPTPROBES select HAVE_KPROBES_ON_FTRACE
arch/x86/include/asm/swiotlb.h +7 -0
@@ -29,4 +29,11 @@ static inline void pci_swiotlb_late_init(void)static inline void dma_mark_clean (void *addr,size_t size) {} +extern void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, +dma_addr_t *dma_handle,gfp_t flags, +struct dma_attrs *attrs); +extern void x86_swiotlb_free_coherent (struct device *dev,size_t size, +void *vaddr,dma_addr_t dma_addr, +struct dma_attrs *attrs); +#endif /* _ASM_X86_SWIOTLB_H */
arch/x86/kernel/amd_gart_64.c +1 -1
@@ -512,7 +512,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, struct dma_attrs *attrs) {gart_unmap_page (dev ,dma_addr ,size ,DMA_BIDIRECTIONAL ,NULL ); -free_pages ((unsigned long )vaddr,get_order (size)); +dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); }static int gart_mapping_error (struct device *dev,dma_addr_t dma_addr)
arch/x86/kernel/pci-swiotlb.c +6 -3
@@ -14,7 +14,7 @@#include <asm/iommu_table.h> int swiotlb__read_mostly ; -static void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, +void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size,dma_addr_t *dma_handle,gfp_t flags,struct dma_attrs *attrs) {
@@ -28,11 +28,14 @@ static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,return swiotlb_alloc_coherent (hwdev, size, dma_handle, flags); } -static void x86_swiotlb_free_coherent (struct device *dev,size_t size, +void x86_swiotlb_free_coherent (struct device *dev,size_t size,void *vaddr,dma_addr_t dma_addr,struct dma_attrs *attrs) { -swiotlb_free_coherent (dev, size, vaddr, dma_addr); +if (is_swiotlb_buffer (dma_to_phys (dev, dma_addr))) +swiotlb_free_coherent (dev, size, vaddr, dma_addr); +else +dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); }static struct dma_map_ops swiotlb_dma_ops = {
arch/x86/pci/sta2x11-fixup.c +2 -4
@@ -173,9 +173,7 @@ static void *sta2x11_swiotlb_alloc_coherent(struct device *dev, {void *vaddr; - vaddr =dma_generic_alloc_coherent (dev, size, dma_handle, flags, attrs); -if (!vaddr) - vaddr =swiotlb_alloc_coherent (dev, size, dma_handle, flags); + vaddr =x86_swiotlb_alloc_coherent (dev, size, dma_handle, flags, attrs); *dma_handle =p2a (*dma_handle,to_pci_dev (dev));return vaddr; }
@@ -183,7 +181,7 @@ static void *sta2x11_swiotlb_alloc_coherent(struct device *dev,/* We have our own dma_ops: the same as swiotlb but from alloc (above) */ static struct dma_map_ops sta2x11_dma_ops = { .alloc = sta2x11_swiotlb_alloc_coherent, - .free = swiotlb_free_coherent, + .free = x86_swiotlb_free_coherent, .map_page = swiotlb_map_page, .unmap_page = swiotlb_unmap_page, .map_sg = swiotlb_map_sg_attrs,
include/linux/swiotlb.h +2 -0
@@ -116,4 +116,6 @@ static inline void swiotlb_free(void) { }#endif extern void swiotlb_print_info (void ); +extern int is_swiotlb_buffer (phys_addr_t paddr); +#endif /* __LINUX_SWIOTLB_H */
lib/swiotlb.c +1 -1
@@ -369,7 +369,7 @@ void __init swiotlb_free(void) io_tlb_nslabs =0 ; } -static int is_swiotlb_buffer (phys_addr_t paddr) +int is_swiotlb_buffer (phys_addr_t paddr) {return paddr >= io_tlb_start && paddr < io_tlb_end; }
Signature · 1 line
1.8.3.2
Re: [PATCH 1/2] x86: enable DMA CMA with swiotlb
Quoted 3 lines
This also needs to change .free callback in the
dma_map_opsforamd_gartand sta2x11, because thesedma_opsare also usingdma_generic_alloc_coherent().
Looks good to me from a quick read.
-Andi
Re: [PATCH 1/2] x86: enable DMA CMA with swiotlb
In reply to Akinobu MitaQuoted 20 lines · Akinobu Mita
The DMA Contiguous Memory Allocator support on x86 is disabled when swiotlb config option is enabled. So DMA CMA is always disabled on
x86_64because swiotlb is always enabled. This attempts to support for DMA CMA with enabling swiotlb config option.The contiguous memory allocator on x86 is integrated in the function
dma_generic_alloc_coherent()which is .alloc callback innommu_dma_opsfordma_alloc_coherent().
x86_swiotlb_alloc_coherent()which is .alloc callback inswiotlb_dma_opstries to allocate withdma_generic_alloc_coherent()firstly and thenswiotlb_alloc_coherent()is called as a fallback.The main part of supporting DMA CMA with swiotlb is that changing
x86_swiotlb_free_coherent()which is .free callback inswiotlb_dma_opsfordma_free_coherent()so that it can distinguish memory allocated bydma_generic_alloc_coherent()from one allocated byswiotlb_alloc_coherent()and release it withdma_generic_free_coherent()which can handle contiguous memory. This change requires makingis_swiotlb_buffer()global function.
Quoted 4 lines
This also needs to change .free callback in the
dma_map_opsforamd_gartand sta2x11, because thesedma_opsare also usingdma_generic_alloc_coherent().
Before I queue it up and start testing it - Peter were your concerns answered?
Thanks!
Quoted 144 lines
Cc Marek Szyprowski <[email protected]>Cc Konrad Rzeszutek Wilk <[email protected]>Cc Thomas Gleixner <[email protected]>Cc Ingo Molnar <[email protected]>Cc "H. Peter Anvin" <[email protected]>Cc Andi Kleen <[email protected]>Signed-off-by Akinobu Mita <[email protected]>Acked-by Marek Szyprowski <[email protected]>---
No change from the previous versionarch/x86/Kconfig | 2 +- arch/x86/include/asm/swiotlb.h | 7 +++++++ arch/x86/kernel/amd_gart_64.c | 2 +- arch/x86/kernel/pci-swiotlb.c | 9 ++++++--- arch/x86/pci/sta2x11-fixup.c | 6 ++---- include/linux/swiotlb.h | 2 ++ lib/swiotlb.c | 2 +- 7 files changed, 20 insertions(+), 10 deletions(-)arch/x86/Kconfig +1
-1@@ -39,7 +39,7 @@ config X86 select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_WANT_FRAME_POINTERS select HAVE_DMA_ATTRS - select HAVE_DMA_CONTIGUOUS if !SWIOTLB + select HAVE_DMA_CONTIGUOUS select HAVE_KRETPROBES select HAVE_OPTPROBES select HAVE_KPROBES_ON_FTRACEarch/x86/include/asm/swiotlb.h +7
-0@@ -29,4 +29,11 @@ static inline void pci_swiotlb_late_init(void)static inline void dma_mark_clean (void *addr,size_t size) {} +extern void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, +dma_addr_t *dma_handle,gfp_t flags, +struct dma_attrs *attrs); +extern void x86_swiotlb_free_coherent (struct device *dev,size_t size, +void *vaddr,dma_addr_t dma_addr, +struct dma_attrs *attrs); +#endif /* _ASM_X86_SWIOTLB_H */ arch/x86/kernel/amd_gart_64.c +1
-1@@ -512,7 +512,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, struct dma_attrs *attrs) {gart_unmap_page (dev ,dma_addr ,size ,DMA_BIDIRECTIONAL ,NULL ); -free_pages ((unsigned long )vaddr,get_order (size)); +dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); }static int gart_mapping_error (struct device *dev,dma_addr_t dma_addr)arch/x86/kernel/pci-swiotlb.c +6
-3@@ -14,7 +14,7 @@#include <asm/iommu_table.h> int swiotlb__read_mostly ; -static void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size, +void *x86_swiotlb_alloc_coherent (struct device *hwdev,size_t size,dma_addr_t *dma_handle,gfp_t flags,struct dma_attrs *attrs) {@@ -28,11 +28,14 @@ static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,return swiotlb_alloc_coherent (hwdev, size, dma_handle, flags); } -static void x86_swiotlb_free_coherent (struct device *dev,size_t size, +void x86_swiotlb_free_coherent (struct device *dev,size_t size,void *vaddr,dma_addr_t dma_addr,struct dma_attrs *attrs) { -swiotlb_free_coherent (dev, size, vaddr, dma_addr); +if (is_swiotlb_buffer (dma_to_phys (dev, dma_addr))) +swiotlb_free_coherent (dev, size, vaddr, dma_addr); +else +dma_generic_free_coherent (dev, size, vaddr, dma_addr, attrs); }static struct dma_map_ops swiotlb_dma_ops = {arch/x86/pci/sta2x11-fixup.c +2
-4@@ -173,9 +173,7 @@ static void *sta2x11_swiotlb_alloc_coherent(struct device *dev, {void *vaddr; - vaddr =dma_generic_alloc_coherent (dev, size, dma_handle, flags, attrs); -if (!vaddr) - vaddr =swiotlb_alloc_coherent (dev, size, dma_handle, flags); + vaddr =x86_swiotlb_alloc_coherent (dev, size, dma_handle, flags, attrs); *dma_handle =p2a (*dma_handle,to_pci_dev (dev));return vaddr; }@@ -183,7 +181,7 @@ static void *sta2x11_swiotlb_alloc_coherent(struct device *dev,/* We have our own dma_ops: the same as swiotlb but from alloc (above) */ static struct dma_map_ops sta2x11_dma_ops = { .alloc = sta2x11_swiotlb_alloc_coherent, - .free = swiotlb_free_coherent, + .free = x86_swiotlb_free_coherent, .map_page = swiotlb_map_page, .unmap_page = swiotlb_unmap_page, .map_sg = swiotlb_map_sg_attrs,include/linux/swiotlb.h +2
-0@@ -116,4 +116,6 @@ static inline void swiotlb_free(void) { }#endif extern void swiotlb_print_info (void ); +extern int is_swiotlb_buffer (phys_addr_t paddr); +#endif /* __LINUX_SWIOTLB_H */ lib/swiotlb.c +1
-1@@ -369,7 +369,7 @@ void __init swiotlb_free(void) io_tlb_nslabs =0 ; } -static int is_swiotlb_buffer (phys_addr_t paddr) +int is_swiotlb_buffer (phys_addr_t paddr) {return paddr >= io_tlb_start && paddr < io_tlb_end; }Signature · 1 line
1.8.3.2
PATCH 2/2 intel-iommu: integrate DMA CMA
In reply to Akinobu MitaThis adds support for the DMA Contiguous Memory Allocator for intel-iommu. This change enables dma_alloc_coherent() to allocate big contiguous memory.
It is achieved in the same way as nommu_dma_ops currently does, i.e. trying to allocate memory by dma_alloc_from_contiguous() and alloc_pages() is used as a fallback.
--- drivers/iommu/intel-iommu.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-)
drivers/iommu/intel-iommu.c +24 -8
@@ -3004,7 +3004,7 @@ static void *intel_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, struct dma_attrs *attrs) { - void *vaddr; + struct page *page = NULL;int order; size =PAGE_ALIGN (size);
@@ -3019,17 +3019,31 @@ static void *intel_alloc_coherent(struct device *hwdev, size_t size, flags |=GFP_DMA32 ; } - vaddr = (void *)__get_free_pages (flags, order); -if (!vaddr) +if (!(flags &GFP_ATOMIC )) { +unsigned int count = size >>PAGE_SHIFT ; + + page =dma_alloc_from_contiguous (hwdev, count, order); +if (page &&iommu_no_mapping (hwdev) && +page_to_phys (page) + size > hwdev->coherent_dma_mask) { +dma_release_from_contiguous (hwdev, page, count); + page =NULL ; + } + } + +if (!page) + page =alloc_pages (flags, order); +if (!page)return NULL ; -memset (vaddr,0 , size); +memset (page_address (page),0 , size); - *dma_handle =__intel_map_single (hwdev,virt_to_bus (vaddr), size, + *dma_handle =__intel_map_single (hwdev,page_to_phys (page), size,DMA_BIDIRECTIONAL , hwdev->coherent_dma_mask);if (*dma_handle) -return vaddr; -free_pages ((unsigned long )vaddr, order); +return page_address (page); +if (!dma_release_from_contiguous (hwdev, page, size >>PAGE_SHIFT )) +__free_pages (page, order); +return NULL ; }
@@ -3037,12 +3051,14 @@ static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { int order; +struct page *page =virt_to_page (vaddr); size =PAGE_ALIGN (size); order =get_order (size);intel_unmap_page (hwdev, dma_handle, size,DMA_BIDIRECTIONAL ,NULL ); -free_pages ((unsigned long )vaddr, order); +if (!dma_release_from_contiguous (hwdev, page, size >>PAGE_SHIFT )) +__free_pages (page, order); } static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
Signature · 1 line
1.8.3.2
Quoted 8 lines · Akinobu Mita
This patch set extends support for the DMA Contiguous Memory Allocator on x86. Currently it is only supported on pci-nommu. So this aims at enabling it on swiotlb and intel-iommu, too.
Regardless of which dma mapping implementation is actually used in the system, I would like to allocate big contiguous memory with
dma_alloc_coherent()and tell the base address to the device that requires it. This is why I need this change.
I think it's generally the right direction. CMA is much better than the DMA zone (e.g. it can actually (mostly) share memory, while lower zone protection keeps the DMA zone free) Also it works for more than 16MB / 64MB.
So i hope longer term we can just kill GFP_DMA and replace it with some CMA variant everywhere.
There are also other potential users that could make use of it, but they need to interoperate with swiotlb, so something liike this is needed.
-Andi