provide in-place uncached remapping for dma-direct
Hi all,
this series provides support for remapping places uncached in-place in the generic dma-direct code, and moves openrisc over from its own in-place remapping scheme. The arm64 folks also had interest in such a scheme to avoid problems with speculating into cache aliases.
Also all architectures that always use small page mappings for the kernel and have non-coherent DMA should look into enabling this scheme, as it is much more efficient than the vmap remapping.
PATCH 1/2 dma-mapping: support setting memory uncached in place
We currently only support remapping memory as uncached through vmap or a magic uncached segment provided by some architectures. But there is a simpler and much better way available on some architectures where we can just remap the memory in place. The advantages are:
1) no aliasing is possible, which prevents speculating into the cached alias
2) there is no need to allocate new ptes and thus no need for a special pre-allocated pool of memory that can be used with GFP_ATOMIC DMA allocations
The downside is that architectures must provide a way to set arbitrary pages uncached in the kernel mapping, which might not be possible on architecture that have a special implicit kernel mapping, and requires splitting of huge page kernel mappings where they exist.
--- include/linux/dma-noncoherent.h | 3 +++ kernel/dma/Kconfig | 8 ++++++++ kernel/dma/direct.c | 28 ++++++++++++++++++---------- 3 files changed, 29 insertions(+), 10 deletions(-)
include/linux/dma-noncoherent.h +3 -0
@@ -111,4 +111,7 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size)void *uncached_kernel_address (void *addr);void *cached_kernel_address (void *addr); +int arch_dma_set_uncached (void *cpu_addr,size_t size); +void arch_dma_clear_uncached (void *cpu_addr,size_t size); +#endif /* _LINUX_DMA_NONCOHERENT_H */
kernel/dma/Kconfig +8 -0
@@ -83,6 +83,14 @@ config DMA_DIRECT_REMAP bool select DMA_REMAP +# +# Should be selected if the architecture can remap memory from the page +# allocator and CMA as uncached and provides the arch_dma_set_uncached and +# arch_dma_clear_uncached helpers +# +config ARCH_HAS_DMA_SET_UNCACHED + bool + config DMA_CMA bool "DMA Contiguous Memory Allocator" depends on HAVE_DMA_CONTIGUOUS && CMA
kernel/dma/direct.c +18 -10
@@ -171,11 +171,8 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size, ret =dma_common_contiguous_remap (page,PAGE_ALIGN (size),dma_pgprot (dev,PAGE_KERNEL , attrs),__builtin_return_address (0 )); -if (!ret) { -dma_free_contiguous (dev, page, size); -return ret; - } - +if (!ret) + goto out_free_pages;memset (ret,0 , size); goto done; }
@@ -188,8 +185,7 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,* so log an error and fail. */ dev_info (dev,"Rejecting highmem page from CMA.\n" ); -dma_free_contiguous (dev, page, size); -return NULL ; + goto out_free_pages; } ret =page_address (page);
@@ -198,10 +194,15 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,memset (ret,0 , size); -if (IS_ENABLED (CONFIG_ARCH_HAS_UNCACHED_SEGMENT ) && -dma_alloc_need_uncached (dev, attrs)) { +if (dma_alloc_need_uncached (dev, attrs)) {arch_dma_prep_coherent (page, size); - ret =uncached_kernel_address (ret); + +if (IS_ENABLED (CONFIG_ARCH_HAS_DMA_SET_UNCACHED )) { +if (!arch_dma_set_uncached (ret, size)) + goto out_free_pages; + }else if (IS_ENABLED (CONFIG_ARCH_HAS_UNCACHED_SEGMENT )) { + ret =uncached_kernel_address (ret); + } } done:if (force_dma_unencrypted (dev))
@@ -209,6 +210,9 @@ void *dma_direct_alloc_pages(struct device *dev, size_t size,else *dma_handle =phys_to_dma (dev,page_to_phys (page));return ret; +out_free_pages: +dma_free_contiguous (dev, page, size); +return NULL ; } void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,
@@ -232,6 +236,8 @@ void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr,if (IS_ENABLED (CONFIG_DMA_REMAP ) &&is_vmalloc_addr (cpu_addr))vunmap (cpu_addr); +else if (IS_ENABLED (CONFIG_ARCH_HAS_DMA_SET_UNCACHED )) +arch_dma_clear_uncached (cpu_addr, size);dma_free_contiguous (dev,dma_direct_to_page (dev, dma_addr), size); }
@@ -240,6 +246,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) && !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && dma_alloc_need_uncached(dev, attrs)) return arch_dma_alloc(dev, size, dma_handle, gfp, attrs);
@@ -250,6 +257,7 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { if (!IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) && + !IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) && !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && dma_alloc_need_uncached(dev, attrs)) arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);
Signature · 1 line
2.20.1
PATCH 2/2 openrisc: use the generic in-place uncached DMA allocator
In reply to Christoph HellwigSwitch openrisc to use the dma-direct allocator and just provide the hooks for setting memory uncached or cached.
--- arch/openrisc/Kconfig | 1 + arch/openrisc/kernel/dma.c | 51 +++++--------------------------------- 2 files changed, 7 insertions(+), 45 deletions(-)
arch/openrisc/Kconfig +1 -0
@@ -7,6 +7,7 @@ config OPENRISC def_bool y select ARCH_32BIT_OFF_T + select ARCH_HAS_DMA_SET_UNCACHED select ARCH_HAS_SYNC_DMA_FOR_DEVICE select OF select OF_EARLY_FLATTREE
arch/openrisc/kernel/dma.c +6 -45
76 lines not on this page · raw message
Signature · 1 line
2.20.1