DMA pagecache
cover DMA pagecache
This RFC improves the performance of indirect mapping on all tested DMA usages, based on a mlx5 device, ranging from 64k packages to 1-byte packages, from 1 thread to 64 threads.
In all workloads tested, the performance of indirect mapping gets very near to direct mapping case.
The whole thing is designed to have as much perfomance as possible, so the impact of the pagecache is not too big.
As I am not very experienced in XArrays usage, nor in lockless algorithms, I would specially appreaciate feedback on possible failures on it's usage, missing barriers, and so on.
Also, this size for the FIFO is just for testing purposes. It's also very possible that it will not be a good idea in platforms other than pseries, (i have not tested them).
I can plan I bypass for those cases without much work.
Thank you!
Leonardo Bras (2):
dma-direction: Add DMA_DIR_COMPAT() macro to test direction
compability
powerpc/kernel/iommu: Introduce IOMMU DMA pagecache
arch/powerpc/include/asm/iommu-cache.h | 31 ++++ arch/powerpc/include/asm/iommu.h | 4 + arch/powerpc/kernel/Makefile | 2 +- arch/powerpc/kernel/iommu-cache.c | 247 +++++++++++++++++++++++++ arch/powerpc/kernel/iommu.c | 15 +- include/linux/dma-direction.h | 3 + 6 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 arch/powerpc/include/asm/iommu-cache.h create mode 100644 arch/powerpc/kernel/iommu-cache.c
Signature · 1 line
2.25.4
RFC PATCH 1/2 dma-direction: Add DMA_DIR_COMPAT() macro to test direction compability
Given a existing mapping with 'current' direction, and a 'wanted' direction for using that mapping, check if 'wanted' is satisfied by 'current'.
current acceptsDMA_BIDIRECTIONAL DMA_BIDIRECTIONAL, DMA_TO_DEVICE,
DMA_FROM_DEVICE, DMA_NONEDMA_TO_DEVICE DMA_TO_DEVICE, DMA_NONEDMA_FROM_DEVICE DMA_FROM_DEVICE, DMA_NONEDMA_NONE DMA_NONE
This macro is useful for checking if a DMA mapping can be reused.
--- include/linux/dma-direction.h | 3 +++ 1 file changed, 3 insertions(+)
include/linux/dma-direction.h +3 -0
@@ -9,4 +9,7 @@ enum dma_data_direction { DMA_NONE = 3, }; +/* Checks if wanted direction is satisfied by current mapping direction*/ +#define DMA_DIR_COMPAT (current, wanted) (((current) & ~(wanted)) == 0) +#endif
Signature · 1 line
2.25.4
RFC PATCH 2/2 powerpc/kernel/iommu: Introduce IOMMU DMA pagecache
In reply to Leonardo BrasIn pseries, mapping a DMA page for a cpu memory range requires a H_PUT_TCE* hypercall, and unmapping requires a H_STUFF_TCE hypercall. When doing a lot of I/O, a thread can spend a lot of time doing such hcalls, specially when a DMA mapping don't get reused.
The purpose of this change is to introduce a mechanism similar to a pagecache, but for reusing DMA mappings, improving performance and avoiding multiple DMA mappings of the same cpupage.
This works based in a few current behaviors:
- Page reuse:
It's common for userspace process to reuse the same page for several allocations This is probably caused by page buffering behavior that come from libc, so getting pages from the kernel is less expensive.
- A lot of small allocations are used:
Some workloads do a lot of allocations that do not fill a whole page causing multiple DMA mappings for a single page.
How it work:
- When a DMA mapping is required for given allocation, it first searches the DMA pagecache for a matching mapping. When found, increment refcount, when not found, a new mapping is created.
- Every time a new mapping is created, it's added to the DMA pagecache, with refcount = 1;
- When the mapping is not needed anymore (iommu_free()), it looks in the DMA pagecache for the entry, and the decrements it's refcount.
- When there are more than IOMMU_MAP_LIST_MAX entries in the DMA pagecache, it removes the older ones.
What is inside:
- 1 XArray which indexes the DMA page addresses, used for removing mappings and decreasing refcounts.
- 1 XArray which indexes CPU page addresses, for finding matching mappings.
- As there can be multiple mappings (directions) for the same cpupage, this one keeps a llist for looking into the entries.
- Every entry points (page) in the XArray points to the mapping struct.
- The mapping struct have:
- DMA & CPU page addresses, size (pages) , direction and refcount.
- 1 llist used for multiple mappings at the same cpupage
- 1 llist used for the FIFO (removing old unused entries)
- The cache struct, added to iommu_table, have:
- Both XArrays,
- 2 llist for entry/removal point on FIFO,
- 1 Atomic Cachesize, to manage the FIFO.
--- arch/powerpc/include/asm/iommu-cache.h | 31 ++++ arch/powerpc/include/asm/iommu.h | 4 + arch/powerpc/kernel/Makefile | 2 +- arch/powerpc/kernel/iommu-cache.c | 247 +++++++++++++++++++++++++ arch/powerpc/kernel/iommu.c | 15 +- 5 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 arch/powerpc/include/asm/iommu-cache.h create mode 100644 arch/powerpc/kernel/iommu-cache.c
arch/powerpc/include/asm/iommu-cache.h new file +31 -0
@@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _IOMMU_CACHE_H +#define _IOMMU_CACHE_H +#ifdef __KERNEL__ + +#include <linux/llist.h> +#include <linux/xarray.h> +#include <linux/atomic.h> + +struct dmacache { +struct llist_head fifo_add; +struct llist_head fifo_del; +struct xarray cpupages; +struct xarray dmapages; +atomic64_t cachesize; +}; + +#include <asm/iommu.h> + +void iommu_cache_init (struct iommu_table *tbl); +void iommu_dmacache_add (struct iommu_table *tbl,void *page,unsigned int npages,dma_addr_t addr, +enum dma_data_direction direction); +dma_addr_t iommu_dmacache_use (struct iommu_table *tbl,void *page,unsigned int npages, +enum dma_data_direction direction); +void iommu_dmacache_free (struct iommu_table *tbl,dma_addr_t dma_handle,unsigned int npages); + +#define IOMMU_MAP_LIST_MAX 8192 +#define IOMMU_MAP_LIST_THRES 128 + +#endif /* __KERNEL__ */ +#endif /* _IOMMU_CACHE_H */
arch/powerpc/include/asm/iommu.h +4 -0
@@ -18,6 +18,7 @@#include <asm/types.h> #include <asm/pci-bridge.h> #include <asm/asm-const.h> +#include <asm/iommu-cache.h> #define IOMMU_PAGE_SHIFT_4K 12#define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K)
@@ -114,6 +115,7 @@ struct iommu_table {int it_nid;unsigned long it_reserved_start;/* Start of not-DMA-able (MMIO) area */ unsigned long it_reserved_end; +struct dmacache cache; };#define IOMMU_TABLE_USERSPACE_ENTRY_RO (tbl, entry) \
@@ -317,6 +319,8 @@ extern void iommu_release_ownership(struct iommu_table *tbl);extern enum dma_data_direction iommu_tce_direction (unsigned long tce);extern unsigned long iommu_direction_to_tce_perm (enum dma_data_direction dir); +void __iommu_free (struct iommu_table *tbl,dma_addr_t dma_addr,unsigned int npages); +#ifdef CONFIG_PPC_CELL_NATIVE extern bool iommu_fixed_is_weak;#else
arch/powerpc/kernel/Makefile +1 -1
@@ -104,7 +104,7 @@ extra-y += vmlinux.lds obj-$(CONFIG_RELOCATABLE) += reloc_$(BITS).o obj-$(CONFIG_PPC32) += entry_32.o setup_32.o early_32.o -obj-$(CONFIG_PPC64) += dma-iommu.o iommu.o +obj-$(CONFIG_PPC64) += dma-iommu.o iommu.o iommu-cache.o obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_BOOTX_TEXT) += btext.o obj-$(CONFIG_SMP) += smp.o
arch/powerpc/kernel/iommu-cache.c new file +247 -0
247 lines not on this page · raw message
arch/powerpc/kernel/iommu.c +10 -5
@@ -40,8 +40,6 @@static int novmerge; -static void __iommu_free (struct iommu_table *,dma_addr_t ,unsigned int ); -static int __init setup_iommu (char *str) {if (!strcmp (str,"novmerge" ))
@@ -308,6 +306,10 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,dma_addr_t ret =DMA_MAPPING_ERROR ;int build_fail; + ret =iommu_dmacache_use (tbl, page, npages, direction); +if (ret !=DMA_MAPPING_ERROR ) +return ret; + entry =iommu_range_alloc (dev, tbl, npages,NULL , mask, align_order);if (unlikely (entry ==DMA_MAPPING_ERROR ))
@@ -338,6 +340,8 @@ static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,/* Make sure updates are seen by hardware */ mb (); +iommu_dmacache_add (tbl, page, npages, ret, direction); +return ret; }
@@ -388,8 +392,7 @@ static struct iommu_pool *get_pool(struct iommu_table *tbl,return p; } -static void __iommu_free (struct iommu_table *tbl,dma_addr_t dma_addr, -unsigned int npages) +void __iommu_free (struct iommu_table *tbl,dma_addr_t dma_addr,unsigned int npages) {unsigned long entry, free_entry;unsigned long flags;
@@ -413,7 +416,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,static void iommu_free (struct iommu_table *tbl,dma_addr_t dma_addr,unsigned int npages) { -__iommu_free (tbl, dma_addr, npages); +iommu_dmacache_free (tbl, dma_addr, npages);/* Make sure TLB cache is flushed if the HW needs it. We do * not do an mb() here on purpose, it is not needed on any of
@@ -719,6 +722,8 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid,iommu_table_clear (tbl); +iommu_cache_init (tbl); +if (!welcomed) {printk (KERN_INFO "IOMMU table initialized, virtual merging %s\n" , novmerge ?"disabled" :"enabled" );
Signature · 1 line
2.25.4