dma-buf: heaps: Add deferred-free-helper library code
PATCH v2 1/3 dma-buf: heaps: Add deferred-free-helper library code
This patch provides infrastructure for deferring buffer frees.
This is a feature ION provided which when used with some form of a page pool, provides a nice performance boost in an allocation microbenchmark. The reason it helps is it allows the page-zeroing to be done out of the normal allocation/free path, and pushed off to a kthread.
As not all heaps will find this useful, its implemented as a optional helper library that heaps can utilize.
---
v2:
* Fix sleep in atomic issue from using a mutex, by switching to a spinlock as Reported-by: kernel test robot <[email protected]>
* Cleanup API to use a reason enum for clarity and add some documentation comments as suggested by Suren Baghdasaryan.
--- drivers/dma-buf/heaps/Kconfig | 3 + drivers/dma-buf/heaps/Makefile | 1 + drivers/dma-buf/heaps/deferred-free-helper.c | 136 +++++++++++++++++++ drivers/dma-buf/heaps/deferred-free-helper.h | 55 ++++++++ 4 files changed, 195 insertions(+) create mode 100644 drivers/dma-buf/heaps/deferred-free-helper.c create mode 100644 drivers/dma-buf/heaps/deferred-free-helper.h
drivers/dma-buf/heaps/Kconfig +3 -0
@@ -1,3 +1,6 @@ +config DMABUF_HEAPS_DEFERRED_FREE + bool + config DMABUF_HEAPS_SYSTEM bool "DMA-BUF System Heap" depends on DMABUF_HEAPS
drivers/dma-buf/heaps/Makefile +1 -0
@@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_DMABUF_HEAPS_DEFERRED_FREE) += deferred-free-helper.o obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o
drivers/dma-buf/heaps/deferred-free-helper.c new file +136 -0
136 lines not on this page · raw message
drivers/dma-buf/heaps/deferred-free-helper.h new file +55 -0
55 lines not on this page · raw message
Signature · 6 lines
2.17.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
PATCH v2 2/3 dma-buf: system_heap: Add pagepool support to system heap
Reuse/abuse the pagepool code from the network code to speed up allocation performance.
This is similar to the ION pagepool usage, but tries to utilize generic code instead of a custom implementation.
---
v2:
* Fix build issue caused by selecting PAGE_POOL w/o NET as Reported-by: kernel test robot <[email protected]>
--- drivers/dma-buf/heaps/Kconfig | 2 + drivers/dma-buf/heaps/system_heap.c | 68 +++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 4 deletions(-)
drivers/dma-buf/heaps/Kconfig +2 -0
@@ -4,6 +4,8 @@ config DMABUF_HEAPS_DEFERRED_FREE config DMABUF_HEAPS_SYSTEM bool "DMA-BUF System Heap" depends on DMABUF_HEAPS + select NET + select PAGE_POOL help Choose this option to enable the system dmabuf heap. The system heap is backed by pages from the buddy allocator. If in doubt, say Y.
drivers/dma-buf/heaps/system_heap.c +64 -4
108 lines not on this page · raw message
Signature · 6 lines
2.17.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH v2 2/3] dma-buf: system_heap: Add pagepool support to system heap
drivers/dma-buf/heaps/system_heap.c static void system_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) · line 295
Quoted 10 lines · John Stultz
drivers/dma-buf/heaps/system_heap.c +10
-0+static int system_heap_clear_pages (struct page **pages,int num,pgprot_t pgprot) +{ +void *addr =vmap (pages, num,VM_MAP , pgprot); + +if (!addr) +return -ENOMEM ; +memset (addr,0 ,PAGE_SIZE * num); +vunmap (addr); +return 0 ; +}
I thought that vmap/vunmap are expensive, and I am wondering if there's a faster way that avoids vmap.
How about lifting this code from lib/iov_iter.c
static void memzero_page(struct page *page, size_t offset, size_t len)
{
char *addr = kmap_atomic(page);
memset(addr + offset, 0, len);
kunmap_atomic(addr);
}Or what about lifting that code from the old ion_cma_heap.c
if (PageHighMem(pages)) {
unsigned long nr_clear_pages = nr_pages;
struct page *page = pages;
while (nr_clear_pages > 0) {
void *vaddr = kmap_atomic(page);
memset(vaddr, 0, PAGE_SIZE);
kunmap_atomic(vaddr);
page++;
nr_clear_pages--;
}
} else {
memset(page_address(pages), 0, size);
}Re: [PATCH v2 2/3] dma-buf: system_heap: Add pagepool support to system heap
Quoted 26 lines · Daniel Mentz
drivers/dma-buf/heaps/system_heap.c static void system_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) · line 295
Quoted 10 lines · John Stultz
drivers/dma-buf/heaps/system_heap.c +10
-0+static int system_heap_clear_pages (struct page **pages,int num,pgprot_t pgprot) +{ +void *addr =vmap (pages, num,VM_MAP , pgprot); + +if (!addr) +return -ENOMEM ; +memset (addr,0 ,PAGE_SIZE * num); +vunmap (addr); +return 0 ; +}I thought that vmap/vunmap are expensive, and I am wondering if there's a faster way that avoids vmap.
How about lifting this code from
lib/iov_iter.c
static voidmemzero_page(struct page *page, size_t offset, size_t len){ char *addr = kmap_atomic(page); memset(addr + offset, 0, len); kunmap_atomic(addr); }Or what about lifting that code from the old
ion_cma_heap.c
Quoted 15 lines
if (PageHighMem(pages)) {
unsigned long nr_clear_pages = nr_pages;
struct page *page = pages;
while (nr_clear_pages > 0) {
void *vaddr = kmap_atomic(page);
memset(vaddr, 0, PAGE_SIZE);
kunmap_atomic(vaddr);
page++;
nr_clear_pages--;
}
} else {
memset(page_address(pages), 0, size);
}Though, this last memset only works since CMA is contiguous, so it probably needs to always do the kmap_atomic for each page, right?
I'm still a little worried if this is right, as the current implementation with the vmap comes from the old ion_heap_sglist_zero logic, which similarly tries to batch the vmaps 32 pages at at time, but I'll give it a try.
thanks
-john
_______________________________________________
dri-devel mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/dri-devel
Re: [PATCH v2 2/3] dma-buf: system_heap: Add pagepool support to system heap
Quoted 44 lines · John Stultz
Quoted 26 lines · Daniel Mentz
drivers/dma-buf/heaps/system_heap.c static void system_heap_vunmap(struct dma_buf *dmabuf, struct dma_buf_map *map) · line 295
Quoted 10 lines · John Stultz
drivers/dma-buf/heaps/system_heap.c +10
-0+static int system_heap_clear_pages (struct page **pages,int num,pgprot_t pgprot) +{ +void *addr =vmap (pages, num,VM_MAP , pgprot); + +if (!addr) +return -ENOMEM ; +memset (addr,0 ,PAGE_SIZE * num); +vunmap (addr); +return 0 ; +}I thought that vmap/vunmap are expensive, and I am wondering if there's a faster way that avoids vmap.
How about lifting this code from
lib/iov_iter.c
static voidmemzero_page(struct page *page, size_t offset, size_t len){ char *addr = kmap_atomic(page); memset(addr + offset, 0, len); kunmap_atomic(addr); }Or what about lifting that code from the old
ion_cma_heap.cQuoted 15 lines
if (PageHighMem(pages)) { unsigned long nr_clear_pages = nr_pages; struct page *page = pages; while (nr_clear_pages > 0) { void *vaddr = kmap_atomic(page); memset(vaddr, 0, PAGE_SIZE); kunmap_atomic(vaddr); page++; nr_clear_pages--; } } else { memset(page_address(pages), 0, size); }
Quoted 2 lines
Though, this last memset only works since CMA is contiguous, so it probably needs to always do the
kmap_atomicfor each page, right?
Yeah, but with the system heap page pool, some of these pages might be 64KB or 1MB large. kmap_atomic(page) just maps to page_address(page) in most cases. I think iterating over all pages individually in this manner might still be faster than using vmap.
Quoted 5 lines
I'm still a little worried if this is right, as the current implementation with the vmap comes from the old
ion_heap_sglist_zerologic, which similarly tries to batch the vmaps 32 pages at at time, but I'll give it a try.
PATCH v2 3/3 dma-buf: system_heap: Add deferred freeing to the system heap
In reply to John StultzUtilize the deferred free helper library in the system heap.
This provides a nice performance bump and puts the system heap performance on par with ION.
---
v2:
* Rework deferred-free api to use reason enum as suggested by Suren Baghdasaryan
--- drivers/dma-buf/heaps/Kconfig | 1 + drivers/dma-buf/heaps/system_heap.c | 32 ++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 7 deletions(-)
drivers/dma-buf/heaps/Kconfig +1 -0
@@ -6,6 +6,7 @@ config DMABUF_HEAPS_SYSTEM depends on DMABUF_HEAPS select NET select PAGE_POOL + select DMABUF_HEAPS_DEFERRED_FREE help Choose this option to enable the system dmabuf heap. The system heap is backed by pages from the buddy allocator. If in doubt, say Y.
drivers/dma-buf/heaps/system_heap.c +25 -7
@@ -22,6 +22,8 @@#include <linux/vmalloc.h> #include <net/page_pool.h> +#include "deferred-free-helper.h" +static struct dma_heap *sys_heap;struct system_heap_buffer {
@@ -32,6 +34,7 @@ struct system_heap_buffer {struct sg_table sg_table;int vmap_cnt;void *vaddr; +struct deferred_freelist_item deferred_free; };struct dma_heap_attachment {
@@ -317,30 +320,45 @@ static int system_heap_zero_buffer(struct system_heap_buffer *buffer)return ret; } -static void system_heap_dma_buf_release (struct dma_buf *dmabuf) +static void system_heap_buf_free (struct deferred_freelist_item *item, +enum df_reason reason) { -struct system_heap_buffer *buffer = dmabuf->priv; +struct system_heap_buffer *buffer;struct sg_table *table;struct scatterlist *sg;int i, j; + buffer = container_of(item, struct system_heap_buffer, deferred_free);/* Zero the buffer pages before adding back to the pool */ -system_heap_zero_buffer (buffer); +if (reason ==DF_NORMAL ) +if (system_heap_zero_buffer (buffer)) + reason =DF_UNDER_PRESSURE ;// On failure, just free table = &buffer->sg_table;for_each_sg (table->sgl, sg, table->nents, i) {struct page *page =sg_page (sg); -for (j =0 ; j <NUM_ORDERS ; j++) { -if (compound_order (page) == orders[j]) -break ; +if (reason ==DF_UNDER_PRESSURE ) { +__free_pages (page,compound_order (page)); + }else { +for (j =0 ; j <NUM_ORDERS ; j++) { +if (compound_order (page) == orders[j]) +break ; + } +page_pool_put_full_page (pools[j], page, false); } -page_pool_put_full_page (pools[j], page, false); }sg_free_table (table);kfree (buffer); } +static void system_heap_dma_buf_release (struct dma_buf *dmabuf) +{ +struct system_heap_buffer *buffer = dmabuf->priv; + +deferred_free (&buffer->deferred_free, system_heap_buf_free, buffer->len); +} +static const struct dma_buf_ops system_heap_buf_ops = { .attach = system_heap_attach, .detach = system_heap_detach,
Signature · 6 lines
2.17.1 _______________________________________________ dri-devel mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/dri-devel