Skip to contentSkip to conversation

zcrx huge pages support Vol 1

PATCH Superseded Pavel Begunkov 2 participants 12 messages Jun 17, 2025 – Jun 30, 2025

Subsystems (2)

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Pavel Begunkov

author maintainer
#0 lore raw

cover zcrx huge pages support Vol 1

Deduplicate some umem vs dmabuf code by creating sg_table for umem areas, and add huge page coalescing on top. It improves iommu mapping and compacts the page array, but leaves optimising the NIC page sizes to follow ups.

Pavel Begunkov (8):
io_uring/zcrx: return error from io_zcrx_map_area_* io_uring/zcrx: introduce io_populate_area_dma io_uring/zcrx: allocate sgtable for umem areas io_uring/zcrx: assert area type in io_zcrx_iov_page io_uring/zcrx: convert io_zcrx_iov_page to use folios io_uring/zcrx: add infra for large pages
io_uring: export io_coalesce_buffer() io_uring/zcrx: try to coalesce area pages

 io_uring/rsrc.c |   2 +-
 io_uring/rsrc.h |   2 +
 io_uring/zcrx.c | 179 +++++++++++++++++++++++++-----------------------
 io_uring/zcrx.h |   2 +
 4 files changed, 98 insertions(+), 87 deletions(-)
Signature · 1 line
2.49.0
author maintainer
#1 lore raw

PATCH 1/8 io_uring/zcrx: return error from io_zcrx_map_area_*

io_zcrx_map_area_*() helpers return the number of processed niovs, which we use to unroll some of the mappings for user memory areas. It's unhandy, and dmabuf doesn't care about it. Return an error code instead and move failure partial unmapping into io_zcrx_map_area_umem().

---
 io_uring/zcrx.c | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)
io_uring/zcrx.c +14 -13
index 797247a34cb7..7d361b661ccb 100644
@@ -139,13 +139,13 @@ static int io_zcrx_map_area_dmabuf(struct io_zcrx_ifq *ifq, struct io_zcrx_area
 			struct net_iov *niov = &area->nia.niovs[niov_idx];
 
 			if (net_mp_niov_set_dma_addr(niov, dma))
-				return 0;
+				return -EFAULT;
 			sg_len -= PAGE_SIZE;
 			dma += PAGE_SIZE;
 			niov_idx++;
 		}
 	}
-	return niov_idx;
+	return 0;
 }
 
 static int io_import_umem(struct io_zcrx_ifq *ifq,
@@ -254,29 +254,30 @@ static int io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *a
 			break;
 		}
 	}
-	return i;
+
+	if (i != area->nia.num_niovs) {
+		__io_zcrx_unmap_area(ifq, area, i);
+		return -EINVAL;
+	}
+	return 0;
 }
 
 static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
 {
-	unsigned nr;
+	int ret;
 
 	guard(mutex)(&ifq->dma_lock);
 	if (area->is_mapped)
 		return 0;
 
 	if (area->mem.is_dmabuf)
-		nr = io_zcrx_map_area_dmabuf(ifq, area);
+		ret = io_zcrx_map_area_dmabuf(ifq, area);
 	else
-		nr = io_zcrx_map_area_umem(ifq, area);
+		ret = io_zcrx_map_area_umem(ifq, area);
 
-	if (nr != area->nia.num_niovs) {
-		__io_zcrx_unmap_area(ifq, area, nr);
-		return -EINVAL;
-	}
-
-	area->is_mapped = true;
-	return 0;
+	if (ret == 0)
+		area->is_mapped = true;
+	return ret;
 }
 
 static void io_zcrx_sync_for_device(const struct page_pool *pool,
Signature · 1 line
2.49.0
author maintainer
#2 lore raw

PATCH 2/8 io_uring/zcrx: introduce io_populate_area_dma

In reply to Pavel Begunkov

Add a helper that initialises page-pool dma addresses from a sg table. It'll be reused in following patches.

---
 io_uring/zcrx.c | 56 +++++++++++++++++++++++++++----------------------
 1 file changed, 31 insertions(+), 25 deletions(-)
io_uring/zcrx.c +31 -25
index 7d361b661ccb..2b82fda6d934 100644
@@ -47,6 +47,35 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
 	return area->mem.pages[net_iov_idx(niov)];
 }
 
+static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
+				struct io_zcrx_area *area,
+				struct sg_table *sgt, unsigned long off)
+{
+	struct scatterlist *sg;
+	unsigned i, niov_idx = 0;
+
+	for_each_sgtable_dma_sg(sgt, sg, i) {
+		dma_addr_t dma = sg_dma_address(sg);
+		unsigned long sg_len = sg_dma_len(sg);
+		unsigned long sg_off = min(sg_len, off);
+
+		off -= sg_off;
+		sg_len -= sg_off;
+		dma += sg_off;
+
+		while (sg_len && niov_idx < area->nia.num_niovs) {
+			struct net_iov *niov = &area->nia.niovs[niov_idx];
+
+			if (net_mp_niov_set_dma_addr(niov, dma))
+				return -EFAULT;
+			sg_len -= PAGE_SIZE;
+			dma += PAGE_SIZE;
+			niov_idx++;
+		}
+	}
+	return 0;
+}
+
 static void io_release_dmabuf(struct io_zcrx_mem *mem)
 {
 	if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
@@ -119,33 +148,10 @@ static int io_import_dmabuf(struct io_zcrx_ifq *ifq,
 
 static int io_zcrx_map_area_dmabuf(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
 {
-	unsigned long off = area->mem.dmabuf_offset;
-	struct scatterlist *sg;
-	unsigned i, niov_idx = 0;
-
 	if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER))
 		return -EINVAL;
-
-	for_each_sgtable_dma_sg(area->mem.sgt, sg, i) {
-		dma_addr_t dma = sg_dma_address(sg);
-		unsigned long sg_len = sg_dma_len(sg);
-		unsigned long sg_off = min(sg_len, off);
-
-		off -= sg_off;
-		sg_len -= sg_off;
-		dma += sg_off;
-
-		while (sg_len && niov_idx < area->nia.num_niovs) {
-			struct net_iov *niov = &area->nia.niovs[niov_idx];
-
-			if (net_mp_niov_set_dma_addr(niov, dma))
-				return -EFAULT;
-			sg_len -= PAGE_SIZE;
-			dma += PAGE_SIZE;
-			niov_idx++;
-		}
-	}
-	return 0;
+	return io_populate_area_dma(ifq, area, area->mem.sgt,
+				    area->mem.dmabuf_offset);
 }
 
 static int io_import_umem(struct io_zcrx_ifq *ifq,
Signature · 1 line
2.49.0
author maintainer
#3 lore raw

PATCH 3/8 io_uring/zcrx: allocate sgtable for umem areas

In reply to Pavel Begunkov

Currently, dma addresses for umem areas are stored directly in niovs. It's memory efficient but inconvenient. I need a better format 1) to share code with dmabuf areas, and 2) for disentangling page, folio and niov sizes. dmabuf already provides sg_table, create one for user memory as well.

---
 io_uring/zcrx.c | 80 ++++++++++++++++++-------------------------------
 io_uring/zcrx.h |  1 +
 2 files changed, 30 insertions(+), 51 deletions(-)
io_uring/zcrx.c +29 -51
index 2b82fda6d934..91e795e6ae1a 100644
@@ -158,8 +158,10 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 			  struct io_zcrx_mem *mem,
 			  struct io_uring_zcrx_area_reg *area_reg)
 {
+	struct scatterlist *sgl;
 	struct page **pages;
-	int nr_pages;
+	int nr_pages, ret;
+	unsigned i = 0;
 
 	if (area_reg->dmabuf_fd)
 		return -EINVAL;
@@ -170,6 +172,12 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
+	ret = sg_alloc_table(&mem->page_sg_table, nr_pages, GFP_KERNEL_ACCOUNT);
+	if (ret)
+		return ret;
+	for_each_sg(mem->page_sg_table.sgl, sgl, nr_pages, i)
+		sg_set_page(sgl, pages[i], PAGE_SIZE, 0);
+
 	mem->pages = pages;
 	mem->nr_folios = nr_pages;
 	mem->size = area_reg->len;
@@ -184,6 +192,7 @@ static void io_release_area_mem(struct io_zcrx_mem *mem)
 	}
 	if (mem->pages) {
 		unpin_user_pages(mem->pages, mem->nr_folios);
+		sg_free_table(&mem->page_sg_table);
 		kvfree(mem->pages);
 	}
 }
@@ -205,67 +214,36 @@ static int io_import_area(struct io_zcrx_ifq *ifq,
 	return io_import_umem(ifq, mem, area_reg);
 }
 
-static void io_zcrx_unmap_umem(struct io_zcrx_ifq *ifq,
-				struct io_zcrx_area *area, int nr_mapped)
-{
-	int i;
-
-	for (i = 0; i < nr_mapped; i++) {
-		netmem_ref netmem = net_iov_to_netmem(&area->nia.niovs[i]);
-		dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem);
-
-		dma_unmap_page_attrs(ifq->dev, dma, PAGE_SIZE,
-				     DMA_FROM_DEVICE, IO_DMA_ATTR);
-	}
-}
-
-static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
-				 struct io_zcrx_area *area, int nr_mapped)
+static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
+				struct io_zcrx_area *area)
 {
 	int i;
 
-	if (area->mem.is_dmabuf)
-		io_release_dmabuf(&area->mem);
-	else
-		io_zcrx_unmap_umem(ifq, area, nr_mapped);
+	guard(mutex)(&ifq->dma_lock);
+	if (!area->is_mapped)
+		return;
+	area->is_mapped = false;
 
 	for (i = 0; i < area->nia.num_niovs; i++)
 		net_mp_niov_set_dma_addr(&area->nia.niovs[i], 0);
-}
-
-static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
-{
-	guard(mutex)(&ifq->dma_lock);
 
-	if (area->is_mapped)
-		__io_zcrx_unmap_area(ifq, area, area->nia.num_niovs);
-	area->is_mapped = false;
+	if (area->mem.is_dmabuf) {
+		io_release_dmabuf(&area->mem);
+	} else {
+		dma_unmap_sgtable(ifq->dev, &area->mem.page_sg_table,
+				  DMA_FROM_DEVICE, IO_DMA_ATTR);
+	}
 }
 
-static int io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
+static unsigned io_zcrx_map_area_umem(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
 {
-	int i;
-
-	for (i = 0; i < area->nia.num_niovs; i++) {
-		struct net_iov *niov = &area->nia.niovs[i];
-		dma_addr_t dma;
-
-		dma = dma_map_page_attrs(ifq->dev, area->mem.pages[i], 0,
-					 PAGE_SIZE, DMA_FROM_DEVICE, IO_DMA_ATTR);
-		if (dma_mapping_error(ifq->dev, dma))
-			break;
-		if (net_mp_niov_set_dma_addr(niov, dma)) {
-			dma_unmap_page_attrs(ifq->dev, dma, PAGE_SIZE,
-					     DMA_FROM_DEVICE, IO_DMA_ATTR);
-			break;
-		}
-	}
+	int ret;
 
-	if (i != area->nia.num_niovs) {
-		__io_zcrx_unmap_area(ifq, area, i);
-		return -EINVAL;
-	}
-	return 0;
+	ret = dma_map_sgtable(ifq->dev, &area->mem.page_sg_table,
+				DMA_FROM_DEVICE, IO_DMA_ATTR);
+	if (ret < 0)
+		return ret;
+	return io_populate_area_dma(ifq, area, &area->mem.page_sg_table, 0);
 }
 
 static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
io_uring/zcrx.h +1 -0
index 2f5e26389f22..89015b923911 100644
@@ -14,6 +14,7 @@ struct io_zcrx_mem {
 
 	struct page			**pages;
 	unsigned long			nr_folios;
+	struct sg_table			page_sg_table;
 
 	struct dma_buf_attachment	*attach;
 	struct dma_buf			*dmabuf;
Signature · 1 line
2.49.0
author maintainer
#4 lore raw

PATCH 4/8 io_uring/zcrx: assert area type in io_zcrx_iov_page

In reply to Pavel Begunkov

Add a simple debug assertion to io_zcrx_iov_page() making it's not trying to return pages for a dmabuf area.

---
 io_uring/zcrx.c | 2 ++
 1 file changed, 2 insertions(+)
io_uring/zcrx.c +2 -0
index 91e795e6ae1a..a2493798e6f8 100644
@@ -44,6 +44,8 @@ static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
 {
 	struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
 
+	lockdep_assert(!area->mem.is_dmabuf);
+
 	return area->mem.pages[net_iov_idx(niov)];
 }
 
Signature · 1 line
2.49.0
author maintainer
#5 lore raw

PATCH 5/8 io_uring/zcrx: convert io_zcrx_iov_page to use folios

In reply to Pavel Begunkov

Modernise io_zcrx_iov_page() and make it return folios + offset. That will be used for the next patches implementing area's page coalescing.

---
 io_uring/zcrx.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)
io_uring/zcrx.c +10 -5
index a2493798e6f8..a83b80c16491 100644
@@ -40,13 +40,17 @@ static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *nio
 	return container_of(owner, struct io_zcrx_area, nia);
 }
 
-static inline struct page *io_zcrx_iov_page(const struct net_iov *niov)
+static inline struct folio *io_niov_folio(const struct net_iov *niov,
+					  unsigned long *off)
 {
 	struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
+	struct page *page;
 
 	lockdep_assert(!area->mem.is_dmabuf);
 
-	return area->mem.pages[net_iov_idx(niov)];
+	page = area->mem.pages[net_iov_idx(niov)];
+	*off = (page - compound_head(page)) << PAGE_SHIFT;
+	return page_folio(page);
 }
 
 static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
@@ -944,7 +948,8 @@ static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
 		size_t copy_size = min_t(size_t, PAGE_SIZE, len);
 		const int dst_off = 0;
 		struct net_iov *niov;
-		struct page *dst_page;
+		struct folio *dst_folio;
+		unsigned long dst_folio_off;
 		void *dst_addr;
 
 		niov = io_zcrx_alloc_fallback(area);
@@ -953,8 +958,8 @@ static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
 			break;
 		}
 
-		dst_page = io_zcrx_iov_page(niov);
-		dst_addr = kmap_local_page(dst_page);
+		dst_folio = io_niov_folio(niov, &dst_folio_off);
+		dst_addr = kmap_local_folio(dst_folio, dst_folio_off);
 		if (src_page)
 			src_base = kmap_local_page(src_page);
 
Signature · 1 line
2.49.0
author maintainer
#6 lore raw

PATCH 6/8 io_uring/zcrx: add infra for large pages

In reply to Pavel Begunkov

Currently, zcrx assumes PAGE_SIZE pages, add infrastructure to support uniformly sized higher order pages. sg_table hides most of the details and the only place that need index recalculation is the copy fallback.

---
 io_uring/zcrx.c | 13 ++++++++-----
 io_uring/zcrx.h |  1 +
 2 files changed, 9 insertions(+), 5 deletions(-)
io_uring/zcrx.c +8 -5
index a83b80c16491..44b5f8084279 100644
@@ -44,13 +44,14 @@ static inline struct folio *io_niov_folio(const struct net_iov *niov,
 					  unsigned long *off)
 {
 	struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
-	struct page *page;
+	unsigned long niov_off, folio_idx;
 
 	lockdep_assert(!area->mem.is_dmabuf);
 
-	page = area->mem.pages[net_iov_idx(niov)];
-	*off = (page - compound_head(page)) << PAGE_SHIFT;
-	return page_folio(page);
+	niov_off = net_iov_idx(niov) << PAGE_SHIFT;
+	folio_idx = niov_off >> area->mem.folio_shift;
+	*off = niov_off - (folio_idx << area->mem.folio_shift);
+	return page_folio(area->mem.pages[folio_idx]);
 }
 
 static int io_populate_area_dma(struct io_zcrx_ifq *ifq,
@@ -164,6 +165,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 			  struct io_zcrx_mem *mem,
 			  struct io_uring_zcrx_area_reg *area_reg)
 {
+	unsigned folio_shift = PAGE_SHIFT;
 	struct scatterlist *sgl;
 	struct page **pages;
 	int nr_pages, ret;
@@ -182,11 +184,12 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 	if (ret)
 		return ret;
 	for_each_sg(mem->page_sg_table.sgl, sgl, nr_pages, i)
-		sg_set_page(sgl, pages[i], PAGE_SIZE, 0);
+		sg_set_page(sgl, pages[i], 1U << folio_shift, 0);
 
 	mem->pages = pages;
 	mem->nr_folios = nr_pages;
 	mem->size = area_reg->len;
+	mem->folio_shift = folio_shift;
 	return 0;
 }
 
io_uring/zcrx.h +1 -0
index 89015b923911..4f718b3088d9 100644
@@ -14,6 +14,7 @@ struct io_zcrx_mem {
 
 	struct page			**pages;
 	unsigned long			nr_folios;
+	unsigned			folio_shift;
 	struct sg_table			page_sg_table;
 
 	struct dma_buf_attachment	*attach;
Signature · 1 line
2.49.0
author maintainer
#7 lore raw

PATCH 7/8 io_uring: export io_coalesce_buffer()

In reply to Pavel Begunkov

We'll need io_coalesce_buffer() in the next patch for zcrx, export it.

---
 io_uring/rsrc.c | 2 +-
 io_uring/rsrc.h | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)
io_uring/rsrc.c +1 -1
index c592ceace97d..bab0ea45046d 100644
@@ -684,7 +684,7 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
 	return ret;
 }
 
-static bool io_coalesce_buffer(struct page ***pages, int *nr_pages,
+bool io_coalesce_buffer(struct page ***pages, int *nr_pages,
 				struct io_imu_folio_data *data)
 {
 	struct page **page_array = *pages, **new_array = NULL;
io_uring/rsrc.h +2 -0
index 0d2138f16322..4dcedfa69b8c 100644
@@ -87,6 +87,8 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen);
 
 bool io_check_coalesce_buffer(struct page **page_array, int nr_pages,
 			      struct io_imu_folio_data *data);
+bool io_coalesce_buffer(struct page ***pages, int *nr_pages,
+			struct io_imu_folio_data *data);
 
 static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data,
 						       int index)
Signature · 1 line
2.49.0
author maintainer
#8 lore raw

PATCH 8/8 io_uring/zcrx: try to coalesce area pages

In reply to Pavel Begunkov

Try to shrink the page array into fewer larger folios if possible. This reduces the footprint, optimises dma mappings and will be used in the future for further huge page optimisations.

---
 io_uring/zcrx.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)
io_uring/zcrx.c +12 -0
index 44b5f8084279..9f81682ccf0c 100644
@@ -166,6 +166,7 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 			  struct io_uring_zcrx_area_reg *area_reg)
 {
 	unsigned folio_shift = PAGE_SHIFT;
+	struct io_imu_folio_data data;
 	struct scatterlist *sgl;
 	struct page **pages;
 	int nr_pages, ret;
@@ -180,6 +181,17 @@ static int io_import_umem(struct io_zcrx_ifq *ifq,
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
+	if (nr_pages > 1 && io_check_coalesce_buffer(pages, nr_pages, &data)) {
+		/*
+		 * Only coalesce folio addr-aligned pages and when we can
+		 * improve the size.
+		 */
+		if (data.nr_pages_mid != 1 &&
+		    data.nr_pages_head == data.nr_pages_mid &&
+		    io_coalesce_buffer(&pages, &nr_pages, &data))
+			folio_shift = data.folio_shift;
+	}
+
 	ret = sg_alloc_table(&mem->page_sg_table, nr_pages, GFP_KERNEL_ACCOUNT);
 	if (ret)
 		return ret;
Signature · 1 line
2.49.0
author maintainer
#9 lore raw
In reply to Pavel Begunkov
Quoted 4 lines · Pavel Begunkov

Deduplicate some umem vs dmabuf code by creating sg_table for umem areas, and add huge page coalescing on top. It improves iommu mapping and compacts the page array, but leaves optimising the NIC page sizes to follow ups.

Any comments?

Signature · 1 line
Pavel Begunkov
maintainer
#10 lore raw
Quoted 7 lines · Pavel Begunkov
Quoted 4 lines · Pavel Begunkov

Deduplicate some umem vs dmabuf code by creating sg_table for umem areas, and add huge page coalescing on top. It improves iommu mapping and compacts the page array, but leaves optimising the NIC page sizes to follow ups.

Any comments?

I'll take a closer look tomorrow, it's on the backlog after being away for 10 days.

Signature · 1 line
Jens Axboe
author maintainer
#11 lore raw
Quoted 11 lines · Jens Axboe
Quoted 7 lines · Pavel Begunkov
Quoted 4 lines · Pavel Begunkov

Deduplicate some umem vs dmabuf code by creating sg_table for umem areas, and add huge page coalescing on top. It improves iommu mapping and compacts the page array, but leaves optimising the NIC page sizes to follow ups.

Any comments?

I'll take a closer look tomorrow, it's on the backlog after being away for 10 days.

No worries

Signature · 1 line
Pavel Begunkov