preparation for zcrx with huge pages
Subsystems (2)
Pavel Begunkov
maintainercover preparation for zcrx with huge pages
Add barebone support for huge pages for zcrx, with the only real effect is shrinking the page array. However, it's a prerequisite for other huge page optimisations, like improved dma mappings and large page pool allocation sizes.
There is no new uapi, but there is a basic example:
https://github.com/isilence/liburing/tree/zcrx-huge-page
Pavel Begunkov (4):
io_uring/zcrx: add helper for importing user memory io_uring/zcrx: add initial infra for large pages
io_uring: export io_coalesce_buffer() io_uring/zcrx: coalesce areas with huge pages
io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 2 ++ io_uring/zcrx.c | 88 +++++++++++++++++++++++++++++++++++++------------ io_uring/zcrx.h | 3 ++ 4 files changed, 73 insertions(+), 22 deletions(-)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH 1/4 io_uring/zcrx: add helper for importing user memory
There are two distinct steps for creating an area. First, we import user memory, and then populate net_iov. In preparation to changes for the first step, extract a helper function for it.
--- io_uring/zcrx.c | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-)
io_uring/zcrx.c +31 -16
@@ -203,13 +203,38 @@ static void io_zcrx_free_area(struct io_zcrx_area *area)kfree (area); } +static int io_import_area_memory (struct io_zcrx_ifq *ifq, +struct io_zcrx_area *area, +struct io_uring_zcrx_area_reg *area_reg) +{ +struct iovec iov; +int nr_pages; +int ret; + + iov.iov_base =u64_to_user_ptr (area_reg->addr); + iov.iov_len = area_reg->len; + ret =io_buffer_validate (&iov); +if (ret) +return ret; + + area->pages =io_pin_pages ((unsigned long )area_reg->addr, area_reg->len, + &nr_pages); +if (IS_ERR (area->pages)) { + ret =PTR_ERR (area->pages); + area->pages =NULL ; +return ret; + } + area->nr_folios = nr_pages; +return 0 ; +} +static int io_zcrx_create_area (struct io_zcrx_ifq *ifq,struct io_zcrx_area **res,struct io_uring_zcrx_area_reg *area_reg) { +unsigned nr_iovs = area_reg->len >>PAGE_SHIFT ;struct io_zcrx_area *area; -int i, ret, nr_pages, nr_iovs; -struct iovec iov; +int i, ret;if (area_reg->flags || area_reg->rq_area_token)return -EINVAL ;
@@ -218,27 +243,17 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,if (area_reg->addr & ~PAGE_MASK || area_reg->len & ~PAGE_MASK )return -EINVAL ; - iov.iov_base =u64_to_user_ptr (area_reg->addr); - iov.iov_len = area_reg->len; - ret =io_buffer_validate (&iov); -if (ret) -return ret; - ret = -ENOMEM ; area =kzalloc (sizeof (*area),GFP_KERNEL );if (!area) goto err; + area->nia.num_niovs = nr_iovs; - area->pages =io_pin_pages ((unsigned long )area_reg->addr, area_reg->len, - &nr_pages); -if (IS_ERR (area->pages)) { - ret =PTR_ERR (area->pages); - area->pages =NULL ; + ret =io_import_area_memory (ifq, area, area_reg); +if (ret) goto err; - } - area->nr_folios = nr_iovs = nr_pages; - area->nia.num_niovs = nr_iovs; + ret = -ENOMEM ; area->nia.niovs =kvmalloc_array (nr_iovs,sizeof (area->nia.niovs[0 ]),GFP_KERNEL | __GFP_ZERO);if (!area->nia.niovs)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH 2/4 io_uring/zcrx: add initial infra for large pages
In reply to Pavel BegunkovCurrently, the page array and net_iovs are 4K sized and have the same number of elements. Allow the page array to be of a different shape, which will be needed to support huge pages. The total size should always match, but now we can store fewer larger pages / folios. The only restriction here is that the folios size should always be equal or larger than the niov size.
Note, there is no way just yet to really shrink the page array, and it'll be added in following patches.
--- io_uring/zcrx.c | 24 +++++++++++++++++++----- io_uring/zcrx.h | 3 +++ 2 files changed, 22 insertions(+), 5 deletions(-)
io_uring/zcrx.c +19 -5
@@ -38,11 +38,21 @@ static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *nio return container_of(owner, struct io_zcrx_area, nia); } -static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) +/* shift from chunk / niov to folio size */ +static inline unsigned io_chunk_folio_shift (struct io_zcrx_area *area) { -struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); +return area->folio_shift -PAGE_SHIFT ; +} + +static struct page *io_zcrx_iov_page (struct io_zcrx_area *area, +const struct net_iov *niov) +{ +unsigned chunk_gid =net_iov_idx (niov) + area->chunk_id_offset; +unsigned folio_idx, base_chunk_gid; -return area->pages[net_iov_idx (niov)]; + folio_idx = chunk_gid >>io_chunk_folio_shift (area); + base_chunk_gid = folio_idx <<io_chunk_folio_shift (area); +return area->pages[folio_idx] + (chunk_gid - base_chunk_gid); }#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
@@ -82,9 +92,11 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)for (i =0 ; i < area->nia.num_niovs; i++) {struct net_iov *niov = &area->nia.niovs[i]; +struct page *page;dma_addr_t dma; - dma =dma_map_page_attrs (ifq->dev, area->pages[i],0 ,PAGE_SIZE , + page =io_zcrx_iov_page (area, niov); + dma =dma_map_page_attrs (ifq->dev, page,0 ,PAGE_SIZE ,DMA_FROM_DEVICE ,IO_DMA_ATTR );if (dma_mapping_error (ifq->dev, dma))break ;
@@ -225,6 +237,8 @@ static int io_import_area_memory(struct io_zcrx_ifq *ifq,return ret; } area->nr_folios = nr_pages; + area->folio_shift =PAGE_SHIFT ; + area->chunk_id_offset =0 ;return 0 ; }
@@ -807,7 +821,7 @@ static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,break ; } - dst_page =io_zcrx_iov_page (niov); + dst_page =io_zcrx_iov_page (area, niov); dst_addr =kmap_local_page (dst_page);if (src_page) src_base =kmap_local_page (src_page);
io_uring/zcrx.h +3 -0
@@ -15,7 +15,10 @@ struct io_zcrx_area {bool is_mapped;u16 area_id;struct page **pages; +/* offset into the first folio in allocation chunks */ +unsigned long chunk_id_offset;unsigned long nr_folios; +unsigned folio_shift;/* freelist */ spinlock_t freelist_lock____cacheline_aligned_in_smp ;
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH 3/4 io_uring: export io_coalesce_buffer()
In reply to Pavel BegunkovWe'll need io_coalesce_buffer() in the next patch for zcrx, export it.
--- io_uring/rsrc.c | 2 +- io_uring/rsrc.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-)
io_uring/rsrc.c +1 -1
@@ -681,7 +681,7 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,return ret; } -static bool io_coalesce_buffer (struct page ***pages,int *nr_pages, +bool io_coalesce_buffer (struct page ***pages,int *nr_pages,struct io_imu_folio_data *data) {struct page **page_array = *pages, **new_array =NULL ;
io_uring/rsrc.h +2 -0
@@ -87,6 +87,8 @@ int io_buffer_validate(struct iovec *iov);bool io_check_coalesce_buffer (struct page **page_array,int nr_pages,struct io_imu_folio_data *data); +bool io_coalesce_buffer (struct page ***pages,int *nr_pages, +struct io_imu_folio_data *data);static inline struct io_rsrc_node *io_rsrc_node_lookup (struct io_rsrc_data *data,int index)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH 4/4 io_uring/zcrx: coalesce areas with huge pages
In reply to Pavel BegunkovTry to shrink the page array into fewer larger folios if possible. This reduces the footprint for the array and prepares us for future huge page optimisations.
--- io_uring/zcrx.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
io_uring/zcrx.c +17 -0
@@ -219,6 +219,8 @@ static int io_import_area_memory(struct io_zcrx_ifq *ifq,struct io_zcrx_area *area, struct io_uring_zcrx_area_reg *area_reg) { + struct io_imu_folio_data data; +bool coalesced = false;struct iovec iov;int nr_pages;int ret;
@@ -239,6 +241,21 @@ static int io_import_area_memory(struct io_zcrx_ifq *ifq, area->nr_folios = nr_pages; area->folio_shift =PAGE_SHIFT ; area->chunk_id_offset =0 ; + +if (nr_pages >1 &&io_check_coalesce_buffer (area->pages, nr_pages, &data)) { +if (data.nr_pages_mid !=1 ) + coalesced =io_coalesce_buffer (&area->pages, &nr_pages, &data); + } + +if (coalesced) { +size_t folio_size =1UL << data.folio_shift; +size_t offset = folio_size - (data.nr_pages_head <<PAGE_SHIFT ); + + area->nr_folios = nr_pages; + area->folio_shift = data.folio_shift; + area->chunk_id_offset = offset >>PAGE_SHIFT ; + } +return 0 ; }
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerQuoted 4 lines · Pavel Begunkov
Add barebone support for huge pages for zcrx, with the only real effect is shrinking the page array. However, it's a prerequisite for other huge page optimisations, like improved dma mappings and large page pool allocation sizes.
I'm going to resend the series later with more changes.
Signature · 1 line
Pavel Begunkov
Quoted 8 lines · Pavel Begunkov
Quoted 4 lines · Pavel Begunkov
Add barebone support for huge pages for zcrx, with the only real effect is shrinking the page array. However, it's a prerequisite for other huge page optimisations, like improved dma mappings and large page pool allocation sizes.
I'm going to resend the series later with more changes.
Thanks Pavel, sorry for the lack of responses this week. I will look at the v2.
Pavel Begunkov
maintainerQuoted 7 lines · David Wei
Quoted 6 lines · Pavel Begunkov
Quoted 4 lines · Pavel Begunkov
Add barebone support for huge pages for zcrx, with the only real effect is shrinking the page array. However, it's a prerequisite for other huge page optimisations, like improved dma mappings and large page pool allocation sizes.
Quoted 5 lines
Quoted 2 lines
I'm going to resend the series later with more changes.
Thanks Pavel, sorry for the lack of responses this week. I will look at the v2.
No worries at all! Thanks
Signature · 1 line
Pavel Begunkov