add support for multiple ifqs per io_uring
Subsystems (2)
Pavel Begunkov
maintainerv2 cover add support for multiple ifqs per io_uring
Note: depends on patches queued for 6.15-rcN.
Patches 3-5 allow to register multiple ifqs within a single io_uring instance. That should be useful for setups with multiple interfaces.
Patch 1 and 2 and not related but I just bundled them together.
v2: fix returning the user an incorrect offset for mmap
add IORING_OFF_ZCRX_SHIFT
split Patch 3 out of Patch 6
Pavel Begunkov (6):
io_uring/zcrx: remove duplicated freelist init io_uring/zcrx: move io_zcrx_iov_page io_uring/zcrx: remove sqe->file_index check io_uring/zcrx: let zcrx choose region for mmaping io_uring/zcrx: move zcrx region to struct io_zcrx_ifq io_uring/zcrx: add support for multiple ifqs
include/linux/io_uring_types.h | 7 +-- io_uring/io_uring.c | 3 +- io_uring/memmap.c | 11 ++-- io_uring/memmap.h | 2 + io_uring/net.c | 8 +-- io_uring/zcrx.c | 107 ++++++++++++++++++++++----------- io_uring/zcrx.h | 8 +++ 7 files changed, 96 insertions(+), 50 deletions(-)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 1/6 io_uring/zcrx: remove duplicated freelist init
Several lines below we already initialise the freelist, don't do it twice.
--- io_uring/zcrx.c | 3 --- 1 file changed, 3 deletions(-)
io_uring/zcrx.c +0 -3
@@ -238,9 +238,6 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,if (!area->freelist) goto err; -for (i =0 ; i < nr_iovs; i++) - area->freelist[i] = i; - area->user_refs =kvmalloc_array (nr_iovs,sizeof (area->user_refs[0 ]),GFP_KERNEL | __GFP_ZERO);if (!area->user_refs)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 2/6 io_uring/zcrx: move io_zcrx_iov_page
In reply to Pavel BegunkovWe'll need io_zcrx_iov_page at the top to keep offset calculations closer together, move it there.
--- io_uring/zcrx.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-)
io_uring/zcrx.c +14 -14
@@ -31,6 +31,20 @@ static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)return pp->mp_priv; } +static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) +{ +struct net_iov_area *owner =net_iov_owner (niov); + + return container_of(owner, struct io_zcrx_area, nia); +} + +static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) +{ +struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); + +return area->pages[net_iov_idx (niov)]; +} +#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
@@ -111,13 +125,6 @@ struct io_zcrx_args {static const struct memory_provider_ops io_uring_pp_zc_ops; -static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) -{ -struct net_iov_area *owner =net_iov_owner (niov); - - return container_of(owner, struct io_zcrx_area, nia); -} -static inline atomic_t *io_get_user_counter (struct net_iov *niov) {struct io_zcrx_area *area =io_zcrx_iov_to_area (niov);
@@ -140,13 +147,6 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)atomic_inc (io_get_user_counter (niov)); } -static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) -{ -struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); - -return area->pages[net_iov_idx (niov)]; -} -static int io_allocate_rbuf_ring (struct io_zcrx_ifq *ifq,struct io_uring_zcrx_ifq_reg *reg,struct io_uring_region_desc *rd)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 3/6 io_uring/zcrx: remove sqe->file_index check
In reply to Pavel Begunkovsqe->file_index and sqe->zcrx_ifq_idx are aliased, so even though io_recvzc_prep() has all necessary handling and checking for ->zcrx_ifq_idx, it's already rejected a couple of lines above.
It's not a real problem, however, as we can only have one ifq at the moment, which index is always zero, and it returns correct error codes to the user.
--- io_uring/net.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/net.c +1 -2
@@ -1185,8 +1185,7 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);unsigned ifq_idx; -if (unlikely (sqe->file_index || sqe->addr2 || sqe->addr || - sqe->addr3)) +if (unlikely (sqe->addr2 || sqe->addr || sqe->addr3))return -EINVAL ; ifq_idx =READ_ONCE (sqe->zcrx_ifq_idx);
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 4/6 io_uring/zcrx: let zcrx choose region for mmaping
In reply to Pavel BegunkovIn preparation for adding multiple ifqs, add a helper returning a region for mmaping zcrx refill queue. For now it's trivial and returns the same ctx global ->zcrx_region.
--- io_uring/memmap.c | 11 +++++++---- io_uring/memmap.h | 2 ++ io_uring/zcrx.c | 10 ++++++++++ io_uring/zcrx.h | 7 +++++++ 4 files changed, 26 insertions(+), 4 deletions(-)
io_uring/memmap.c +7 -4
@@ -13,6 +13,7 @@#include "memmap.h" #include "kbuf.h" #include "rsrc.h" +#include "zcrx.h" static void *io_mem_alloc_compound (struct page **pages,int nr_pages,size_t size,gfp_t gfp)
@@ -258,7 +259,8 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, loff_t pgoff) {loff_t offset = pgoff <<PAGE_SHIFT ; -unsigned int bgid; +unsigned int id; +switch (offset &IORING_OFF_MMAP_MASK ) {case IORING_OFF_SQ_RING :
@@ -267,12 +269,13 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx,case IORING_OFF_SQES :return &ctx->sq_region;case IORING_OFF_PBUF_RING : - bgid = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ; -return io_pbuf_get_region (ctx, bgid); + id = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ; +return io_pbuf_get_region (ctx, id);case IORING_MAP_OFF_PARAM_REGION :return &ctx->param_region;case IORING_MAP_OFF_ZCRX_REGION : -return &ctx->zcrx_region; + id = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_ZCRX_SHIFT ; +return io_zcrx_get_region (ctx, id); }return NULL ; }
io_uring/memmap.h +2 -0
@@ -4,6 +4,8 @@#define IORING_MAP_OFF_PARAM_REGION 0x20000000ULL#define IORING_MAP_OFF_ZCRX_REGION 0x30000000ULL +#define IORING_OFF_ZCRX_SHIFT 16 +struct page **io_pin_pages (unsigned long ubuf,unsigned long len,int *npages);#ifndef CONFIG_MMU
io_uring/zcrx.c +10 -0
@@ -329,6 +329,16 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)kfree (ifq); } +struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id) +{ +lockdep_assert_held (&ctx->mmap_lock); + +if (id !=0 ) +return NULL ; +return &ctx->zcrx_region; +} +int io_register_zcrx_ifq (struct io_ring_ctx *ctx,struct io_uring_zcrx_ifq_reg __user *arg) {
io_uring/zcrx.h +7 -0
@@ -48,6 +48,8 @@ void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);int io_zcrx_recv (struct io_kiocb *req,struct io_zcrx_ifq *ifq,struct socket *sock,unsigned int flags,unsigned issue_flags,unsigned int *len); +struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id);#else static inline int io_register_zcrx_ifq (struct io_ring_ctx *ctx,struct io_uring_zcrx_ifq_reg __user *arg)
@@ -66,6 +68,11 @@ static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, {return -EOPNOTSUPP ; } +static inline struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id) +{ +return NULL ; +}#endif int io_recvzc (struct io_kiocb *req,unsigned int issue_flags);
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 5/6 io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
In reply to Pavel BegunkovRefill queue region is a part of zcrx and should stay in struct io_zcrx_ifq. We can't have multiple queues without it, so move it there.
As a result there is no context global zcrx region anymore, and the region is looked up together with its ifq. To protect a concurrent mmap from seeing an inconsistent region we were protecting changes to ->zcrx_region with mmap_lock, but now it protect the publishing of the ifq.
--- include/linux/io_uring_types.h | 2 -- io_uring/zcrx.c | 20 ++++++++++++-------- io_uring/zcrx.h | 1 + 3 files changed, 13 insertions(+), 10 deletions(-)
include/linux/io_uring_types.h +0 -2
@@ -448,8 +448,6 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; -/* just one zcrx per ring for now, will move to io_zcrx_ifq eventually */ -struct io_mapped_region zcrx_region; };/*
io_uring/zcrx.c +12 -8
@@ -160,12 +160,11 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,if (size > rd->size)return -EINVAL ; - ret =io_create_region_mmap_safe (ifq->ctx, &ifq->ctx->zcrx_region, rd, -IORING_MAP_OFF_ZCRX_REGION ); + ret =io_create_region (ifq->ctx, &ifq->region, rd,IORING_MAP_OFF_ZCRX_REGION );if (ret <0 )return ret; - ptr =io_region_get_ptr (&ifq->ctx->zcrx_region); + ptr =io_region_get_ptr (&ifq->region); ifq->rq_ring = (struct io_uring *)ptr; ifq->rqes = (struct io_uring_zcrx_rqe *)(ptr + off);return 0 ;
@@ -173,7 +172,10 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,static void io_free_rbuf_ring (struct io_zcrx_ifq *ifq) { -io_free_region (ifq->ctx, &ifq->ctx->zcrx_region); +if (WARN_ON_ONCE (ifq->ctx->ifq)) +return ; + +io_free_region (ifq->ctx, &ifq->region); ifq->rq_ring =NULL ; ifq->rqes =NULL ; }
@@ -334,9 +336,9 @@ struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, {lockdep_assert_held (&ctx->mmap_lock); -if (id !=0 ) +if (id !=0 || !ctx->ifq)return NULL ; -return &ctx->zcrx_region; +return &ctx->ifq->region; } int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
@@ -428,7 +430,8 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, ret = -EFAULT ; goto err; } - ctx->ifq = ifq; +scoped_guard (mutex, &ctx->mmap_lock) + ctx->ifq = ifq;return 0 ; err:io_zcrx_ifq_free (ifq);
@@ -444,7 +447,8 @@ void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)if (!ifq)return ; - ctx->ifq =NULL ; +scoped_guard (mutex, &ctx->mmap_lock) + ctx->ifq = NULL;io_zcrx_ifq_free (ifq); }
io_uring/zcrx.h +1 -0
@@ -38,6 +38,7 @@ struct io_zcrx_ifq {struct net_device *netdev;netdevice_tracker netdev_tracker;spinlock_t lock; +struct io_mapped_region region; };#if defined(CONFIG_IO_URING_ZCRX )
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH v2 6/6 io_uring/zcrx: add support for multiple ifqs
In reply to Pavel BegunkovAllow the user to register multiple ifqs / zcrx contexts. With that we can use multiple interfaces / interface queues in a single io_uring instance.
--- include/linux/io_uring_types.h | 5 +-- io_uring/io_uring.c | 3 +- io_uring/net.c | 5 +-- io_uring/zcrx.c | 72 +++++++++++++++++++++++----------- 4 files changed, 55 insertions(+), 30 deletions(-)
include/linux/io_uring_types.h +2 -3
@@ -40,8 +40,6 @@ enum io_uring_cmd_flags { IO_URING_F_TASK_DEAD = (1 << 13), }; -struct io_zcrx_ifq ; -struct io_wq_work_node {struct io_wq_work_node *next; };
@@ -394,7 +392,8 @@ struct io_ring_ctx {struct wait_queue_head poll_wq;struct io_restriction restrictions; -struct io_zcrx_ifq *ifq; +/* Stores zcrx object pointers of type struct io_zcrx_ifq */ +struct xarray zcrx_ctxs;u32 pers_next;struct xarray personalities;
io_uring/io_uring.c +2 -1
@@ -359,6 +359,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->tctx_list); ctx->submit_state.free_list.next =NULL ;INIT_HLIST_HEAD (&ctx->waitid_list); +xa_init_flags (&ctx->zcrx_ctxs,XA_FLAGS_ALLOC );#ifdef CONFIG_FUTEX INIT_HLIST_HEAD (&ctx->futex_list);#endif
@@ -2888,7 +2889,7 @@ static __cold void io_ring_exit_work(struct work_struct *work)io_cqring_overflow_kill (ctx);mutex_unlock (&ctx->uring_lock); } -if (ctx->ifq) { +if (!xa_empty (&ctx->zcrx_ctxs)) {mutex_lock (&ctx->uring_lock);io_shutdown_zcrx_ifqs (ctx);mutex_unlock (&ctx->uring_lock);
io_uring/net.c +2 -3
@@ -1189,11 +1189,10 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -EINVAL ; ifq_idx =READ_ONCE (sqe->zcrx_ifq_idx); -if (ifq_idx !=0 ) -return -EINVAL ; - zc->ifq = req->ctx->ifq; + zc->ifq =xa_load (&req->ctx->zcrx_ctxs, ifq_idx);if (!zc->ifq)return -EINVAL ; + zc->len =READ_ONCE (sqe->len); zc->flags =READ_ONCE (sqe->ioprio); zc->msg_flags =READ_ONCE (sqe->msg_flags);
io_uring/zcrx.c +49 -23
149 lines not on this page · raw message
Signature · 1 line
2.48.1
Quoted 8 lines · Pavel Begunkov
Note: depends on patches queued for 6.15-rcN.
Patches 3-5 allow to register multiple ifqs within a single
io_uringinstance. That should be useful for setups with multiple interfaces.Patch 1 and 2 and not related but I just bundled them together.
[...]
Applied, thanks!
[1/6] io_uring/zcrx: remove duplicated freelist init
commit: 37d26edd6bb4984849a71e21c6824c961fcd19db
[2/6] io_uring/zcrx: move io_zcrx_iov_page
commit: a79154ae5df9e21dbacb1eb77fad984fd4c45cca
[3/6] io_uring/zcrx: remove sqe->file_index check
commit: 59bc1ab922bbb36558292c204e56ab951e833384
[4/6] io_uring/zcrx: let zcrx choose region for mmaping
commit: 77231d4e46555d30289b1909c2a2f26bcf00f08c
[5/6] io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
commit: 632b3186726984319e2337987de86a442407f30e
[6/6] io_uring/zcrx: add support for multiple ifqs
commit: 9c2a1c50844265152b7011599a1a9dfe473d1f51Best regards,
Signature · 1 line
Jens Axboe