add support for multiple ifqs per io_uring
Subsystems (2)
Pavel Begunkov
maintainercover add support for multiple ifqs per io_uring
Note: depends on patches queued for 6.15-rcN.
Patches 3-5 allow to register multiple ifqs within a single io_uring instance. That should be useful for setups with multiple interfaces.
Patch 1 and 2 and not related but I just bundled them together.
Pavel Begunkov (5):
io_uring/zcrx: remove duplicated freelist init io_uring/zcrx: move io_zcrx_iov_page io_uring/zcrx: let zcrx choose region for mmaping io_uring/zcrx: move zcrx region to struct io_zcrx_ifq io_uring/zcrx: add support for multiple ifqs
include/linux/io_uring_types.h | 7 +-- io_uring/io_uring.c | 3 +- io_uring/memmap.c | 10 ++-- io_uring/net.c | 8 ++- io_uring/zcrx.c | 90 +++++++++++++++++++++------------- io_uring/zcrx.h | 8 +++ 6 files changed, 78 insertions(+), 48 deletions(-)
Signature · 1 line
2.48.1
Pavel Begunkov
maintainerPATCH 1/5 io_uring/zcrx: remove duplicated freelist init
Several lines below we already initialise the freelist, don't do it twice.
--- io_uring/zcrx.c | 3 --- 1 file changed, 3 deletions(-)
io_uring/zcrx.c +0 -3
@@ -238,9 +238,6 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,if (!area->freelist) goto err; -for (i =0 ; i < nr_iovs; i++) - area->freelist[i] = i; - area->user_refs =kvmalloc_array (nr_iovs,sizeof (area->user_refs[0 ]),GFP_KERNEL | __GFP_ZERO);if (!area->user_refs)
Signature · 1 line
2.48.1
Re: [PATCH 1/5] io_uring/zcrx: remove duplicated freelist init
Quoted 13 lines · Pavel Begunkov
Several lines below we already initialise the freelist, don't do it twice.
Signed-off-by Pavel Begunkov <[email protected]>--- io_uring/zcrx.c | 3 --- 1 file changed, 3 deletions(-)io_uring/zcrx.c +0
-0
io_uring/zcrx.c static int io_zcrx_create_area(struct io_zcrx_ifq *ifq, · line 243
Quoted 9 lines
io_uring/zcrx.c +0
-3@@ -238,9 +238,6 @@ static int io_zcrx_create_area(struct io_zcrx_ifq *ifq,if (!area->freelist) goto err; -for (i =0 ; i < nr_iovs; i++) - area->freelist[i] = i; - area->user_refs =kvmalloc_array (nr_iovs,sizeof (area->user_refs[0 ]),GFP_KERNEL | __GFP_ZERO);if (!area->user_refs)
Pavel Begunkov
maintainerPATCH 2/5 io_uring/zcrx: move io_zcrx_iov_page
In reply to Pavel BegunkovWe'll need io_zcrx_iov_page at the top to keep offset calculations closer together, move it there.
--- io_uring/zcrx.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-)
io_uring/zcrx.c +14 -14
@@ -31,6 +31,20 @@ static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)return pp->mp_priv; } +static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) +{ +struct net_iov_area *owner =net_iov_owner (niov); + + return container_of(owner, struct io_zcrx_area, nia); +} + +static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) +{ +struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); + +return area->pages[net_iov_idx (niov)]; +} +#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
@@ -111,13 +125,6 @@ struct io_zcrx_args {static const struct memory_provider_ops io_uring_pp_zc_ops; -static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) -{ -struct net_iov_area *owner =net_iov_owner (niov); - - return container_of(owner, struct io_zcrx_area, nia); -} -static inline atomic_t *io_get_user_counter (struct net_iov *niov) {struct io_zcrx_area *area =io_zcrx_iov_to_area (niov);
@@ -140,13 +147,6 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)atomic_inc (io_get_user_counter (niov)); } -static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) -{ -struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); - -return area->pages[net_iov_idx (niov)]; -} -static int io_allocate_rbuf_ring (struct io_zcrx_ifq *ifq,struct io_uring_zcrx_ifq_reg *reg,struct io_uring_region_desc *rd)
Signature · 1 line
2.48.1
Re: [PATCH 2/5] io_uring/zcrx: move io_zcrx_iov_page
Quoted 51 lines · Pavel Begunkov
We'll need
io_zcrx_iov_pageat the top to keep offset calculations closer together, move it there.Signed-off-by Pavel Begunkov <[email protected]>--- io_uring/zcrx.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-)io_uring/zcrx.c +14
-7@@ -31,6 +31,20 @@ static inline struct io_zcrx_ifq *io_pp_to_ifq(struct page_pool *pp)return pp->mp_priv; } +static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) +{ +struct net_iov_area *owner =net_iov_owner (niov); + + return container_of(owner, struct io_zcrx_area, nia); +} + +static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) +{ +struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); + +return area->pages[net_iov_idx (niov)]; +} +#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,@@ -111,13 +125,6 @@ struct io_zcrx_args {static const struct memory_provider_ops io_uring_pp_zc_ops; -static inline struct io_zcrx_area *io_zcrx_iov_to_area (const struct net_iov *niov) -{ -struct net_iov_area *owner =net_iov_owner (niov); - - return container_of(owner, struct io_zcrx_area, nia); -} -static inline atomic_t *io_get_user_counter (struct net_iov *niov) {struct io_zcrx_area *area =io_zcrx_iov_to_area (niov);@@ -140,13 +147,6 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)atomic_inc (io_get_user_counter (niov)); }
io_uring/zcrx.c static void io_zcrx_get_niov_uref(struct net_iov *niov) · line 152
Quoted 10 lines
io_uring/zcrx.c +0
-7-static inline struct page *io_zcrx_iov_page (const struct net_iov *niov) -{ -struct io_zcrx_area *area =io_zcrx_iov_to_area (niov); - -return area->pages[net_iov_idx (niov)]; -} -static int io_allocate_rbuf_ring (struct io_zcrx_ifq *ifq,struct io_uring_zcrx_ifq_reg *reg,struct io_uring_region_desc *rd)
Pavel Begunkov
maintainerPATCH 3/5 io_uring/zcrx: let zcrx choose region for mmaping
In reply to Pavel BegunkovIn preparation for adding multiple ifqs, add a helper returning a region for mmaping zcrx refill queue. For now it's trivial and returns the same ctx global ->zcrx_region.
--- io_uring/memmap.c | 10 ++++++---- io_uring/zcrx.c | 10 ++++++++++ io_uring/zcrx.h | 7 +++++++ 3 files changed, 23 insertions(+), 4 deletions(-)
io_uring/memmap.c +6 -4
@@ -13,6 +13,7 @@#include "memmap.h" #include "kbuf.h" #include "rsrc.h" +#include "zcrx.h" static void *io_mem_alloc_compound (struct page **pages,int nr_pages,size_t size,gfp_t gfp)
@@ -258,7 +259,9 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, loff_t pgoff) {loff_t offset = pgoff <<PAGE_SHIFT ; -unsigned int bgid; +unsigned int id; + + id = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ;switch (offset &IORING_OFF_MMAP_MASK ) {case IORING_OFF_SQ_RING :
@@ -267,12 +270,11 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx,case IORING_OFF_SQES :return &ctx->sq_region;case IORING_OFF_PBUF_RING : - bgid = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ; -return io_pbuf_get_region (ctx, bgid); +return io_pbuf_get_region (ctx, id);case IORING_MAP_OFF_PARAM_REGION :return &ctx->param_region;case IORING_MAP_OFF_ZCRX_REGION : -return &ctx->zcrx_region; +return io_zcrx_get_region (ctx, id); }return NULL ; }
io_uring/zcrx.c +10 -0
@@ -329,6 +329,16 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)kfree (ifq); } +struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id) +{ +lockdep_assert_held (&ctx->mmap_lock); + +if (id !=0 ) +return NULL ; +return &ctx->zcrx_region; +} +int io_register_zcrx_ifq (struct io_ring_ctx *ctx,struct io_uring_zcrx_ifq_reg __user *arg) {
io_uring/zcrx.h +7 -0
@@ -48,6 +48,8 @@ void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);int io_zcrx_recv (struct io_kiocb *req,struct io_zcrx_ifq *ifq,struct socket *sock,unsigned int flags,unsigned issue_flags,unsigned int *len); +struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id);#else static inline int io_register_zcrx_ifq (struct io_ring_ctx *ctx,struct io_uring_zcrx_ifq_reg __user *arg)
@@ -66,6 +68,11 @@ static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, {return -EOPNOTSUPP ; } +static inline struct io_mapped_region *io_zcrx_get_region (struct io_ring_ctx *ctx, +unsigned int id) +{ +return NULL ; +}#endif int io_recvzc (struct io_kiocb *req,unsigned int issue_flags);
Signature · 1 line
2.48.1
Re: [PATCH 3/5] io_uring/zcrx: let zcrx choose region for mmaping
Quoted 24 lines · Pavel Begunkov
In preparation for adding multiple ifqs, add a helper returning a region for mmaping zcrx refill queue. For now it's trivial and returns the same ctx global ->zcrx_region.
Signed-off-by Pavel Begunkov <[email protected]>--- io_uring/memmap.c | 10 ++++++---- io_uring/zcrx.c | 10 ++++++++++ io_uring/zcrx.h | 7 +++++++ 3 files changed, 23 insertions(+), 4 deletions(-)io_uring/memmap.c +1
-0@@ -13,6 +13,7 @@#include "memmap.h" #include "kbuf.h" #include "rsrc.h" +#include "zcrx.h" static void *io_mem_alloc_compound (struct page **pages,int nr_pages,size_t size,gfp_t gfp)
io_uring/memmap.c static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, · line 264
Quoted 7 lines
io_uring/memmap.c +3
-1@@ -258,7 +259,9 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, loff_t pgoff) {loff_t offset = pgoff <<PAGE_SHIFT ; -unsigned int bgid; +unsigned int id; + + id = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ;
We using the same IORING_OFF_PBUF_SHIFT?
Pavel Begunkov
maintainerRe: [PATCH 3/5] io_uring/zcrx: let zcrx choose region for mmaping
Quoted 22 lines · David Wei
Quoted 21 lines · Pavel Begunkov
In preparation for adding multiple ifqs, add a helper returning a region for mmaping zcrx refill queue. For now it's trivial and returns the same ctx global ->zcrx_region.
Signed-off-by Pavel Begunkov <[email protected]>--- io_uring/memmap.c | 10 ++++++---- io_uring/zcrx.c | 10 ++++++++++ io_uring/zcrx.h | 7 +++++++ 3 files changed, 23 insertions(+), 4 deletions(-)io_uring/memmap.c +1
-0@@ -13,6 +13,7 @@#include "memmap.h" #include "kbuf.h" #include "rsrc.h" +#include "zcrx.h"
Quoted 12 lines
io_uring/memmap.c static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, · line 264
Quoted 10 lines
io_uring/memmap.c +3
-1static void *io_mem_alloc_compound (struct page **pages,int nr_pages,size_t size,gfp_t gfp)@@ -258,7 +259,9 @@ static struct io_mapped_region *io_mmap_get_region(struct io_ring_ctx *ctx, loff_t pgoff) {loff_t offset = pgoff <<PAGE_SHIFT ; -unsigned int bgid; +unsigned int id; + + id = (offset & ~IORING_OFF_MMAP_MASK ) >>IORING_OFF_PBUF_SHIFT ;We using the same
IORING_OFF_PBUF_SHIFT?
Let me check that, and also it likely doesn't return the right value to the user as well.
Signature · 1 line
Pavel Begunkov
Pavel Begunkov
maintainerPATCH 4/5 io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
In reply to Pavel BegunkovRefill queue region is a part of zcrx and should stay in struct io_zcrx_ifq. We can't have multiple queues without it.
Note: ctx->ifq assignments are now protected by mmap_lock as it's in the mmap region look up path.
--- include/linux/io_uring_types.h | 2 -- io_uring/zcrx.c | 20 ++++++++++++-------- io_uring/zcrx.h | 1 + 3 files changed, 13 insertions(+), 10 deletions(-)
include/linux/io_uring_types.h +0 -2
@@ -448,8 +448,6 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; -/* just one zcrx per ring for now, will move to io_zcrx_ifq eventually */ -struct io_mapped_region zcrx_region; };/*
io_uring/zcrx.c +12 -8
@@ -160,12 +160,11 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,if (size > rd->size)return -EINVAL ; - ret =io_create_region_mmap_safe (ifq->ctx, &ifq->ctx->zcrx_region, rd, -IORING_MAP_OFF_ZCRX_REGION ); + ret =io_create_region (ifq->ctx, &ifq->region, rd,IORING_MAP_OFF_ZCRX_REGION );if (ret <0 )return ret; - ptr =io_region_get_ptr (&ifq->ctx->zcrx_region); + ptr =io_region_get_ptr (&ifq->region); ifq->rq_ring = (struct io_uring *)ptr; ifq->rqes = (struct io_uring_zcrx_rqe *)(ptr + off);return 0 ;
@@ -173,7 +172,10 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,static void io_free_rbuf_ring (struct io_zcrx_ifq *ifq) { -io_free_region (ifq->ctx, &ifq->ctx->zcrx_region); +if (WARN_ON_ONCE (ifq->ctx->ifq)) +return ; + +io_free_region (ifq->ctx, &ifq->region); ifq->rq_ring =NULL ; ifq->rqes =NULL ; }
@@ -334,9 +336,9 @@ struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx, {lockdep_assert_held (&ctx->mmap_lock); -if (id !=0 ) +if (id !=0 || !ctx->ifq)return NULL ; -return &ctx->zcrx_region; +return &ctx->ifq->region; } int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
@@ -428,7 +430,8 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, ret = -EFAULT ; goto err; } - ctx->ifq = ifq; +scoped_guard (mutex, &ctx->mmap_lock) + ctx->ifq = ifq;return 0 ; err:io_zcrx_ifq_free (ifq);
@@ -444,7 +447,8 @@ void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)if (!ifq)return ; - ctx->ifq =NULL ; +scoped_guard (mutex, &ctx->mmap_lock) + ctx->ifq = NULL;io_zcrx_ifq_free (ifq); }
io_uring/zcrx.h +1 -0
@@ -38,6 +38,7 @@ struct io_zcrx_ifq {struct net_device *netdev;netdevice_tracker netdev_tracker;spinlock_t lock; +struct io_mapped_region region; };#if defined(CONFIG_IO_URING_ZCRX )
Signature · 1 line
2.48.1
Re: [PATCH 4/5] io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
Quoted 31 lines · Pavel Begunkov
Refill queue region is a part of zcrx and should stay in struct
io_zcrx_ifq. We can't have multiple queues without it.Note:
ctx->ifqassignments are now protected bymmap_lockas it's in the mmap region look up path.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 2 -- io_uring/zcrx.c | 20 ++++++++++++-------- io_uring/zcrx.h | 1 + 3 files changed, 13 insertions(+), 10 deletions(-)include/linux/io_uring_types.h +0
-2@@ -448,8 +448,6 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; -/* just one zcrx per ring for now, will move to io_zcrx_ifq eventually */ -struct io_mapped_region zcrx_region; };/* io_uring/zcrx.c +0
-0
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 163
Quoted 6 lines
io_uring/zcrx.c +1
-2@@ -160,12 +160,11 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,if (size > rd->size)return -EINVAL ; - ret =io_create_region_mmap_safe (ifq->ctx, &ifq->ctx->zcrx_region, rd, -IORING_MAP_OFF_ZCRX_REGION ); + ret =io_create_region (ifq->ctx, &ifq->region, rd,IORING_MAP_OFF_ZCRX_REGION );
Why is this changed to io_create_region()? I don't see the caller of io_allocate_rbuf_ring() changing or taking mmap_lock.
Pavel Begunkov
maintainerRe: [PATCH 4/5] io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
Quoted 35 lines · David Wei
Quoted 34 lines · Pavel Begunkov
Refill queue region is a part of zcrx and should stay in struct
io_zcrx_ifq. We can't have multiple queues without it.Note:
ctx->ifqassignments are now protected bymmap_lockas it's in the mmap region look up path.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 2 -- io_uring/zcrx.c | 20 ++++++++++++-------- io_uring/zcrx.h | 1 + 3 files changed, 13 insertions(+), 10 deletions(-)include/linux/io_uring_types.h +0
-2@@ -448,8 +448,6 @@ struct io_ring_ctx {struct io_mapped_region ring_region;/* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; -/* just one zcrx per ring for now, will move to io_zcrx_ifq eventually */ -struct io_mapped_region zcrx_region; };/* io_uring/zcrx.c +0
-0@@ -160,12 +160,11 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,if (size > rd->size)return -EINVAL ;
Quoted 6 lines
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 163
Quoted 3 lines
io_uring/zcrx.c +1
-2- ret =io_create_region_mmap_safe (ifq->ctx, &ifq->ctx->zcrx_region, rd, -IORING_MAP_OFF_ZCRX_REGION ); + ret =io_create_region (ifq->ctx, &ifq->region, rd,IORING_MAP_OFF_ZCRX_REGION );Why is this changed to
io_create_region()? I don't see the caller ofio_allocate_rbuf_ring()changing or takingmmap_lock.
We only care about mmap seeing a consistent region. The mmap holds the ctx->mmap_lock, and *mmap_safe was making sure the region is updated atomically from its perspective.
Now, instead of protecting the region itself, this patch is protecting how ifq and subsequently the region is published:
- ctx->ifq = ifq;
+ scoped_guard(mutex, &ctx->mmap_lock)
+ ctx->ifq = ifq;
And io_zcrx_get_region() is either sees NULL or ifq with a correct region.
Signature · 1 line
Pavel Begunkov
David Wei
Reviewed-byRe: [PATCH 4/5] io_uring/zcrx: move zcrx region to struct io_zcrx_ifq
Quoted 48 lines · Pavel Begunkov
Quoted 31 lines · David Wei
Quoted 30 lines · Pavel Begunkov
Refill queue region is a part of zcrx and should stay in struct
io_zcrx_ifq. We can't have multiple queues without it.Note:
ctx->ifqassignments are now protected bymmap_lockas it's in the mmap region look up path.Signed-off-by Pavel Begunkov <[email protected]>---
include/linux/io_uring_types.h| 2 -- io_uring/zcrx.c | 20 ++++++++++++-------- io_uring/zcrx.h | 1 +3 files changed, 13 insertions(+), 10 deletions(-)include/linux/io_uring_types.h +0
-2@@ -448,8 +448,6 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; - /* just one zcrx per ring for now, will move to io_zcrx_ifq eventually */ - struct io_mapped_region zcrx_region; }; /*io_uring/zcrx.c +0
-0@@ -160,12 +160,11 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,Quoted 8 lines
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 163
Quoted 5 lines
io_uring/zcrx.c +1
-1if (size > rd->size) return -EINVAL; - ret = io_create_region_mmap_safe(ifq->ctx, &ifq->ctx->zcrx_region, rd, - IORING_MAP_OFF_ZCRX_REGION); + ret = io_create_region(ifq->ctx, &ifq->region, rd, IORING_MAP_OFF_ZCRX_REGION);Why is this changed to
io_create_region()? I don't see the caller ofio_allocate_rbuf_ring()changing or takingmmap_lock.We only care about mmap seeing a consistent region. The mmap holds the
ctx->mmap_lock, and*mmap_safewas making sure the region is updated atomically from its perspective.Now, instead of protecting the region itself, this patch is protecting how ifq and subsequently the region is published:
Quoted 7 lines
-
ctx->ifq= ifq;
+scoped_guard(mutex, &ctx->mmap_lock)
+ctx->ifq= ifq;And
io_zcrx_get_region()is either sees NULL or ifq with a correct region.
Ah I see, thanks, make sense.
Pavel Begunkov
maintainerPATCH 5/5 io_uring/zcrx: add support for multiple ifqs
In reply to Pavel BegunkovAllow the user to register multiple ifqs / zcrx contexts. With that we can use multiple interfaces / interface queues in a single io_uring instance.
--- include/linux/io_uring_types.h | 5 ++-- io_uring/io_uring.c | 3 +- io_uring/net.c | 8 ++--- io_uring/zcrx.c | 53 +++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 29 deletions(-)
include/linux/io_uring_types.h +2 -3
@@ -40,8 +40,6 @@ enum io_uring_cmd_flags { IO_URING_F_TASK_DEAD = (1 << 13), }; -struct io_zcrx_ifq ; -struct io_wq_work_node {struct io_wq_work_node *next; };
@@ -394,7 +392,8 @@ struct io_ring_ctx {struct wait_queue_head poll_wq;struct io_restriction restrictions; -struct io_zcrx_ifq *ifq; +/* Stores zcrx object pointers of type struct io_zcrx_ifq */ +struct xarray zcrx_ctxs;u32 pers_next;struct xarray personalities;
io_uring/io_uring.c +2 -1
@@ -359,6 +359,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->tctx_list); ctx->submit_state.free_list.next =NULL ;INIT_HLIST_HEAD (&ctx->waitid_list); +xa_init_flags (&ctx->zcrx_ctxs,XA_FLAGS_ALLOC );#ifdef CONFIG_FUTEX INIT_HLIST_HEAD (&ctx->futex_list);#endif
@@ -2888,7 +2889,7 @@ static __cold void io_ring_exit_work(struct work_struct *work)io_cqring_overflow_kill (ctx);mutex_unlock (&ctx->uring_lock); } -if (ctx->ifq) { +if (!xa_empty (&ctx->zcrx_ctxs)) {mutex_lock (&ctx->uring_lock);io_shutdown_zcrx_ifqs (ctx);mutex_unlock (&ctx->uring_lock);
io_uring/net.c +3 -5
@@ -1185,16 +1185,14 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);unsigned ifq_idx; -if (unlikely (sqe->file_index || sqe->addr2 || sqe->addr || - sqe->addr3)) +if (unlikely (sqe->addr2 || sqe->addr || sqe->addr3))return -EINVAL ; ifq_idx =READ_ONCE (sqe->zcrx_ifq_idx); -if (ifq_idx !=0 ) -return -EINVAL ; - zc->ifq = req->ctx->ifq; + zc->ifq =xa_load (&req->ctx->zcrx_ctxs, ifq_idx);if (!zc->ifq)return -EINVAL ; + zc->len =READ_ONCE (sqe->len); zc->flags =READ_ONCE (sqe->ioprio); zc->msg_flags =READ_ONCE (sqe->msg_flags);
io_uring/zcrx.c +33 -20
109 lines not on this page · raw message
Signature · 1 line
2.48.1
Re: [PATCH 5/5] io_uring/zcrx: add support for multiple ifqs
Quoted 12 lines · Pavel Begunkov
Allow the user to register multiple ifqs / zcrx contexts. With that we can use multiple interfaces / interface queues in a single
io_uringinstance.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 ++-- io_uring/io_uring.c | 3 +- io_uring/net.c | 8 ++--- io_uring/zcrx.c | 53 +++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 29 deletions(-)
[...]
io_uring/net.c int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 1189
Quoted 12 lines
io_uring/net.c +1
-2@@ -1185,16 +1185,14 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);unsigned ifq_idx; -if (unlikely (sqe->file_index || sqe->addr2 || sqe->addr || - sqe->addr3)) +if (unlikely (sqe->addr2 || sqe->addr || sqe->addr3))return -EINVAL ;
Why remove sqe->file_index?
Quoted 17 lines
io_uring/net.c +2
-3ifq_idx =READ_ONCE (sqe->zcrx_ifq_idx); -if (ifq_idx !=0 ) -return -EINVAL ; - zc->ifq = req->ctx->ifq; + zc->ifq =xa_load (&req->ctx->zcrx_ctxs, ifq_idx);if (!zc->ifq)return -EINVAL ; + zc->len =READ_ONCE (sqe->len); zc->flags =READ_ONCE (sqe->ioprio); zc->msg_flags =READ_ONCE (sqe->msg_flags);io_uring/zcrx.c +0
-0
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 177
Quoted 6 lines
io_uring/zcrx.c +0
-3@@ -172,9 +172,6 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,static void io_free_rbuf_ring (struct io_zcrx_ifq *ifq) { -if (WARN_ON_ONCE (ifq->ctx->ifq)) -return ; -
I think this should stay.
Quoted 3 lines
io_uring/zcrx.c +0
-0io_free_region (ifq->ctx, &ifq->region); ifq->rq_ring =NULL ; ifq->rqes =NULL ;
[...]
Quoted 12 lines
io_uring/zcrx.c +2
-3@@ -440,16 +443,23 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,void io_unregister_zcrx_ifqs (struct io_ring_ctx *ctx) { -struct io_zcrx_ifq *ifq = ctx->ifq; +struct io_zcrx_ifq *ifq; +unsigned long id;lockdep_assert_held (&ctx->uring_lock); -if (!ifq) -return ;
io_uring/zcrx.c int io_register_zcrx_ifq(struct io_ring_ctx *ctx, · line 460
Quoted 10 lines
io_uring/zcrx.c +10
-0+while (1 ) { + scoped_guard(mutex, &ctx->mmap_lock) { + ifq =xa_find (&ctx->zcrx_ctxs, &id,ULONG_MAX ,XA_PRESENT ); +if (ifq) +xa_erase (&ctx->zcrx_ctxs, id); + } +if (!ifq) +break ; +io_zcrx_ifq_free (ifq); + }
Why not xa_for_each()? Is it weirdness with scoped_guard macro?
Pavel Begunkov
maintainerRe: [PATCH 5/5] io_uring/zcrx: add support for multiple ifqs
Quoted 22 lines · David Wei
Quoted 12 lines · Pavel Begunkov
Allow the user to register multiple ifqs / zcrx contexts. With that we can use multiple interfaces / interface queues in a single
io_uringinstance.Signed-off-by Pavel Begunkov <[email protected]>--- include/linux/io_uring_types.h | 5 ++-- io_uring/io_uring.c | 3 +- io_uring/net.c | 8 ++--- io_uring/zcrx.c | 53 +++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 29 deletions(-)[...]
Quoted 8 lines
io_uring/net.c +0
-0@@ -1185,16 +1185,14 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc);unsigned ifq_idx;
Quoted 6 lines
io_uring/net.c int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · line 1189
Quoted 4 lines
io_uring/net.c +1
-2-if (unlikely (sqe->file_index || sqe->addr2 || sqe->addr || - sqe->addr3)) +if (unlikely (sqe->addr2 || sqe->addr || sqe->addr3))return -EINVAL ;Why remove
sqe->file_index?
it's aliased with ->zcrx_ifq_idx. The ifq_idx check below essentially does nothing. And fwiw, userspace was getting correct errors, so it's not a fix.
Quoted 17 lines
Quoted 17 lines
io_uring/net.c +2
-3ifq_idx =READ_ONCE (sqe->zcrx_ifq_idx); -if (ifq_idx !=0 ) -return -EINVAL ; - zc->ifq = req->ctx->ifq; + zc->ifq =xa_load (&req->ctx->zcrx_ctxs, ifq_idx);if (!zc->ifq)return -EINVAL ; + zc->len =READ_ONCE (sqe->len); zc->flags =READ_ONCE (sqe->ioprio); zc->msg_flags =READ_ONCE (sqe->msg_flags);io_uring/zcrx.c +0
-0@@ -172,9 +172,6 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq,
Quoted 7 lines
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 177
Quoted 5 lines
io_uring/zcrx.c +0
-3static void io_free_rbuf_ring (struct io_zcrx_ifq *ifq) { -if (WARN_ON_ONCE (ifq->ctx->ifq)) -return ; -I think this should stay.
There is not ctx->ifq anymore. You may look up in the xarray, but for that you need to know the index, and it's easier to just remove it.
Quoted 14 lines
Quoted 3 lines
io_uring/zcrx.c +0
-0io_free_region (ifq->ctx, &ifq->region); ifq->rq_ring =NULL ; ifq->rqes =NULL ;[...]
Quoted 10 lines
io_uring/zcrx.c +2
-1@@ -440,16 +443,23 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,void io_unregister_zcrx_ifqs (struct io_ring_ctx *ctx) { -struct io_zcrx_ifq *ifq = ctx->ifq; +struct io_zcrx_ifq *ifq; +unsigned long id;lockdep_assert_held (&ctx->uring_lock);
Quoted 14 lines
io_uring/zcrx.c int io_register_zcrx_ifq(struct io_ring_ctx *ctx, · line 460
Quoted 12 lines
io_uring/zcrx.c +10
-2-if (!ifq) -return ; +while (1 ) { + scoped_guard(mutex, &ctx->mmap_lock) { + ifq =xa_find (&ctx->zcrx_ctxs, &id,ULONG_MAX ,XA_PRESENT ); +if (ifq) +xa_erase (&ctx->zcrx_ctxs, id); + } +if (!ifq) +break ; +io_zcrx_ifq_free (ifq); + }Why not
xa_for_each()? Is it weirdness withscoped_guardmacro?
I don't want io_zcrx_ifq_free() to be under mmap_lock, that might complicate sync with mmap. It's good enough for now, but I'd like to have sth like this in the future:
struct xarray tmp_onstack_xarray;
scoped_guard(mutex, &ctx->mmap_lock)
xarray_swap(&tmp_onstack_xarray, &ctx->zcrx); for_each_xarray(tmp_onstack_xarray) io_zcrx_ifq_free();
but there is no xarray_swap AFAIK.
Signature · 1 line
Pavel Begunkov
David Wei
Reviewed-byRe: [PATCH 5/5] io_uring/zcrx: add support for multiple ifqs
Quoted 29 lines · Pavel Begunkov
Quoted 14 lines · David Wei
Quoted 12 lines · Pavel Begunkov
Allow the user to register multiple ifqs / zcrx contexts. With that we can use multiple interfaces / interface queues in a single
io_uringinstance.Signed-off-by Pavel Begunkov <[email protected]>---
include/linux/io_uring_types.h| 5 ++-- io_uring/io_uring.c | 3 +-
io_uring/net.c | 8 ++---
io_uring/zcrx.c | 53 +++++++++++++++++++++-------------4 files changed, 40 insertions(+), 29 deletions(-)[...]
Quoted 13 lines
Quoted 11 lines
io_uring/net.c +0
-0@@ -1185,16 +1185,14 @@ int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); unsigned ifq_idx;- if (
unlikely(sqe->file_index || sqe->addr2 || sqe->addr || - sqe->addr3))
+ if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3)) return-EINVAL;Why remove
sqe->file_index?
Quoted 4 lines
it's aliased with ->zcrx_ifq_idx. The
ifq_idxcheck below essentially does nothing. And fwiw, userspace was getting correct errors, so it's not a fix.
Got it.
Quoted 24 lines
Quoted 15 lines
Quoted 15 lines
ifq_idx=READ_ONCE(sqe->zcrx_ifq_idx); - if (ifq_idx!= 0)
- return-EINVAL;
-zc->ifq=req->ctx->ifq;
+zc->ifq=xa_load(&req->ctx->zcrx_ctxs, ifq_idx); if (!zc->ifq)
return-EINVAL;
+
zc->len=READ_ONCE(sqe->len);zc->flags=READ_ONCE(sqe->ioprio);zc->msg_flags=READ_ONCE(sqe->msg_flags);io_uring/zcrx.c +0
-0Quoted 8 lines
io_uring/zcrx.c static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, · line 176
Quoted 6 lines
io_uring/zcrx.c +0
-3@@ -172,9 +172,6 @@ static int io_allocate_rbuf_ring(struct io_zcrx_ifq *ifq, static void io_free_rbuf_ring(struct io_zcrx_ifq *ifq) { - if (WARN_ON_ONCE(ifq->ctx->ifq)) - return; -I think this should stay.
Quoted 3 lines
There is not
ctx->ifqanymore. You may look up in the xarray, but for that you need to know the index, and it's easier to just remove it.
Yeah, and it's rather defensive to check.
Quoted 32 lines
Quoted 17 lines
Quoted 3 lines
io_uring/zcrx.c +0
-0io_free_region(ifq->ctx, &ifq->region); ifq->rq_ring = NULL; ifq->rqes = NULL;[...]
Quoted 13 lines
io_uring/zcrx.c +6
-2@@ -440,16 +443,23 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx, void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) { - struct io_zcrx_ifq *ifq = ctx->ifq; + struct io_zcrx_ifq *ifq; + unsigned long id; lockdep_assert_held(&ctx->uring_lock); - if (!ifq) - return; + while (1) { + scoped_guard(mutex, &ctx->mmap_lock) { + ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); + if (ifq)Quoted 8 lines
io_uring/zcrx.c int io_register_zcrx_ifq(struct io_ring_ctx *ctx, · line 460
Quoted 6 lines
io_uring/zcrx.c +6
-0+ xa_erase(&ctx->zcrx_ctxs, id); + } + if (!ifq) + break; + io_zcrx_ifq_free(ifq); + }Why not
xa_for_each()? Is it weirdness withscoped_guardmacro?I don't want
io_zcrx_ifq_free()to be undermmap_lock, that might complicate sync with mmap. It's good enough for now, but I'd like to have sth like this in the future:
Quoted 9 lines
struct xarray
tmp_onstack_xarray;
scoped_guard(mutex, &ctx->mmap_lock)
xarray_swap(&tmp_onstack_xarray,&ctx->zcrx);for_each_xarray(tmp_onstack_xarray)
io_zcrx_ifq_free();but there is no
xarray_swapAFAIK.