Add alloc cache for sendmsg/recvmsg
Jens Axboe
maintainerPATCH 1/1 Add alloc cache for sendmsg/recvmsg
Hi,
This abstracts out the alloc cache we have for apoll entries, and extends it to be usable for recv/sendmsg as well. During that abstraction, impose an upper limit for cached entries as well.
This yields a 4-5% performance increase running netbench using sendmsg/recvmsg rather than plan send and recv.
Post 5.20, I suspect we can get rid of using io_async_msghdr for single vector sendmsg/recvmsg, which will make this less relevant. But as this work isn't done yet, and the support for eg ITER_UBUF isn't upstream yet either, this can help fill the gap.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io_uring: move apoll cache to poll.c
This is where it's used, move the flush handler in there.
--- io_uring/io_uring.c | 12 ------------ io_uring/poll.c | 12 ++++++++++++ io_uring/poll.h | 2 ++ 3 files changed, 14 insertions(+), 12 deletions(-)
io_uring/io_uring.c +0 -12
@@ -2445,18 +2445,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx)mutex_unlock (&ctx->uring_lock); } -static void io_flush_apoll_cache (struct io_ring_ctx *ctx) -{ -struct async_poll *apoll; - -while (!list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del (&apoll->poll.wait.entry); -kfree (apoll); - } -} -static __cold void io_ring_ctx_free (struct io_ring_ctx *ctx) {io_sq_thread_finish (ctx);
io_uring/poll.c +12 -0
@@ -958,3 +958,15 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)io_req_set_res (req, ret,0 );return IOU_OK ; } + +void io_flush_apoll_cache (struct io_ring_ctx *ctx) +{ +struct async_poll *apoll; + +while (!list_empty (&ctx->apoll_cache)) { + apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, + poll.wait.entry); +list_del (&apoll->poll.wait.entry); +kfree (apoll); + } +}
io_uring/poll.h +2 -0
@@ -30,3 +30,5 @@ int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,int io_arm_poll_handler (struct io_kiocb *req,unsigned issue_flags);bool io_poll_remove_all (struct io_ring_ctx *ctx,struct task_struct *tsk,bool cancel_all); + +void io_flush_apoll_cache (struct io_ring_ctx *ctx);
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 2/4 io_uring: add abstraction around apoll cache
In reply to Jens AxboeIn preparation for adding limits, and one more user, abstract out the core bits of the allocation+free cache.
--- include/linux/io_uring_types.h | 6 +++++- io_uring/alloc_cache.h | 4 ++++ io_uring/io_uring.c | 7 ++++--- io_uring/poll.c | 16 ++++++++-------- io_uring/poll.h | 5 ++++- 5 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 io_uring/alloc_cache.h
include/linux/io_uring_types.h +5 -1
@@ -158,6 +158,10 @@ struct io_ev_fd {struct rcu_head rcu; }; +struct io_alloc_cache { +struct hlist_head list; +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {
@@ -216,7 +220,7 @@ struct io_ring_ctx {struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list; -struct list_head apoll_cache; +struct io_alloc_cache apoll_cache;struct xarray personalities;u32 pers_next; }____cacheline_aligned_in_smp ;
io_uring/alloc_cache.h new file +4 -0
@@ -0,0 +1,4 @@ +static inline void io_alloc_cache_init (struct io_alloc_cache *cache) +{ +INIT_HLIST_HEAD (&cache->list); +}
io_uring/io_uring.c +4 -3
@@ -92,6 +92,7 @@#include "timeout.h" #include "poll.h" +#include "alloc_cache.h" #define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
@@ -295,7 +296,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->sqd_list);INIT_LIST_HEAD (&ctx->cq_overflow_list);INIT_LIST_HEAD (&ctx->io_buffers_cache); -INIT_LIST_HEAD (&ctx->apoll_cache); +io_alloc_cache_init (&ctx->apoll_cache);init_completion (&ctx->ref_comp);xa_init_flags (&ctx->personalities,XA_FLAGS_ALLOC1 );mutex_init (&ctx->uring_lock);
@@ -1180,8 +1181,8 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)if (apoll->double_poll)kfree (apoll->double_poll); -list_add (&apoll->poll.wait.entry, - &ctx->apoll_cache); +hlist_add_head (&apoll->cache_list, + &ctx->apoll_cache.list); req->flags &= ~REQ_F_POLLED ; }if (req->flags &IO_REQ_LINK_FLAGS )
io_uring/poll.c +8 -8
@@ -589,10 +589,10 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, apoll = req->apoll;kfree (apoll->double_poll); }else if (!(issue_flags &IO_URING_F_UNLOCKED ) && - !list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del_init (&apoll->poll.wait.entry); + !hlist_empty (&ctx->apoll_cache.list)) { + apoll =hlist_entry (ctx->apoll_cache.list.first, + struct async_poll, cache_list); +hlist_del (&apoll->cache_list); }else { apoll =kmalloc (sizeof (*apoll),GFP_ATOMIC );if (unlikely (!apoll))
@@ -963,10 +963,10 @@ void io_flush_apoll_cache(struct io_ring_ctx *ctx) {struct async_poll *apoll; -while (!list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del (&apoll->poll.wait.entry); +while (!hlist_empty (&ctx->apoll_cache.list)) { + apoll =hlist_entry (ctx->apoll_cache.list.first, + struct async_poll, cache_list); +hlist_del (&apoll->cache_list);kfree (apoll); } }
io_uring/poll.h +4 -1
@@ -14,7 +14,10 @@ struct io_poll { };struct async_poll { -struct io_poll poll; +union { +struct io_poll poll; +struct hlist_node cache_list; + };struct io_poll *double_poll; };
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 3/4 io_uring: impose max limit on apoll cache
In reply to Jens AxboeCaches like this tend to grow to the peak size, and then never get any smaller. Impose a max limit on the size, to prevent it from growing too big.
A somewhat randomly chosen 512 is the max size we'll allow the cache to get. If a batch of frees come in and would bring it over that, we simply start kfree'ing the surplus.
--- include/linux/io_uring_types.h | 1 + io_uring/alloc_cache.h | 15 +++++++++++++++ io_uring/io_uring.c | 8 ++++++-- io_uring/poll.c | 2 ++ 4 files changed, 24 insertions(+), 2 deletions(-)
include/linux/io_uring_types.h +1 -0
@@ -160,6 +160,7 @@ struct io_ev_fd {struct io_alloc_cache {struct hlist_head list; +unsigned int nr_cached; };struct io_ring_ctx {
io_uring/alloc_cache.h +15 -0
@@ -1,4 +1,19 @@ +/* +* Don't allow the cache to grow beyond this size. +*/ +#define IO_ALLOC_CACHE_MAX 512 + +static inline bool io_alloc_cache_store (struct io_alloc_cache *cache) +{ +if (cache->nr_cached <IO_ALLOC_CACHE_MAX ) { + cache->nr_cached++; +return true; + } +return false; +} +static inline void io_alloc_cache_init (struct io_alloc_cache *cache) {INIT_HLIST_HEAD (&cache->list); + cache->nr_cached =0 ; }
io_uring/io_uring.c +6 -2
@@ -1181,8 +1181,12 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)if (apoll->double_poll)kfree (apoll->double_poll); -hlist_add_head (&apoll->cache_list, - &ctx->apoll_cache.list); +if (io_alloc_cache_store (&ctx->apoll_cache)) { +hlist_add_head (&apoll->cache_list, + &ctx->apoll_cache.list); + }else { +kfree (apoll); + } req->flags &= ~REQ_F_POLLED ; }if (req->flags &IO_REQ_LINK_FLAGS )
io_uring/poll.c +2 -0
@@ -593,6 +593,7 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, apoll =hlist_entry (ctx->apoll_cache.list.first, struct async_poll, cache_list);hlist_del (&apoll->cache_list); + ctx->apoll_cache.nr_cached--; }else { apoll =kmalloc (sizeof (*apoll),GFP_ATOMIC );if (unlikely (!apoll))
@@ -969,4 +970,5 @@ void io_flush_apoll_cache(struct io_ring_ctx *ctx)hlist_del (&apoll->cache_list);kfree (apoll); } + ctx->apoll_cache.nr_cached =0 ; }
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 4/4 io_uring: add netmsg cache
In reply to Jens AxboeFor recvmsg/sendmsg, if they don't complete inline, we currently need to allocate a struct io_async_msghdr for each request. This is a somewhat large struct.
Hook up sendmsg/recvmsg to use the io_alloc_cache. This reduces the alloc + free overhead considerably, yielding 4-5% of extra performance running netbench.
--- include/linux/io_uring_types.h | 6 ++- io_uring/io_uring.c | 3 ++ io_uring/net.c | 73 +++++++++++++++++++++++++++++----- io_uring/net.h | 11 ++++- 4 files changed, 81 insertions(+), 12 deletions(-)
include/linux/io_uring_types.h +4 -2
@@ -222,8 +222,7 @@ struct io_ring_ctx {struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list;struct io_alloc_cache apoll_cache; -struct xarray personalities; -u32 pers_next; +struct io_alloc_cache netmsg_cache; }____cacheline_aligned_in_smp ;/* IRQ completion list, under ->completion_lock */
@@ -241,6 +240,9 @@ struct io_ring_ctx {unsigned int file_alloc_start;unsigned int file_alloc_end; +struct xarray personalities; +u32 pers_next; +struct {/* * We cache a range of free CQEs we can use, once exhausted it
io_uring/io_uring.c +3 -0
@@ -89,6 +89,7 @@#include "kbuf.h" #include "rsrc.h" #include "cancel.h" +#include "net.h" #include "timeout.h" #include "poll.h"
@@ -297,6 +298,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->cq_overflow_list);INIT_LIST_HEAD (&ctx->io_buffers_cache);io_alloc_cache_init (&ctx->apoll_cache); +io_alloc_cache_init (&ctx->netmsg_cache);init_completion (&ctx->ref_comp);xa_init_flags (&ctx->personalities,XA_FLAGS_ALLOC1 );mutex_init (&ctx->uring_lock);
@@ -2473,6 +2475,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)__io_cqring_overflow_flush (ctx, true);io_eventfd_unregister (ctx);io_flush_apoll_cache (ctx); +io_flush_netmsg_cache (ctx);mutex_unlock (&ctx->uring_lock);io_destroy_buffers (ctx);if (ctx->sq_creds)
io_uring/net.c +64 -9
144 lines not on this page · raw message
io_uring/net.h +10 -1
@@ -5,7 +5,10 @@ #if defined(CONFIG_NET) struct io_async_msghdr { - struct iovec fast_iov[UIO_FASTIOV]; + union { + struct iovec fast_iov[UIO_FASTIOV]; + struct hlist_node cache_list; + }; /* points to an allocated iov, if NULL we use fast_iov instead */ struct iovec *free_iov; struct sockaddr __user *uaddr;
@@ -40,4 +43,10 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags);int io_connect_prep_async (struct io_kiocb *req);int io_connect_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_connect (struct io_kiocb *req,unsigned int issue_flags); + +void io_flush_netmsg_cache (struct io_ring_ctx *ctx); +#else +static inline void io_flush_netmsg_cache (struct io_ring_ctx *ctx) +{ +}#endif
Signature · 1 line
2.35.1
Re: [PATCH 4/4] io_uring: add netmsg cache
Quoted 102 lines · Jens Axboe
For recvmsg/sendmsg, if they don't complete inline, we currently need to allocate a struct
io_async_msghdrfor each request. This is a somewhat large struct.Hook up sendmsg/recvmsg to use the
io_alloc_cache. This reduces the alloc + free overhead considerably, yielding 4-5% of extra performance
running netbench.Signed-off-by Jens Axboe <[email protected]>---
include/linux/io_uring_types.h | 6 ++- io_uring/io_uring.c | 3 ++
io_uring/net.c | 73 +++++++++++++++++++++++++++++--- --
io_uring/net.h | 11 ++++-4 files changed, 81 insertions(+), 12 deletions(-)include/linux/io_uring_types.h +1
-2@@ -222,8 +222,7 @@ struct io_ring_ctx { struct io_hash_table cancel_table_locked; struct list_head cq_overflow_list; struct io_alloc_cache apoll_cache; - struct xarray personalities; - u32 pers_next; + struct io_alloc_cache netmsg_cache; } ____cacheline_aligned_in_smp;
/* IRQ completion list, under ->completion_lock */include/linux/io_uring_types.h +0
-0@@ -241,6 +240,9 @@ struct io_ring_ctx { unsigned int file_alloc_start; unsigned int file_alloc_end;
+ struct xarray personalities;
+ u32 pers_next;
+
struct {
/*
* We cache a range of free CQEs we can use, once exhausted itio_uring/io_uring.c +1
-0@@ -89,6 +89,7 @@ #include "kbuf.h" #include "rsrc.h" #include "cancel.h" +#include "net.h"
#include "timeout.h"
#include "poll.h"io_uring/io_uring.c +1
-0@@ -297,6 +298,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->cq_overflow_list); INIT_LIST_HEAD(&ctx->io_buffers_cache); io_alloc_cache_init(&ctx->apoll_cache); + io_alloc_cache_init(&ctx->netmsg_cache); init_completion(&ctx->ref_comp); xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);mutex_init(&ctx->uring_lock);
io_uring/io_uring.c +1
-0@@ -2473,6 +2475,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) __io_cqring_overflow_flush(ctx, true); io_eventfd_unregister(ctx); io_flush_apoll_cache(ctx); + io_flush_netmsg_cache(ctx); mutex_unlock(&ctx->uring_lock); io_destroy_buffers(ctx);if (
ctx->sq_creds)io_uring/net.c +0
-0@@ -12,6 +12,7 @@
#include "io_uring.h"
#include "kbuf.h"
+#include "alloc_cache.h"
#include "net.h"
#ifdefined(CONFIG_NET)io_uring/net.c +0
-0@@ -97,18 +98,57 @@ static bool io_net_retry(struct socket *sock, int flags)return
sock->type==SOCK_STREAM||sock->type==SOCK_SEQPACKET;
}
+static voidio_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)io_uring/net.c +8
-0+{ + struct io_async_msghdr *hdr = req->async_data; + + if (!hdr || issue_flags & IO_URING_F_UNLOCKED) + return; + + if (io_alloc_cache_store(&req->ctx->netmsg_cache)) { + hlist_add_head(&hdr->cache_list, &req->ctx-
Quoted 1 line
Quoted 1 line
netmsg_cache.list);
can io_alloc_cache_store just do the store?
would be nicer to have cache::list be generally unused outside of the cache code.
Quoted 18 lines
+ req->async_data = NULL;
+ req->flags &= ~REQ_F_ASYNC_DATA;
+ }io_uring/net.c +3
-0+} + +static struct io_async_msghdr *io_recvmsg_alloc_async(struct
io_kiocb*req,
+ unsigned intissue_flags)io_uring/net.c +9
-0+{ + struct io_ring_ctx *ctx = req->ctx; + + if (!(issue_flags & IO_URING_F_UNLOCKED) && + !hlist_empty(&ctx->netmsg_cache.list)) { + struct io_async_msghdr *hdr; + + hdr = hlist_entry(ctx->netmsg_cache.list.first, + struct io_async_msghdr,
Quoted 3 lines
cache_list);
+ ctx->netmsg_cache.nr_cached--;
+ hlist_del(&hdr->cache_list);
ditto here. I think all the hlist stuff and the nr_cached manipulation can be wrapped up
Quoted 122 lines
+ req->flags |=
REQ_F_ASYNC_DATA;
+ req->async_data = hdr;
+ return hdr;
+ }io_uring/net.c +7
-0+ + if (!io_alloc_async_data(req)) + return req->async_data; + + return NULL; +} +static int io_setup_async_msg(struct
io_kiocb*req, - structio_async_msghdr*kmsg) + structio_async_msghdr*kmsg, + unsigned intissue_flags) {
structio_async_msghdr*async_msg=req->async_data;
if (async_msg)
return-EAGAIN;
- if (io_alloc_async_data(req)) {
+ async_msg =io_recvmsg_alloc_async(req, issue_flags); + if (!async_msg) {
kfree(kmsg->free_iov);
return-ENOMEM;
}
- async_msg =req->async_data;
req->flags |=REQ_F_NEED_CLEANUP;memcpy(async_msg, kmsg, sizeof(*kmsg));async_msg->msg.msg_name=&async_msg->addr;io_uring/net.c +0
-0@@ -195,7 +235,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
if (!(req->flags &REQ_F_POLLED) &&
(sr->flags&IORING_RECVSEND_POLL_FIRST))
- returnio_setup_async_msg(req, kmsg);
+ returnio_setup_async_msg(req, kmsg, issue_flags);
flags =sr->msg_flags;
if (issue_flags&IO_URING_F_NONBLOCK)io_uring/net.c +0
-0@@ -207,13 +247,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
if (ret <min_ret) {
if (ret ==-EAGAIN&& (issue_flags&IO_URING_F_NONBLOCK))
- returnio_setup_async_msg(req, kmsg); + returnio_setup_async_msg(req, kmsg, issue_flags);
if (ret ==-ERESTARTSYS)
ret =-EINTR;
if (ret > 0 &&io_net_retry(sock, flags)) {sr->done_io+= ret;req->flags|=REQ_F_PARTIAL_IO; - returnio_setup_async_msg(req, kmsg); + returnio_setup_async_msg(req, kmsg, issue_flags);
}
req_set_fail(req);
}io_uring/net.c +1
-0@@ -221,6 +261,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (kmsg->free_iov) kfree(kmsg->free_iov); req->flags &= ~REQ_F_NEED_CLEANUP; + io_netmsg_recycle(req, issue_flags); if (ret >= 0) ret += sr->done_io;else if (
sr->done_io)io_uring/net.c +0
-0@@ -495,7 +536,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
if (!(req->flags &REQ_F_POLLED) &&
(sr->flags&IORING_RECVSEND_POLL_FIRST))
- returnio_setup_async_msg(req, kmsg);
+ returnio_setup_async_msg(req, kmsg, issue_flags);
if (io_do_buffer_select(req)) {
void__user*buf;io_uring/net.c +0
-0@@ -519,13 +560,13 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)ret = __sys_recvmsg_sock(sock,
&kmsg->msg,sr->umsg, kmsg-Quoted 1 line
uaddr, flags);
if (ret <
min_ret) {
if (ret ==-EAGAIN&&force_nonblock)
- returnio_setup_async_msg(req, kmsg); + returnio_setup_async_msg(req, kmsg, issue_flags);
if (ret ==-ERESTARTSYS)
ret =-EINTR;
if (ret > 0 &&io_net_retry(sock, flags)) {sr->done_io+= ret;
req->flags |=REQ_F_PARTIAL_IO; - returnio_setup_async_msg(req, kmsg); + returnio_setup_async_msg(req, kmsg, issue_flags);
}
req_set_fail(req);
} else if ((flags &MSG_WAITALL) && (kmsg->msg.msg_flags& (MSG_TRUNC|MSG_CTRUNC))) {io_uring/net.c +1
-0@@ -535,6 +576,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) /* fast path, check for non-NULL to avoid function call */ if (kmsg->free_iov) kfree(kmsg->free_iov); + io_netmsg_recycle(req, issue_flags); req->flags &= ~REQ_F_NEED_CLEANUP; if (ret > 0)ret +=
sr->done_io;io_uring/net.c +8
-0@@ -848,4 +890,17 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) io_req_set_res(req, ret, 0); return IOU_OK; } + +void io_flush_netmsg_cache(struct io_ring_ctx *ctx) +{ + while (!hlist_empty(&ctx->netmsg_cache.list)) { + struct io_async_msghdr *hdr; + + hdr = hlist_entry(ctx->netmsg_cache.list.first, + struct io_async_msghdr,
Quoted 6 lines
cache_list);
+ hlist_del(&hdr->cache_list);
+ kfree(hdr);
+ }
+ ctx->netmsg_cache.nr_cached = 0;
+}
again - could be put somewhere common. I assume there will not be much more cleanup than simple kfree
Quoted 31 lines
#endif
io_uring/net.h +0
-0@@ -5,7 +5,10 @@
#ifdefined(CONFIG_NET)
structio_async_msghdr{
- struct iovec fast_iov[UIO_FASTIOV]; + union {
+ struct iovec fast_iov[UIO_FASTIOV]; + struct hlist_node cache_list; + };
/* points to an allocated iov, if NULL we usefast_iovinstead */
struct iovec *free_iov; struct sockaddr __user *uaddr;io_uring/net.h +0
-0@@ -40,4 +43,10 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags); int io_connect_prep_async(struct io_kiocb *req);int
io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
intio_connect(struct io_kiocb *req, unsigned int issue_flags);io_uring/net.h +6
-0+ +void io_flush_netmsg_cache (struct io_ring_ctx *ctx); +#else +static inline void io_flush_netmsg_cache (struct io_ring_ctx *ctx) +{ +}#endif
Jens Axboe
maintainerRe: [PATCH 4/4] io_uring: add netmsg cache
Quoted 103 lines · Dylan Yudaken
Quoted 102 lines · Jens Axboe
For recvmsg/sendmsg, if they don't complete inline, we currently need to allocate a struct
io_async_msghdrfor each request. This is a somewhat large struct.Hook up sendmsg/recvmsg to use the
io_alloc_cache. This reduces the alloc + free overhead considerably, yielding 4-5% of extra performance
running netbench.Signed-off-by Jens Axboe <[email protected]>--- include/linux/io_uring_types.h | 6 ++- io_uring/io_uring.c | 3 ++ io_uring/net.c | 73 +++++++++++++++++++++++++++++--- -- io_uring/net.h | 11 ++++- 4 files changed, 81 insertions(+), 12 deletions(-)include/linux/io_uring_types.h +4
-2@@ -222,8 +222,7 @@ struct io_ring_ctx {struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list;struct io_alloc_cache apoll_cache; -struct xarray personalities; -u32 pers_next; +struct io_alloc_cache netmsg_cache; }____cacheline_aligned_in_smp ;/* IRQ completion list, under ->completion_lock */ @@ -241,6 +240,9 @@ struct io_ring_ctx {unsigned int file_alloc_start;unsigned int file_alloc_end; +struct xarray personalities; +u32 pers_next; +struct {/* * We cache a range of free CQEs we can use, once exhausted it
io_uring/io_uring.c +2
-0@@ -89,6 +89,7 @@#include "kbuf.h" #include "rsrc.h" #include "cancel.h" +#include "net.h" #include "timeout.h" #include "poll.h" @@ -297,6 +298,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->cq_overflow_list);INIT_LIST_HEAD (&ctx->io_buffers_cache);io_alloc_cache_init (&ctx->apoll_cache); +io_alloc_cache_init (&ctx->netmsg_cache);init_completion (&ctx->ref_comp);xa_init_flags (&ctx->personalities,XA_FLAGS_ALLOC1 );mutex_init(&ctx->uring_lock);io_uring/io_uring.c +1
-0@@ -2473,6 +2475,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)__io_cqring_overflow_flush (ctx ,true );io_eventfd_unregister (ctx);io_flush_apoll_cache (ctx); +io_flush_netmsg_cache (ctx);mutex_unlock (&ctx->uring_lock);io_destroy_buffers (ctx);if (ctx->sq_creds)io_uring/net.c +10
-0@@ -12,6 +12,7 @@#include "io_uring.h" #include "kbuf.h" +#include "alloc_cache.h" #include "net.h" #if defined(CONFIG_NET )@@ -97,18 +98,57 @@ static bool io_net_retry(struct socket *sock, int flags)return sock->type ==SOCK_STREAM || sock->type ==SOCK_SEQPACKET ; } +static void io_netmsg_recycle (struct io_kiocb *req,unsigned int issue_flags) +{ +struct io_async_msghdr *hdr = req->async_data; + +if (!hdr || issue_flags &IO_URING_F_UNLOCKED ) +return ; + +if (io_alloc_cache_store (&req->ctx->netmsg_cache)) { + hlist_add_head(&hdr->cache_list, &req->ctx-
Quoted 5 lines
io_uring/net.c static bool io_net_retry(struct socket *sock, int · line 113
Quoted 1 line
io_uring/net.c +0
-0> netmsg_cache.list);can
io_alloc_cache_storejust do the store?
would be nicer to havecache::listbe generally unused outside of the cache code.
We could do that if we just make the hlist_node be inside a struct. Would probably allow cleaning up the get-entry etc too, let me give that a whirl.
Signature · 1 line
Jens Axboe