Add alloc cache for sendmsg/recvmsg
Jens Axboe
maintainerPATCH v2 1/1 Add alloc cache for sendmsg/recvmsg
Hi,
This abstracts out the alloc cache we have for apoll entries, and extends it to be usable for recv/sendmsg as well. During that abstraction, impose an upper limit for cached entries as well.
This yields a 4-5% performance increase running netbench using sendmsg/recvmsg rather than plan send and recv.
Post 5.20, I suspect we can get rid of using io_async_msghdr for single vector sendmsg/recvmsg, which will make this less relevant. But as this work isn't done yet, and the support for eg ITER_UBUF isn't upstream yet either, this can help fill the gap.
V2:
- Abstract cache node out as well, so we can have common helpers for everything.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/4 io_uring: move apoll cache to poll.c
This is where it's used, move the flush handler in there.
--- io_uring/io_uring.c | 12 ------------ io_uring/poll.c | 12 ++++++++++++ io_uring/poll.h | 2 ++ 3 files changed, 14 insertions(+), 12 deletions(-)
io_uring/io_uring.c +0 -12
@@ -2445,18 +2445,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx)mutex_unlock (&ctx->uring_lock); } -static void io_flush_apoll_cache (struct io_ring_ctx *ctx) -{ -struct async_poll *apoll; - -while (!list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del (&apoll->poll.wait.entry); -kfree (apoll); - } -} -static __cold void io_ring_ctx_free (struct io_ring_ctx *ctx) {io_sq_thread_finish (ctx);
io_uring/poll.c +12 -0
@@ -959,3 +959,15 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)io_req_set_res (req, ret,0 );return IOU_OK ; } + +void io_flush_apoll_cache (struct io_ring_ctx *ctx) +{ +struct async_poll *apoll; + +while (!list_empty (&ctx->apoll_cache)) { + apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, + poll.wait.entry); +list_del (&apoll->poll.wait.entry); +kfree (apoll); + } +}
io_uring/poll.h +2 -0
@@ -30,3 +30,5 @@ int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,int io_arm_poll_handler (struct io_kiocb *req,unsigned issue_flags);bool io_poll_remove_all (struct io_ring_ctx *ctx,struct task_struct *tsk,bool cancel_all); + +void io_flush_apoll_cache (struct io_ring_ctx *ctx);
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 2/4 io_uring: add abstraction around apoll cache
In reply to Jens AxboeIn preparation for adding limits, and one more user, abstract out the core bits of the allocation+free cache.
--- include/linux/io_uring_types.h | 6 ++++- io_uring/alloc_cache.h | 41 ++++++++++++++++++++++++++++++++++ io_uring/io_uring.c | 8 +++---- io_uring/poll.c | 18 +++++---------- io_uring/poll.h | 9 ++++++-- 5 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 io_uring/alloc_cache.h
include/linux/io_uring_types.h +5 -1
@@ -158,6 +158,10 @@ struct io_ev_fd {struct rcu_head rcu; }; +struct io_alloc_cache { +struct hlist_head list; +}; +struct io_ring_ctx {/* const or read-mostly hot data */ struct {
@@ -216,7 +220,7 @@ struct io_ring_ctx {struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list; -struct list_head apoll_cache; +struct io_alloc_cache apoll_cache;struct xarray personalities;u32 pers_next; }____cacheline_aligned_in_smp ;
io_uring/alloc_cache.h new file +41 -0
41 lines not on this page · raw message
io_uring/io_uring.c +4 -4
@@ -92,6 +92,7 @@#include "timeout.h" #include "poll.h" +#include "alloc_cache.h" #define IORING_MAX_ENTRIES 32768#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
@@ -295,7 +296,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->sqd_list);INIT_LIST_HEAD (&ctx->cq_overflow_list);INIT_LIST_HEAD (&ctx->io_buffers_cache); -INIT_LIST_HEAD (&ctx->apoll_cache); +io_alloc_cache_init (&ctx->apoll_cache);init_completion (&ctx->ref_comp);xa_init_flags (&ctx->personalities,XA_FLAGS_ALLOC1 );mutex_init (&ctx->uring_lock);
@@ -1180,8 +1181,7 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)if (apoll->double_poll)kfree (apoll->double_poll); -list_add (&apoll->poll.wait.entry, - &ctx->apoll_cache); +io_alloc_cache_put (&ctx->apoll_cache, &apoll->cache); req->flags &= ~REQ_F_POLLED ; }if (req->flags &IO_REQ_LINK_FLAGS )
@@ -2467,7 +2467,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)if (ctx->rings)__io_cqring_overflow_flush (ctx, true);io_eventfd_unregister (ctx); -io_flush_apoll_cache (ctx); +io_alloc_cache_free (&ctx->apoll_cache, io_apoll_cache_free);mutex_unlock (&ctx->uring_lock);io_destroy_buffers (ctx);if (ctx->sq_creds)
io_uring/poll.c +5 -13
@@ -590,16 +590,15 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req, unsigned issue_flags) {struct io_ring_ctx *ctx = req->ctx; +struct io_cache_entry *entry;struct async_poll *apoll;if (req->flags &REQ_F_POLLED ) { apoll = req->apoll;kfree (apoll->double_poll); }else if (!(issue_flags &IO_URING_F_UNLOCKED ) && - !list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del_init (&apoll->poll.wait.entry); + (entry =io_alloc_cache_get (&ctx->apoll_cache)) !=NULL ) { + apoll = container_of(entry, struct async_poll, cache); }else { apoll =kmalloc (sizeof (*apoll),GFP_ATOMIC );if (unlikely (!apoll))
@@ -960,14 +959,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)return IOU_OK ; } -void io_flush_apoll_cache (struct io_ring_ctx *ctx) +void io_apoll_cache_free (struct io_cache_entry *entry) { -struct async_poll *apoll; - -while (!list_empty (&ctx->apoll_cache)) { - apoll = list_first_entry(&ctx->apoll_cache, struct async_poll, - poll.wait.entry); -list_del (&apoll->poll.wait.entry); -kfree (apoll); - } +kfree (container_of(entry ,struct async_poll ,cache )); }
io_uring/poll.h +7 -2
@@ -1,5 +1,7 @@// SPDX-License-Identifier: GPL-2.0 +#include "alloc_cache.h" +enum {IO_APOLL_OK ,IO_APOLL_ABORTED ,
@@ -14,7 +16,10 @@ struct io_poll { };struct async_poll { -struct io_poll poll; +union { +struct io_poll poll; +struct io_cache_entry cache; + };struct io_poll *double_poll; };
@@ -31,4 +36,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);bool io_poll_remove_all (struct io_ring_ctx *ctx,struct task_struct *tsk,bool cancel_all); -void io_flush_apoll_cache (struct io_ring_ctx *ctx); +void io_apoll_cache_free (struct io_cache_entry *entry);
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 3/4 io_uring: impose max limit on apoll cache
In reply to Jens AxboeCaches like this tend to grow to the peak size, and then never get any smaller. Impose a max limit on the size, to prevent it from growing too big.
A somewhat randomly chosen 512 is the max size we'll allow the cache to get. If a batch of frees come in and would bring it over that, we simply start kfree'ing the surplus.
--- include/linux/io_uring_types.h | 1 + io_uring/alloc_cache.h | 16 ++++++++++++++-- io_uring/io_uring.c | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-)
include/linux/io_uring_types.h +1 -0
@@ -160,6 +160,7 @@ struct io_ev_fd {struct io_alloc_cache {struct hlist_head list; +unsigned int nr_cached; };struct io_ring_ctx {
io_uring/alloc_cache.h +14 -2
@@ -1,14 +1,24 @@#ifndef IOU_ALLOC_CACHE_H #define IOU_ALLOC_CACHE_H +/* +* Don't allow the cache to grow beyond this size. +*/ +#define IO_ALLOC_CACHE_MAX 512 +struct io_cache_entry {struct hlist_node node; }; -static inline void io_alloc_cache_put (struct io_alloc_cache *cache, +static inline bool io_alloc_cache_put (struct io_alloc_cache *cache,struct io_cache_entry *entry) { -hlist_add_head (&entry->node, &cache->list); +if (cache->nr_cached <IO_ALLOC_CACHE_MAX ) { + cache->nr_cached++; +hlist_add_head (&entry->node, &cache->list); +return true; + } +return false; }static inline struct io_cache_entry *io_alloc_cache_get (struct io_alloc_cache *cache)
@@ -26,6 +36,7 @@ static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *cstatic inline void io_alloc_cache_init (struct io_alloc_cache *cache) {INIT_HLIST_HEAD (&cache->list); + cache->nr_cached =0 ; } static inline void io_alloc_cache_free(struct io_alloc_cache *cache,
@@ -37,5 +48,6 @@ static inline void io_alloc_cache_free(struct io_alloc_cache *cache,hlist_del (node); free(container_of(node, struct io_cache_entry, node)); } + cache->nr_cached =0 ; }#endif
io_uring/io_uring.c +2 -1
@@ -1181,7 +1181,8 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)if (apoll->double_poll)kfree (apoll->double_poll); -io_alloc_cache_put (&ctx->apoll_cache, &apoll->cache); +if (!io_alloc_cache_put (&ctx->apoll_cache, &apoll->cache)) +kfree (apoll); req->flags &= ~REQ_F_POLLED ; }if (req->flags &IO_REQ_LINK_FLAGS )
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH 4/4 io_uring: add netmsg cache
In reply to Jens AxboeFor recvmsg/sendmsg, if they don't complete inline, we currently need to allocate a struct io_async_msghdr for each request. This is a somewhat large struct.
Hook up sendmsg/recvmsg to use the io_alloc_cache. This reduces the alloc + free overhead considerably, yielding 4-5% of extra performance running netbench.
--- include/linux/io_uring_types.h | 6 ++-- io_uring/io_uring.c | 3 ++ io_uring/net.c | 63 +++++++++++++++++++++++++++++----- io_uring/net.h | 13 ++++++- 4 files changed, 73 insertions(+), 12 deletions(-)
include/linux/io_uring_types.h +4 -2
@@ -222,8 +222,7 @@ struct io_ring_ctx {struct io_hash_table cancel_table_locked;struct list_head cq_overflow_list;struct io_alloc_cache apoll_cache; -struct xarray personalities; -u32 pers_next; +struct io_alloc_cache netmsg_cache; }____cacheline_aligned_in_smp ;/* IRQ completion list, under ->completion_lock */
@@ -241,6 +240,9 @@ struct io_ring_ctx {unsigned int file_alloc_start;unsigned int file_alloc_end; +struct xarray personalities; +u32 pers_next; +struct {/* * We cache a range of free CQEs we can use, once exhausted it
io_uring/io_uring.c +3 -0
@@ -89,6 +89,7 @@#include "kbuf.h" #include "rsrc.h" #include "cancel.h" +#include "net.h" #include "timeout.h" #include "poll.h"
@@ -297,6 +298,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)INIT_LIST_HEAD (&ctx->cq_overflow_list);INIT_LIST_HEAD (&ctx->io_buffers_cache);io_alloc_cache_init (&ctx->apoll_cache); +io_alloc_cache_init (&ctx->netmsg_cache);init_completion (&ctx->ref_comp);xa_init_flags (&ctx->personalities,XA_FLAGS_ALLOC1 );mutex_init (&ctx->uring_lock);
@@ -2469,6 +2471,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)__io_cqring_overflow_flush (ctx, true);io_eventfd_unregister (ctx);io_alloc_cache_free (&ctx->apoll_cache, io_apoll_cache_free); +io_alloc_cache_free (&ctx->netmsg_cache, io_netmsg_cache_free);mutex_unlock (&ctx->uring_lock);io_destroy_buffers (ctx);if (ctx->sq_creds)
io_uring/net.c +54 -9
134 lines not on this page · raw message
io_uring/net.h +12 -1
@@ -3,9 +3,14 @@#include <linux/net.h> #include <linux/uio.h> +#include "alloc_cache.h" + #if defined(CONFIG_NET) struct io_async_msghdr { - struct iovec fast_iov[UIO_FASTIOV]; + union { + struct iovec fast_iov[UIO_FASTIOV]; + struct io_cache_entry cache; + }; /* points to an allocated iov, if NULL we use fast_iov instead */ struct iovec *free_iov; struct sockaddr __user *uaddr;
@@ -40,4 +45,10 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags);int io_connect_prep_async (struct io_kiocb *req);int io_connect_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_connect (struct io_kiocb *req,unsigned int issue_flags); + +void io_netmsg_cache_free (struct io_cache_entry *entry); +#else +static inline void io_netmsg_cache_free (struct io_cache_entry *entry) +{ +}#endif
Signature · 1 line
2.35.1