legacy provided buffer deprecation / deoptimisation
cover legacy provided buffer deprecation / deoptimisation
Legacy provided buffers are slow and discouraged, users are encouraged to use ring provided buffers instead. Clean up the legacy code, remove caching and optimisations. The goal here it to make it simpler and less of a burden to maintain.
Pavel Begunkov (8):
io_uring/kbuf: remove legacy kbuf bulk allocation io_uring/kbuf: remove legacy kbuf kmem cache io_uring/kbuf: move locking into io_kbuf_drop() io_uring/kbuf: simplify __io_put_kbuf io_uring/kbuf: remove legacy kbuf caching io_uring/kbuf: open code __io_put_kbuf() io_uring/kbuf: introduce io_kbuf_drop_legacy() io_uring/kbuf: uninline __io_put_kbufs
include/linux/io_uring_types.h | 3 - io_uring/io_uring.c | 11 +-- io_uring/io_uring.h | 1 - io_uring/kbuf.c | 172 +++++++++++++++------------------ io_uring/kbuf.h | 100 +++---------------- 5 files changed, 89 insertions(+), 198 deletions(-)
Signature · 1 line
2.47.1
PATCH 1/8 io_uring/kbuf: remove legacy kbuf bulk allocation
Legacy provided buffers are slow and discouraged in favour of the ring variant. Remove the bulk allocation to keep it simpler as we don't care about performance.
--- io_uring/kbuf.c | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-)
io_uring/kbuf.c +5 -25
@@ -494,12 +494,9 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqereturn 0 ; } -#define IO_BUFFER_ALLOC_BATCH 64 -static int io_refill_buffer_cache (struct io_ring_ctx *ctx) { -struct io_buffer *bufs[IO_BUFFER_ALLOC_BATCH ]; -int allocated; +struct io_buffer *buf;/* * Completions that don't happen inline (eg not under uring_lock) will
@@ -517,27 +514,10 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx)spin_unlock (&ctx->completion_lock); } -/* -* No free buffers and no completion entries either. Allocate a new -* batch of buffer entries and add those to our freelist. -*/ - - allocated =kmem_cache_alloc_bulk (io_buf_cachep,GFP_KERNEL_ACCOUNT , -ARRAY_SIZE (bufs), (void **) bufs); -if (unlikely (!allocated)) { -/* -* Bulk alloc is all-or-nothing. If we fail to get a batch, -* retry single alloc to be on the safe side. -*/ - bufs[0 ] =kmem_cache_alloc (io_buf_cachep,GFP_KERNEL ); -if (!bufs[0 ]) -return -ENOMEM ; - allocated =1 ; - } - -while (allocated) -list_add_tail (&bufs[--allocated]->list, &ctx->io_buffers_cache); - + buf =kmem_cache_alloc (io_buf_cachep,GFP_KERNEL ); +if (!buf) +return -ENOMEM ; +list_add_tail (&buf->list, &ctx->io_buffers_cache);return 0 ; }
Signature · 1 line
2.47.1
PATCH 2/8 io_uring/kbuf: remove legacy kbuf kmem cache
In reply to Pavel BegunkovRemove the kmem cache used by legacy provided buffers.
--- io_uring/io_uring.c | 2 -- io_uring/io_uring.h | 1 - io_uring/kbuf.c | 6 ++---- 3 files changed, 2 insertions(+), 7 deletions(-)
io_uring/io_uring.c +0 -2
@@ -3927,8 +3927,6 @@ static int __init io_uring_init(void) req_cachep =kmem_cache_create ("io_kiocb" ,sizeof (struct io_kiocb ), &kmem_args,SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT |SLAB_TYPESAFE_BY_RCU ); - io_buf_cachep =KMEM_CACHE (io_buffer, -SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT ); iou_wq =alloc_workqueue ("iou_exit" ,WQ_UNBOUND ,64 );BUG_ON (!iou_wq);
io_uring/io_uring.h +0 -1
@@ -418,7 +418,6 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) }extern struct kmem_cache *req_cachep; -extern struct kmem_cache *io_buf_cachep;static inline struct io_kiocb *io_extract_req (struct io_ring_ctx *ctx) {
io_uring/kbuf.c +2 -4
@@ -20,8 +20,6 @@/* BIDs are addressed by a 16-bit field in a CQE */ #define MAX_BIDS_PER_BGID (1 << 16) -struct kmem_cache *io_buf_cachep; -struct io_provide_buf {struct file *file;__u64 addr;
@@ -411,7 +409,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)list_for_each_safe (item, tmp, &ctx->io_buffers_cache) { buf = list_entry(item, struct io_buffer, list); -kmem_cache_free (io_buf_cachep, buf); +kfree (buf); } }
@@ -514,7 +512,7 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx)spin_unlock (&ctx->completion_lock); } - buf =kmem_cache_alloc (io_buf_cachep,GFP_KERNEL ); + buf =kmalloc (sizeof (*buf),GFP_KERNEL_ACCOUNT );if (!buf)return -ENOMEM ;list_add_tail (&buf->list, &ctx->io_buffers_cache);
Signature · 1 line
2.47.1
PATCH 3/8 io_uring/kbuf: move locking into io_kbuf_drop()
In reply to Pavel BegunkovMove the burden of locking out of the caller into io_kbuf_drop(), that will help with furher refactoring.
--- io_uring/io_uring.c | 5 +---- io_uring/kbuf.h | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-)
io_uring/io_uring.c +1 -4
@@ -398,11 +398,8 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)static void io_clean_op (struct io_kiocb *req) { -if (req->flags &REQ_F_BUFFER_SELECTED ) { -spin_lock (&req->ctx->completion_lock); +if (unlikely (req->flags &REQ_F_BUFFER_SELECTED ))io_kbuf_drop (req); -spin_unlock (&req->ctx->completion_lock); - }if (req->flags &REQ_F_NEED_CLEANUP ) {const struct io_cold_def *def = &io_cold_defs[req->opcode];
io_uring/kbuf.h +2 -2
@@ -174,13 +174,13 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len,static inline void io_kbuf_drop (struct io_kiocb *req) { -lockdep_assert_held (&req->ctx->completion_lock); -if (!(req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )))return ; +spin_lock (&req->ctx->completion_lock);/* len == 0 is fine here, non-ring will always drop all of it */ __io_put_kbuf_list (req,0 , &req->ctx->io_buffers_comp); +spin_unlock (&req->ctx->completion_lock); } static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
Signature · 1 line
2.47.1
PATCH 4/8 io_uring/kbuf: simplify __io_put_kbuf
In reply to Pavel BegunkovAs a preparation step remove an optimisation from __io_put_kbuf() trying to use the locked cache. With that __io_put_kbuf_list() is only used with ->io_buffers_comp, and we remove the explicit list argument.
--- io_uring/kbuf.c | 26 +++----------------------- io_uring/kbuf.h | 7 +++---- 2 files changed, 6 insertions(+), 27 deletions(-)
io_uring/kbuf.c +3 -23
@@ -70,29 +70,9 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)void __io_put_kbuf (struct io_kiocb *req,int len,unsigned issue_flags) { -/* -* We can add this buffer back to two lists: -* -* 1) The io_buffers_cache list. This one is protected by the -* ctx->uring_lock. If we already hold this lock, add back to this -* list as we can grab it from issue as well. -* 2) The io_buffers_comp list. This one is protected by the -* ctx->completion_lock. -* -* We migrate buffers from the comp_list to the issue cache list -* when we need one. -*/ -if (issue_flags &IO_URING_F_UNLOCKED ) { -struct io_ring_ctx *ctx = req->ctx; - -spin_lock (&ctx->completion_lock); -__io_put_kbuf_list (req, len, &ctx->io_buffers_comp); -spin_unlock (&ctx->completion_lock); - }else { -lockdep_assert_held (&req->ctx->uring_lock); - -__io_put_kbuf_list (req, len, &req->ctx->io_buffers_cache); - } +spin_lock (&req->ctx->completion_lock); +__io_put_kbuf_list (req, len); +spin_unlock (&req->ctx->completion_lock); } static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
io_uring/kbuf.h +3 -4
@@ -160,14 +160,13 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)return ret; } -static inline void __io_put_kbuf_list (struct io_kiocb *req,int len, -struct list_head *list) +static inline void __io_put_kbuf_list (struct io_kiocb *req,int len) {if (req->flags &REQ_F_BUFFER_RING ) {__io_put_kbuf_ring (req, len,1 ); }else { req->buf_index = req->kbuf->bgid; -list_add (&req->kbuf->list, list); +list_add (&req->kbuf->list, &req->ctx->io_buffers_comp); req->flags &= ~REQ_F_BUFFER_SELECTED ; } }
@@ -179,7 +178,7 @@ static inline void io_kbuf_drop(struct io_kiocb *req)spin_lock (&req->ctx->completion_lock);/* len == 0 is fine here, non-ring will always drop all of it */ -__io_put_kbuf_list (req,0 , &req->ctx->io_buffers_comp); +__io_put_kbuf_list (req,0 );spin_unlock (&req->ctx->completion_lock); }
Signature · 1 line
2.47.1
PATCH 5/8 io_uring/kbuf: remove legacy kbuf caching
In reply to Pavel BegunkovRemove all struct io_buffer caches. It makes it a fair bit simpler. Apart from from killing a bunch of lines and juggling between lists, __io_put_kbuf_list() doesn't need ->completion_lock locking now.
--- include/linux/io_uring_types.h | 3 -- io_uring/io_uring.c | 2 -- io_uring/kbuf.c | 57 +++++----------------------------- io_uring/kbuf.h | 5 ++- 4 files changed, 9 insertions(+), 58 deletions(-)
include/linux/io_uring_types.h +0 -3
@@ -360,7 +360,6 @@ struct io_ring_ctx {spinlock_t completion_lock; -struct list_head io_buffers_comp;struct list_head cq_overflow_list;struct hlist_head waitid_list;
@@ -379,8 +378,6 @@ struct io_ring_ctx {unsigned int file_alloc_start;unsigned int file_alloc_end; -struct list_head io_buffers_cache; -/* Keep this last, we don't need it for the fast path */ struct wait_queue_head poll_wq;struct io_restriction restrictions;
io_uring/io_uring.c +0 -2
@@ -323,7 +323,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)init_waitqueue_head (&ctx->sqo_sq_wait);INIT_LIST_HEAD (&ctx->sqd_list);INIT_LIST_HEAD (&ctx->cq_overflow_list); -INIT_LIST_HEAD (&ctx->io_buffers_cache); ret =io_alloc_cache_init (&ctx->apoll_cache,IO_POLL_ALLOC_CACHE_MAX ,sizeof (struct async_poll ),0 ); ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
@@ -348,7 +347,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)spin_lock_init (&ctx->completion_lock);raw_spin_lock_init (&ctx->timeout_lock);INIT_WQ_LIST (&ctx->iopoll_list); -INIT_LIST_HEAD (&ctx->io_buffers_comp);INIT_LIST_HEAD (&ctx->defer_list);INIT_LIST_HEAD (&ctx->timeout_list);INIT_LIST_HEAD (&ctx->ltimeout_list);
io_uring/kbuf.c +7 -50
95 lines not on this page · raw message
io_uring/kbuf.h +2 -3
@@ -166,8 +166,9 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len)__io_put_kbuf_ring (req, len,1 ); }else { req->buf_index = req->kbuf->bgid; -list_add (&req->kbuf->list, &req->ctx->io_buffers_comp); req->flags &= ~REQ_F_BUFFER_SELECTED ; +kfree (req->kbuf); + req->kbuf =NULL ; } }
@@ -176,10 +177,8 @@ static inline void io_kbuf_drop(struct io_kiocb *req)if (!(req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING )))return ; -spin_lock (&req->ctx->completion_lock);/* len == 0 is fine here, non-ring will always drop all of it */ __io_put_kbuf_list (req,0 ); -spin_unlock (&req->ctx->completion_lock); } static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
Signature · 1 line
2.47.1
PATCH 6/8 io_uring/kbuf: open code __io_put_kbuf()
In reply to Pavel Begunkov__io_put_kbuf() is a trivial wrapper, open code it into __io_put_kbufs().
--- io_uring/kbuf.c | 5 ----- io_uring/kbuf.h | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-)
io_uring/kbuf.c +0 -5
@@ -68,11 +68,6 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)return true; } -void __io_put_kbuf (struct io_kiocb *req,int len,unsigned issue_flags) -{ -__io_put_kbuf_list (req, len); -} -static void __user *io_provided_buffer_select (struct io_kiocb *req,size_t *len,struct io_buffer_list *bl) {
io_uring/kbuf.h +1 -3
@@ -74,8 +74,6 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);int io_unregister_pbuf_ring (struct io_ring_ctx *ctx,void __user *arg);int io_register_pbuf_status (struct io_ring_ctx *ctx,void __user *arg); -void __io_put_kbuf (struct io_kiocb *req,int len,unsigned issue_flags); -bool io_kbuf_recycle_legacy (struct io_kiocb *req,unsigned issue_flags); struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
@@ -194,7 +192,7 @@ static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,if (!__io_put_kbuf_ring (req, len, nbufs)) ret |=IORING_CQE_F_BUF_MORE ; }else { -__io_put_kbuf (req, len, issue_flags); +__io_put_kbuf_list (req, len); }return ret; }
Signature · 1 line
2.47.1
PATCH 7/8 io_uring/kbuf: introduce io_kbuf_drop_legacy()
In reply to Pavel Begunkovio_kbuf_drop() is only used for legacy provided buffers, and so __io_put_kbuf_list() is never called for REQ_F_BUFFER_RING. Remove the dead branch out of __io_put_kbuf_list(), rename it into io_kbuf_drop_legacy() and use it directly instead of io_kbuf_drop().
--- io_uring/io_uring.c | 2 +- io_uring/kbuf.c | 10 ++++++++++ io_uring/kbuf.h | 24 ++---------------------- 3 files changed, 13 insertions(+), 23 deletions(-)
io_uring/io_uring.c +1 -1
@@ -397,7 +397,7 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)static void io_clean_op (struct io_kiocb *req) {if (unlikely (req->flags &REQ_F_BUFFER_SELECTED )) -io_kbuf_drop (req); +io_kbuf_drop_legacy (req);if (req->flags &REQ_F_NEED_CLEANUP ) {const struct io_cold_def *def = &io_cold_defs[req->opcode];
io_uring/kbuf.c +10 -0
@@ -50,6 +50,16 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx,return xa_err (xa_store (&ctx->io_bl_xa, bgid, bl,GFP_KERNEL )); } +void io_kbuf_drop_legacy (struct io_kiocb *req) +{ +if (WARN_ON_ONCE (!(req->flags &REQ_F_BUFFER_SELECTED ))) +return ; + req->buf_index = req->kbuf->bgid; + req->flags &= ~REQ_F_BUFFER_SELECTED ; +kfree (req->kbuf); + req->kbuf =NULL ; +} +bool io_kbuf_recycle_legacy (struct io_kiocb *req,unsigned issue_flags) {struct io_ring_ctx *ctx = req->ctx;
io_uring/kbuf.h +2 -22
@@ -75,6 +75,7 @@ int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);int io_register_pbuf_status (struct io_ring_ctx *ctx,void __user *arg);bool io_kbuf_recycle_legacy (struct io_kiocb *req,unsigned issue_flags); +void io_kbuf_drop_legacy (struct io_kiocb *req);struct io_mapped_region *io_pbuf_get_region (struct io_ring_ctx *ctx,unsigned int bgid);
@@ -158,27 +159,6 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)return ret; } -static inline void __io_put_kbuf_list (struct io_kiocb *req,int len) -{ -if (req->flags &REQ_F_BUFFER_RING ) { -__io_put_kbuf_ring (req, len,1 ); - }else { - req->buf_index = req->kbuf->bgid; - req->flags &= ~REQ_F_BUFFER_SELECTED ; -kfree (req->kbuf); - req->kbuf =NULL ; - } -} - -static inline void io_kbuf_drop (struct io_kiocb *req) -{ -if (!(req->flags & (REQ_F_BUFFER_SELECTED |REQ_F_BUFFER_RING ))) -return ; - -/* len == 0 is fine here, non-ring will always drop all of it */ -__io_put_kbuf_list (req,0 ); -} -static inline unsigned int __io_put_kbufs (struct io_kiocb *req,int len,int nbufs,unsigned issue_flags) {
@@ -192,7 +172,7 @@ static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,if (!__io_put_kbuf_ring (req, len, nbufs)) ret |=IORING_CQE_F_BUF_MORE ; }else { -__io_put_kbuf_list (req, len); +io_kbuf_drop_legacy (req); }return ret; }
Signature · 1 line
2.47.1
PATCH 8/8 io_uring/kbuf: uninline __io_put_kbufs
In reply to Pavel Begunkov__io_put_kbufs() and other helper functions are too large to be inlined, compilers would normally refuse to do so. Uninline it and move together with io_kbuf_commit into kbuf.c.
--- io_uring/kbuf.c | 60 +++++++++++++++++++++++++++++++++++++++ io_uring/kbuf.h | 74 +++++++------------------------------------------ 2 files changed, 70 insertions(+), 64 deletions(-)
io_uring/kbuf.c +60 -0
@@ -20,6 +20,9 @@/* BIDs are addressed by a 16-bit field in a CQE */ #define MAX_BIDS_PER_BGID (1 << 16) +/* Mapped buffer ring, return io_uring_buf from head */ +#define io_ring_head_to_buf (br, head, mask) &(br)->bufs[(head) & (mask)] +struct io_provide_buf {struct file *file;__u64 addr;
@@ -29,6 +32,34 @@ struct io_provide_buf {__u16 bid; }; +bool io_kbuf_commit (struct io_kiocb *req, +struct io_buffer_list *bl,int len,int nr) +{ +if (unlikely (!(req->flags &REQ_F_BUFFERS_COMMIT ))) +return true; + + req->flags &= ~REQ_F_BUFFERS_COMMIT ; + +if (unlikely (len <0 )) +return true; + +if (bl->flags &IOBL_INC ) { +struct io_uring_buf *buf; + + buf =io_ring_head_to_buf (bl->buf_ring, bl->head, bl->mask); +if (WARN_ON_ONCE (len > buf->len)) + len = buf->len; + buf->len -= len; +if (buf->len) { + buf->addr += len; +return false; + } + } + + bl->head += nr; +return true; +} +static inline struct io_buffer_list *io_buffer_get_list (struct io_ring_ctx *ctx,unsigned int bgid) {
@@ -323,6 +354,35 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg)return io_provided_buffers_select (req, &arg->max_len, bl, arg->iovs); } +static inline bool __io_put_kbuf_ring (struct io_kiocb *req,int len,int nr) +{ +struct io_buffer_list *bl = req->buf_list; +bool ret = true; + +if (bl) { + ret =io_kbuf_commit (req, bl, len, nr); + req->buf_index = bl->bgid; + } + req->flags &= ~REQ_F_BUFFER_RING ; +return ret; +} + +unsigned int __io_put_kbufs (struct io_kiocb *req,int len,int nbufs) +{ +unsigned int ret; + + ret =IORING_CQE_F_BUFFER | (req->buf_index <<IORING_CQE_BUFFER_SHIFT ); + +if (unlikely (!(req->flags &REQ_F_BUFFER_RING ))) { +io_kbuf_drop_legacy (req); +return ret; + } + +if (!__io_put_kbuf_ring (req, len, nbufs)) + ret |=IORING_CQE_F_BUF_MORE ; +return ret; +} +static int __io_remove_buffers (struct io_ring_ctx *ctx,struct io_buffer_list *bl,unsigned nbufs) {
io_uring/kbuf.h +10 -64
93 lines not on this page · raw message
Signature · 1 line
2.47.1
Quoted 15 lines · Pavel Begunkov
Legacy provided buffers are slow and discouraged, users are encouraged to use ring provided buffers instead. Clean up the legacy code, remove caching and optimisations. The goal here it to make it simpler and less of a burden to maintain.
Pavel Begunkov (8):
io_uring/kbuf: remove legacy kbuf bulk allocationio_uring/kbuf: remove legacy kbuf kmem cacheio_uring/kbuf: move locking intoio_kbuf_drop()io_uring/kbuf: simplify__io_put_kbufio_uring/kbuf: remove legacy kbuf cachingio_uring/kbuf: open code__io_put_kbuf()io_uring/kbuf: introduceio_kbuf_drop_legacy()io_uring/kbuf: uninline__io_put_kbufs
Quoted 1 line
[...]
Applied, thanks!
[1/8] io_uring/kbuf: remove legacy kbuf bulk allocation
commit: 95865452e8b06974bb297891acbb7e5a6afc8d4c
[2/8] io_uring/kbuf: remove legacy kbuf kmem cache
commit: 6ad0e0db0d81c3e5ddf3b7ce84cb937590f724a3
[3/8] io_uring/kbuf: move locking into io_kbuf_drop()
commit: 615da6b1d03b53efea22faaab3f1a3d21888ed72
[4/8] io_uring/kbuf: simplify __io_put_kbuf
commit: a6fe909acef9535dc56327b1a872466f080be413
[5/8] io_uring/kbuf: remove legacy kbuf caching
commit: 30205b4708dcd3f2823377ae55afb953a05a2672
[6/8] io_uring/kbuf: open code __io_put_kbuf()
commit: ac6757c5a032c800f927cef1245b81a3b4fabbce
[7/8] io_uring/kbuf: introduce io_kbuf_drop_legacy()
commit: 3d692b5b37fc755eb35881c0f612ed6f00ac7b11
[8/8] io_uring/kbuf: uninline __io_put_kbufs
commit: 641492f1733609b7abebf74ea9ebba6c29b84e79Best regards,
Signature · 1 line
Jens Axboe