Skip to contentSkip to conversation

legacy provided buffer deprecation / deoptimisation

PATCH Applied Pavel Begunkov 2 participants 10 messages Feb 5, 2025 io_uring

lore
Share & tools
lore
#0 lore raw

cover legacy provided buffer deprecation / deoptimisation

Legacy provided buffers are slow and discouraged, users are encouraged to use ring provided buffers instead. Clean up the legacy code, remove caching and optimisations. The goal here it to make it simpler and less of a burden to maintain.

Pavel Begunkov (8):
io_uring/kbuf: remove legacy kbuf bulk allocation io_uring/kbuf: remove legacy kbuf kmem cache io_uring/kbuf: move locking into io_kbuf_drop() io_uring/kbuf: simplify __io_put_kbuf io_uring/kbuf: remove legacy kbuf caching io_uring/kbuf: open code __io_put_kbuf() io_uring/kbuf: introduce io_kbuf_drop_legacy() io_uring/kbuf: uninline __io_put_kbufs

 include/linux/io_uring_types.h |   3 -
 io_uring/io_uring.c            |  11 +--
 io_uring/io_uring.h            |   1 -
 io_uring/kbuf.c                | 172 +++++++++++++++------------------
 io_uring/kbuf.h                | 100 +++----------------
 5 files changed, 89 insertions(+), 198 deletions(-)
Signature · 1 line
2.47.1
#1 lore raw

PATCH 1/8 io_uring/kbuf: remove legacy kbuf bulk allocation

Legacy provided buffers are slow and discouraged in favour of the ring variant. Remove the bulk allocation to keep it simpler as we don't care about performance.

---
 io_uring/kbuf.c | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)
io_uring/kbuf.c +5 -25
index 04bf493eecae0..0bed40f6fe3a5 100644
@@ -494,12 +494,9 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
 	return 0;
 }
 
-#define IO_BUFFER_ALLOC_BATCH 64
-
 static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
 {
-	struct io_buffer *bufs[IO_BUFFER_ALLOC_BATCH];
-	int allocated;
+	struct io_buffer *buf;
 
 	/*
 	 * Completions that don't happen inline (eg not under uring_lock) will
@@ -517,27 +514,10 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
 		spin_unlock(&ctx->completion_lock);
 	}
 
-	/*
-	 * No free buffers and no completion entries either. Allocate a new
-	 * batch of buffer entries and add those to our freelist.
-	 */
-
-	allocated = kmem_cache_alloc_bulk(io_buf_cachep, GFP_KERNEL_ACCOUNT,
-					  ARRAY_SIZE(bufs), (void **) bufs);
-	if (unlikely(!allocated)) {
-		/*
-		 * Bulk alloc is all-or-nothing. If we fail to get a batch,
-		 * retry single alloc to be on the safe side.
-		 */
-		bufs[0] = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
-		if (!bufs[0])
-			return -ENOMEM;
-		allocated = 1;
-	}
-
-	while (allocated)
-		list_add_tail(&bufs[--allocated]->list, &ctx->io_buffers_cache);
-
+	buf = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+	list_add_tail(&buf->list, &ctx->io_buffers_cache);
 	return 0;
 }
 
Signature · 1 line
2.47.1
#2 lore raw

PATCH 2/8 io_uring/kbuf: remove legacy kbuf kmem cache

In reply to Pavel Begunkov

Remove the kmem cache used by legacy provided buffers.

---
 io_uring/io_uring.c | 2 --
 io_uring/io_uring.h | 1 -
 io_uring/kbuf.c     | 6 ++----
 3 files changed, 2 insertions(+), 7 deletions(-)
io_uring/io_uring.c +0 -2
index e34a92c73a5d8..6fa1e88e40fbe 100644
@@ -3927,8 +3927,6 @@ static int __init io_uring_init(void)
 	req_cachep = kmem_cache_create("io_kiocb", sizeof(struct io_kiocb), &kmem_args,
 				SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
 				SLAB_TYPESAFE_BY_RCU);
-	io_buf_cachep = KMEM_CACHE(io_buffer,
-					  SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT);
 
 	iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
 	BUG_ON(!iou_wq);
io_uring/io_uring.h +0 -1
index ab619e63ef39c..85bc8f76ca190 100644
@@ -418,7 +418,6 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
 }
 
 extern struct kmem_cache *req_cachep;
-extern struct kmem_cache *io_buf_cachep;
 
 static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
 {
io_uring/kbuf.c +2 -4
index 0bed40f6fe3a5..ea9fb3c124e56 100644
@@ -20,8 +20,6 @@
 /* BIDs are addressed by a 16-bit field in a CQE */
 #define MAX_BIDS_PER_BGID (1 << 16)
 
-struct kmem_cache *io_buf_cachep;
-
 struct io_provide_buf {
 	struct file			*file;
 	__u64				addr;
@@ -411,7 +409,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)
 
 	list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {
 		buf = list_entry(item, struct io_buffer, list);
-		kmem_cache_free(io_buf_cachep, buf);
+		kfree(buf);
 	}
 }
 
@@ -514,7 +512,7 @@ static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
 		spin_unlock(&ctx->completion_lock);
 	}
 
-	buf = kmem_cache_alloc(io_buf_cachep, GFP_KERNEL);
+	buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
 	if (!buf)
 		return -ENOMEM;
 	list_add_tail(&buf->list, &ctx->io_buffers_cache);
Signature · 1 line
2.47.1
#3 lore raw

PATCH 3/8 io_uring/kbuf: move locking into io_kbuf_drop()

In reply to Pavel Begunkov

Move the burden of locking out of the caller into io_kbuf_drop(), that will help with furher refactoring.

---
 io_uring/io_uring.c | 5 +----
 io_uring/kbuf.h     | 4 ++--
 2 files changed, 3 insertions(+), 6 deletions(-)
io_uring/io_uring.c +1 -4
index 6fa1e88e40fbe..ed7c9081352a4 100644
@@ -398,11 +398,8 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
 
 static void io_clean_op(struct io_kiocb *req)
 {
-	if (req->flags & REQ_F_BUFFER_SELECTED) {
-		spin_lock(&req->ctx->completion_lock);
+	if (unlikely(req->flags & REQ_F_BUFFER_SELECTED))
 		io_kbuf_drop(req);
-		spin_unlock(&req->ctx->completion_lock);
-	}
 
 	if (req->flags & REQ_F_NEED_CLEANUP) {
 		const struct io_cold_def *def = &io_cold_defs[req->opcode];
io_uring/kbuf.h +2 -2
index bd80c44c5af1e..310f94a0727a6 100644
@@ -174,13 +174,13 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len,
 
 static inline void io_kbuf_drop(struct io_kiocb *req)
 {
-	lockdep_assert_held(&req->ctx->completion_lock);
-
 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
 		return;
 
+	spin_lock(&req->ctx->completion_lock);
 	/* len == 0 is fine here, non-ring will always drop all of it */
 	__io_put_kbuf_list(req, 0, &req->ctx->io_buffers_comp);
+	spin_unlock(&req->ctx->completion_lock);
 }
 
 static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
Signature · 1 line
2.47.1
#4 lore raw

PATCH 4/8 io_uring/kbuf: simplify __io_put_kbuf

In reply to Pavel Begunkov

As a preparation step remove an optimisation from __io_put_kbuf() trying to use the locked cache. With that __io_put_kbuf_list() is only used with ->io_buffers_comp, and we remove the explicit list argument.

---
 io_uring/kbuf.c | 26 +++-----------------------
 io_uring/kbuf.h |  7 +++----
 2 files changed, 6 insertions(+), 27 deletions(-)
io_uring/kbuf.c +3 -23
index ea9fb3c124e56..eae6cf502b57f 100644
@@ -70,29 +70,9 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
 
 void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags)
 {
-	/*
-	 * We can add this buffer back to two lists:
-	 *
-	 * 1) The io_buffers_cache list. This one is protected by the
-	 *    ctx->uring_lock. If we already hold this lock, add back to this
-	 *    list as we can grab it from issue as well.
-	 * 2) The io_buffers_comp list. This one is protected by the
-	 *    ctx->completion_lock.
-	 *
-	 * We migrate buffers from the comp_list to the issue cache list
-	 * when we need one.
-	 */
-	if (issue_flags & IO_URING_F_UNLOCKED) {
-		struct io_ring_ctx *ctx = req->ctx;
-
-		spin_lock(&ctx->completion_lock);
-		__io_put_kbuf_list(req, len, &ctx->io_buffers_comp);
-		spin_unlock(&ctx->completion_lock);
-	} else {
-		lockdep_assert_held(&req->ctx->uring_lock);
-
-		__io_put_kbuf_list(req, len, &req->ctx->io_buffers_cache);
-	}
+	spin_lock(&req->ctx->completion_lock);
+	__io_put_kbuf_list(req, len);
+	spin_unlock(&req->ctx->completion_lock);
 }
 
 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
io_uring/kbuf.h +3 -4
index 310f94a0727a6..1f28770648298 100644
@@ -160,14 +160,13 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
 	return ret;
 }
 
-static inline void __io_put_kbuf_list(struct io_kiocb *req, int len,
-				      struct list_head *list)
+static inline void __io_put_kbuf_list(struct io_kiocb *req, int len)
 {
 	if (req->flags & REQ_F_BUFFER_RING) {
 		__io_put_kbuf_ring(req, len, 1);
 	} else {
 		req->buf_index = req->kbuf->bgid;
-		list_add(&req->kbuf->list, list);
+		list_add(&req->kbuf->list, &req->ctx->io_buffers_comp);
 		req->flags &= ~REQ_F_BUFFER_SELECTED;
 	}
 }
@@ -179,7 +178,7 @@ static inline void io_kbuf_drop(struct io_kiocb *req)
 
 	spin_lock(&req->ctx->completion_lock);
 	/* len == 0 is fine here, non-ring will always drop all of it */
-	__io_put_kbuf_list(req, 0, &req->ctx->io_buffers_comp);
+	__io_put_kbuf_list(req, 0);
 	spin_unlock(&req->ctx->completion_lock);
 }
 
Signature · 1 line
2.47.1
#5 lore raw

PATCH 5/8 io_uring/kbuf: remove legacy kbuf caching

In reply to Pavel Begunkov

Remove all struct io_buffer caches. It makes it a fair bit simpler. Apart from from killing a bunch of lines and juggling between lists, __io_put_kbuf_list() doesn't need ->completion_lock locking now.

---
 include/linux/io_uring_types.h |  3 --
 io_uring/io_uring.c            |  2 --
 io_uring/kbuf.c                | 57 +++++-----------------------------
 io_uring/kbuf.h                |  5 ++-
 4 files changed, 9 insertions(+), 58 deletions(-)
include/linux/io_uring_types.h +0 -3
index 3def525a1da37..e2fef264ff8b8 100644
@@ -360,7 +360,6 @@ struct io_ring_ctx {
 
 	spinlock_t		completion_lock;
 
-	struct list_head	io_buffers_comp;
 	struct list_head	cq_overflow_list;
 
 	struct hlist_head	waitid_list;
@@ -379,8 +378,6 @@ struct io_ring_ctx {
 	unsigned int		file_alloc_start;
 	unsigned int		file_alloc_end;
 
-	struct list_head	io_buffers_cache;
-
 	/* Keep this last, we don't need it for the fast path */
 	struct wait_queue_head		poll_wq;
 	struct io_restriction		restrictions;
io_uring/io_uring.c +0 -2
index ed7c9081352a4..969caaccce9d8 100644
@@ -323,7 +323,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	init_waitqueue_head(&ctx->sqo_sq_wait);
 	INIT_LIST_HEAD(&ctx->sqd_list);
 	INIT_LIST_HEAD(&ctx->cq_overflow_list);
-	INIT_LIST_HEAD(&ctx->io_buffers_cache);
 	ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
 			    sizeof(struct async_poll), 0);
 	ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
@@ -348,7 +347,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	spin_lock_init(&ctx->completion_lock);
 	raw_spin_lock_init(&ctx->timeout_lock);
 	INIT_WQ_LIST(&ctx->iopoll_list);
-	INIT_LIST_HEAD(&ctx->io_buffers_comp);
 	INIT_LIST_HEAD(&ctx->defer_list);
 	INIT_LIST_HEAD(&ctx->timeout_list);
 	INIT_LIST_HEAD(&ctx->ltimeout_list);
io_uring/kbuf.c +7 -50

95 lines not on this page · raw message

io_uring/kbuf.h +2 -3
index 1f28770648298..c0b9636c5c4ae 100644
@@ -166,8 +166,9 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len)
 		__io_put_kbuf_ring(req, len, 1);
 	} else {
 		req->buf_index = req->kbuf->bgid;
-		list_add(&req->kbuf->list, &req->ctx->io_buffers_comp);
 		req->flags &= ~REQ_F_BUFFER_SELECTED;
+		kfree(req->kbuf);
+		req->kbuf = NULL;
 	}
 }
 
@@ -176,10 +177,8 @@ static inline void io_kbuf_drop(struct io_kiocb *req)
 	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
 		return;
 
-	spin_lock(&req->ctx->completion_lock);
 	/* len == 0 is fine here, non-ring will always drop all of it */
 	__io_put_kbuf_list(req, 0);
-	spin_unlock(&req->ctx->completion_lock);
 }
 
 static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
Signature · 1 line
2.47.1
#6 lore raw

PATCH 6/8 io_uring/kbuf: open code __io_put_kbuf()

In reply to Pavel Begunkov

__io_put_kbuf() is a trivial wrapper, open code it into __io_put_kbufs().

---
 io_uring/kbuf.c | 5 -----
 io_uring/kbuf.h | 4 +---
 2 files changed, 1 insertion(+), 8 deletions(-)
io_uring/kbuf.c +0 -5
index ef0c06d1bc86f..f41c141ae8eda 100644
@@ -68,11 +68,6 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
 	return true;
 }
 
-void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags)
-{
-	__io_put_kbuf_list(req, len);
-}
-
 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
 					      struct io_buffer_list *bl)
 {
io_uring/kbuf.h +1 -3
index c0b9636c5c4ae..055b7a672f2e0 100644
@@ -74,8 +74,6 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
 int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
 
-void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags);
-
 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
 
 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
@@ -194,7 +192,7 @@ static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
 		if (!__io_put_kbuf_ring(req, len, nbufs))
 			ret |= IORING_CQE_F_BUF_MORE;
 	} else {
-		__io_put_kbuf(req, len, issue_flags);
+		__io_put_kbuf_list(req, len);
 	}
 	return ret;
 }
Signature · 1 line
2.47.1
#7 lore raw

PATCH 7/8 io_uring/kbuf: introduce io_kbuf_drop_legacy()

In reply to Pavel Begunkov

io_kbuf_drop() is only used for legacy provided buffers, and so __io_put_kbuf_list() is never called for REQ_F_BUFFER_RING. Remove the dead branch out of __io_put_kbuf_list(), rename it into io_kbuf_drop_legacy() and use it directly instead of io_kbuf_drop().

---
 io_uring/io_uring.c |  2 +-
 io_uring/kbuf.c     | 10 ++++++++++
 io_uring/kbuf.h     | 24 ++----------------------
 3 files changed, 13 insertions(+), 23 deletions(-)
io_uring/io_uring.c +1 -1
index 969caaccce9d8..ec98a0ec6f34e 100644
@@ -397,7 +397,7 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
 static void io_clean_op(struct io_kiocb *req)
 {
 	if (unlikely(req->flags & REQ_F_BUFFER_SELECTED))
-		io_kbuf_drop(req);
+		io_kbuf_drop_legacy(req);
 
 	if (req->flags & REQ_F_NEED_CLEANUP) {
 		const struct io_cold_def *def = &io_cold_defs[req->opcode];
io_uring/kbuf.c +10 -0
index f41c141ae8eda..6d76108b67e03 100644
@@ -50,6 +50,16 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx,
 	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
 }
 
+void io_kbuf_drop_legacy(struct io_kiocb *req)
+{
+	if (WARN_ON_ONCE(!(req->flags & REQ_F_BUFFER_SELECTED)))
+		return;
+	req->buf_index = req->kbuf->bgid;
+	req->flags &= ~REQ_F_BUFFER_SELECTED;
+	kfree(req->kbuf);
+	req->kbuf = NULL;
+}
+
 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
 {
 	struct io_ring_ctx *ctx = req->ctx;
io_uring/kbuf.h +2 -22
index 055b7a672f2e0..3e18c916afc60 100644
@@ -75,6 +75,7 @@ int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);
 int io_register_pbuf_status(struct io_ring_ctx *ctx, void __user *arg);
 
 bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags);
+void io_kbuf_drop_legacy(struct io_kiocb *req);
 
 struct io_mapped_region *io_pbuf_get_region(struct io_ring_ctx *ctx,
 					    unsigned int bgid);
@@ -158,27 +159,6 @@ static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
 	return ret;
 }
 
-static inline void __io_put_kbuf_list(struct io_kiocb *req, int len)
-{
-	if (req->flags & REQ_F_BUFFER_RING) {
-		__io_put_kbuf_ring(req, len, 1);
-	} else {
-		req->buf_index = req->kbuf->bgid;
-		req->flags &= ~REQ_F_BUFFER_SELECTED;
-		kfree(req->kbuf);
-		req->kbuf = NULL;
-	}
-}
-
-static inline void io_kbuf_drop(struct io_kiocb *req)
-{
-	if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
-		return;
-
-	/* len == 0 is fine here, non-ring will always drop all of it */
-	__io_put_kbuf_list(req, 0);
-}
-
 static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
 					  int nbufs, unsigned issue_flags)
 {
@@ -192,7 +172,7 @@ static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,
 		if (!__io_put_kbuf_ring(req, len, nbufs))
 			ret |= IORING_CQE_F_BUF_MORE;
 	} else {
-		__io_put_kbuf_list(req, len);
+		io_kbuf_drop_legacy(req);
 	}
 	return ret;
 }
Signature · 1 line
2.47.1
#8 lore raw

PATCH 8/8 io_uring/kbuf: uninline __io_put_kbufs

In reply to Pavel Begunkov

__io_put_kbufs() and other helper functions are too large to be inlined, compilers would normally refuse to do so. Uninline it and move together with io_kbuf_commit into kbuf.c.

io_kbuf_commitSigned-off-by Pavel Begunkov <[email protected]>
---
 io_uring/kbuf.c | 60 +++++++++++++++++++++++++++++++++++++++
 io_uring/kbuf.h | 74 +++++++------------------------------------------
 2 files changed, 70 insertions(+), 64 deletions(-)
io_uring/kbuf.c +60 -0
index 6d76108b67e03..319c5a25f72db 100644
@@ -20,6 +20,9 @@
 /* BIDs are addressed by a 16-bit field in a CQE */
 #define MAX_BIDS_PER_BGID (1 << 16)
 
+/* Mapped buffer ring, return io_uring_buf from head */
+#define io_ring_head_to_buf(br, head, mask)	&(br)->bufs[(head) & (mask)]
+
 struct io_provide_buf {
 	struct file			*file;
 	__u64				addr;
@@ -29,6 +32,34 @@ struct io_provide_buf {
 	__u16				bid;
 };
 
+bool io_kbuf_commit(struct io_kiocb *req,
+		    struct io_buffer_list *bl, int len, int nr)
+{
+	if (unlikely(!(req->flags & REQ_F_BUFFERS_COMMIT)))
+		return true;
+
+	req->flags &= ~REQ_F_BUFFERS_COMMIT;
+
+	if (unlikely(len < 0))
+		return true;
+
+	if (bl->flags & IOBL_INC) {
+		struct io_uring_buf *buf;
+
+		buf = io_ring_head_to_buf(bl->buf_ring, bl->head, bl->mask);
+		if (WARN_ON_ONCE(len > buf->len))
+			len = buf->len;
+		buf->len -= len;
+		if (buf->len) {
+			buf->addr += len;
+			return false;
+		}
+	}
+
+	bl->head += nr;
+	return true;
+}
+
 static inline struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
 							unsigned int bgid)
 {
@@ -323,6 +354,35 @@ int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg)
 	return io_provided_buffers_select(req, &arg->max_len, bl, arg->iovs);
 }
 
+static inline bool __io_put_kbuf_ring(struct io_kiocb *req, int len, int nr)
+{
+	struct io_buffer_list *bl = req->buf_list;
+	bool ret = true;
+
+	if (bl) {
+		ret = io_kbuf_commit(req, bl, len, nr);
+		req->buf_index = bl->bgid;
+	}
+	req->flags &= ~REQ_F_BUFFER_RING;
+	return ret;
+}
+
+unsigned int __io_put_kbufs(struct io_kiocb *req, int len, int nbufs)
+{
+	unsigned int ret;
+
+	ret = IORING_CQE_F_BUFFER | (req->buf_index << IORING_CQE_BUFFER_SHIFT);
+
+	if (unlikely(!(req->flags & REQ_F_BUFFER_RING))) {
+		io_kbuf_drop_legacy(req);
+		return ret;
+	}
+
+	if (!__io_put_kbuf_ring(req, len, nbufs))
+		ret |= IORING_CQE_F_BUF_MORE;
+	return ret;
+}
+
 static int __io_remove_buffers(struct io_ring_ctx *ctx,
 			       struct io_buffer_list *bl, unsigned nbufs)
 {
io_uring/kbuf.h +10 -64

93 lines not on this page · raw message

Signature · 1 line
2.47.1
maintainer Applied
#9 lore raw
In reply to Pavel Begunkov
Quoted 15 lines · Pavel Begunkov

Legacy provided buffers are slow and discouraged, users are encouraged to use ring provided buffers instead. Clean up the legacy code, remove caching and optimisations. The goal here it to make it simpler and less of a burden to maintain.

Pavel Begunkov (8):
io_uring/kbuf: remove legacy kbuf bulk allocation io_uring/kbuf: remove legacy kbuf kmem cache io_uring/kbuf: move locking into io_kbuf_drop() io_uring/kbuf: simplify __io_put_kbuf io_uring/kbuf: remove legacy kbuf caching io_uring/kbuf: open code __io_put_kbuf() io_uring/kbuf: introduce io_kbuf_drop_legacy() io_uring/kbuf: uninline __io_put_kbufs

Quoted 1 line

[...]

Applied, thanks!

[1/8] io_uring/kbuf: remove legacy kbuf bulk allocation

      commit: 95865452e8b06974bb297891acbb7e5a6afc8d4c
[2/8] io_uring/kbuf: remove legacy kbuf kmem cache
      commit: 6ad0e0db0d81c3e5ddf3b7ce84cb937590f724a3
[3/8] io_uring/kbuf: move locking into io_kbuf_drop()
      commit: 615da6b1d03b53efea22faaab3f1a3d21888ed72
[4/8] io_uring/kbuf: simplify __io_put_kbuf
      commit: a6fe909acef9535dc56327b1a872466f080be413
[5/8] io_uring/kbuf: remove legacy kbuf caching
      commit: 30205b4708dcd3f2823377ae55afb953a05a2672
[6/8] io_uring/kbuf: open code __io_put_kbuf()
      commit: ac6757c5a032c800f927cef1245b81a3b4fabbce
[7/8] io_uring/kbuf: introduce io_kbuf_drop_legacy()
      commit: 3d692b5b37fc755eb35881c0f612ed6f00ac7b11
[8/8] io_uring/kbuf: uninline __io_put_kbufs
      commit: 641492f1733609b7abebf74ea9ebba6c29b84e79

Best regards,

Signature · 1 line
Jens Axboe