Skip to contentSkip to conversation

another round of rsrc refactoring

PATCH Applied Pavel Begunkov 2 participants 10 messages Apr 18, 2023 – Apr 19, 2023 io_uring

lore
Share & tools
lore
#0 lore raw

cover another round of rsrc refactoring

Further simplify rsrc infrastructure, and make it a little bit faster.

The main part is Patch 3, which establishes 1:1 relation between struct io_rsrc_put and nodes, which removes io_rsrc_node_switch() / io_rsrc_node_switch_start() and all the additional complexity with pre allocations. Note, it doesn't change any guarantees as io_queue_rsrc_removal() was doing allocations anyway and could always fail.

Pavel Begunkov (8):
io_uring/rsrc: remove unused io_rsrc_node::llist io_uring/rsrc: infer node from ctx on io_queue_rsrc_removal io_uring/rsrc: merge nodes and io_rsrc_put io_uring/rsrc: add empty flag in rsrc_node io_uring/rsrc: inline io_rsrc_put_work() io_uring/rsrc: pass node to io_rsrc_put_work() io_uring/rsrc: devirtualise rsrc put callbacks io_uring/rsrc: disassociate nodes and rsrc_data

 io_uring/filetable.c |  14 +----
 io_uring/rsrc.c      | 146 ++++++++++++++++---------------------------
 io_uring/rsrc.h      |  32 ++--------
 3 files changed, 61 insertions(+), 131 deletions(-)
Signature · 1 line
2.40.0
#1 lore raw

PATCH 1/8 io_uring/rsrc: remove unused io_rsrc_node::llist

->llist was needed for rsrc node destruction offload, which is removed now. Get rid of the unused field.

---
 io_uring/rsrc.h | 1 -
 1 file changed, 1 deletion(-)
io_uring/rsrc.h +0 -1
index 732496afed4c..525905a30a55 100644
@@ -44,7 +44,6 @@ struct io_rsrc_node {
 		struct io_rsrc_data		*rsrc_data;
 	};
 	struct list_head		node;
-	struct llist_node		llist;
 	int				refs;
 
 	/*
Signature · 1 line
2.40.0
#2 lore raw

PATCH 2/8 io_uring/rsrc: infer node from ctx on io_queue_rsrc_removal

In reply to Pavel Begunkov

For io_queue_rsrc_removal() we should always use the current active rsrc node, don't pass it directly but let the function grab it from the context.

---
 io_uring/filetable.c | 5 ++---
 io_uring/rsrc.c      | 9 +++++----
 io_uring/rsrc.h      | 3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)
io_uring/filetable.c +2 -3
index 6255fa255ae2..367a44a6c8c5 100644
@@ -85,8 +85,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 			return ret;
 
 		old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
-		ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
-					    ctx->rsrc_node, old_file);
+		ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
 		if (ret)
 			return ret;
 
@@ -163,7 +162,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
 		return -EBADF;
 
 	file = (struct file *)(file_slot->file_ptr & FFS_MASK);
-	ret = io_queue_rsrc_removal(ctx->file_data, offset, ctx->rsrc_node, file);
+	ret = io_queue_rsrc_removal(ctx->file_data, offset, file);
 	if (ret)
 		return ret;
 
io_uring/rsrc.c +5 -4
index db58a51d19da..3be483de613e 100644
@@ -409,7 +409,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
 
 		if (file_slot->file_ptr) {
 			file = (struct file *)(file_slot->file_ptr & FFS_MASK);
-			err = io_queue_rsrc_removal(data, i, ctx->rsrc_node, file);
+			err = io_queue_rsrc_removal(data, i, file);
 			if (err)
 				break;
 			file_slot->file_ptr = 0;
@@ -492,7 +492,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		i = array_index_nospec(up->offset + done, ctx->nr_user_bufs);
 		if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
 			err = io_queue_rsrc_removal(ctx->buf_data, i,
-						    ctx->rsrc_node, ctx->user_bufs[i]);
+						    ctx->user_bufs[i]);
 			if (unlikely(err)) {
 				io_buffer_unmap(ctx, &imu);
 				break;
@@ -680,9 +680,10 @@ int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
 	return IOU_OK;
 }
 
-int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
-			  struct io_rsrc_node *node, void *rsrc)
+int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
 {
+	struct io_ring_ctx *ctx = data->ctx;
+	struct io_rsrc_node *node = ctx->rsrc_node;
 	u64 *tag_slot = io_get_tag_slot(data, idx);
 	struct io_rsrc_put *prsrc;
 
io_uring/rsrc.h +1 -2
index 525905a30a55..8ed3e6a65cf6 100644
@@ -70,8 +70,7 @@ void io_rsrc_put_work(struct work_struct *work);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
 int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
-int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
-			  struct io_rsrc_node *node, void *rsrc);
+int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
 void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 			 struct io_rsrc_data *data_to_kill);
 
Signature · 1 line
2.40.0
#5 lore raw

PATCH 5/8 io_uring/rsrc: inline io_rsrc_put_work()

In reply to Pavel Begunkov

io_rsrc_put_work() is simple enough to be open coded into its only caller.

---
 io_uring/rsrc.c | 19 ++++++-------------
 io_uring/rsrc.h |  1 -
 2 files changed, 6 insertions(+), 14 deletions(-)
io_uring/rsrc.c +6 -13
index 127bd602131e..d1167b0643b7 100644
@@ -140,8 +140,8 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
 	*slot = NULL;
 }
 
-static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
-				 struct io_rsrc_put *prsrc)
+static void io_rsrc_put_work(struct io_rsrc_data *rsrc_data,
+			     struct io_rsrc_put *prsrc)
 {
 	struct io_ring_ctx *ctx = rsrc_data->ctx;
 
@@ -150,16 +150,6 @@ static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
 	rsrc_data->do_put(ctx, prsrc);
 }
 
-static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
-{
-	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
-
-	if (likely(!ref_node->empty))
-		io_rsrc_put_work_one(rsrc_data, &ref_node->item);
-
-	io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
-}
-
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
 {
 	if (!io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache))
@@ -178,7 +168,10 @@ void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
 		if (node->refs)
 			break;
 		list_del(&node->node);
-		__io_rsrc_put_work(node);
+
+		if (likely(!node->empty))
+			io_rsrc_put_work(node->rsrc_data, &node->item);
+		io_rsrc_node_destroy(ctx, node);
 	}
 	if (list_empty(&ctx->rsrc_ref_list) && unlikely(ctx->rsrc_quiesce))
 		wake_up_all(&ctx->rsrc_quiesce_wq);
io_uring/rsrc.h +0 -1
index f3fe455c6c71..232079363f6a 100644
@@ -58,7 +58,6 @@ struct io_mapped_ubuf {
 
 void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
-void io_rsrc_put_work(struct work_struct *work);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
Signature · 1 line
2.40.0
#4 lore raw

PATCH 4/8 io_uring/rsrc: add empty flag in rsrc_node

In reply to Pavel Begunkov

Unless a node was flushed by io_rsrc_ref_quiesce(), it'll carry a resource. Replace ->inline_items with an empty flag, which is initialised to false and only raised in io_rsrc_ref_quiesce().

---
 io_uring/rsrc.c | 6 +++---
 io_uring/rsrc.h | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)
io_uring/rsrc.c +3 -3
index a54a222a20b8..127bd602131e 100644
@@ -154,7 +154,7 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 {
 	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
 
-	if (likely(ref_node->inline_items))
+	if (likely(!ref_node->empty))
 		io_rsrc_put_work_one(rsrc_data, &ref_node->item);
 
 	io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
@@ -199,7 +199,7 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
 	}
 
 	ref_node->rsrc_data = NULL;
-	ref_node->inline_items = 0;
+	ref_node->empty = 0;
 	ref_node->refs = 1;
 	return ref_node;
 }
@@ -218,6 +218,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 	backup = io_rsrc_node_alloc(ctx);
 	if (!backup)
 		return -ENOMEM;
+	ctx->rsrc_node->empty = true;
 	ctx->rsrc_node->rsrc_data = data;
 	list_add_tail(&ctx->rsrc_node->node, &ctx->rsrc_ref_list);
 	io_put_rsrc_node(ctx, ctx->rsrc_node);
@@ -649,7 +650,6 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
 
 	node->item.rsrc = rsrc;
 	node->item.tag = *tag_slot;
-	node->inline_items = 1;
 	*tag_slot = 0;
 
 	node->rsrc_data = data;
io_uring/rsrc.h +1 -1
index bad7103f5033..f3fe455c6c71 100644
@@ -43,7 +43,7 @@ struct io_rsrc_node {
 		struct io_rsrc_data		*rsrc_data;
 	};
 	int				refs;
-	int				inline_items;
+	bool				empty;
 	struct list_head		node;
 	struct io_rsrc_put		item;
 };
Signature · 1 line
2.40.0
#6 lore raw

PATCH 6/8 io_uring/rsrc: pass node to io_rsrc_put_work()

In reply to Pavel Begunkov

Instead of passing rsrc_data and a resource to io_rsrc_put_work() just forward node, that's all the function needs.

---
 io_uring/rsrc.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
io_uring/rsrc.c +6 -6
index d1167b0643b7..9378691d49f5 100644
@@ -140,14 +140,14 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
 	*slot = NULL;
 }
 
-static void io_rsrc_put_work(struct io_rsrc_data *rsrc_data,
-			     struct io_rsrc_put *prsrc)
+static void io_rsrc_put_work(struct io_rsrc_node *node)
 {
-	struct io_ring_ctx *ctx = rsrc_data->ctx;
+	struct io_rsrc_data *data = node->rsrc_data;
+	struct io_rsrc_put *prsrc = &node->item;
 
 	if (prsrc->tag)
-		io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
-	rsrc_data->do_put(ctx, prsrc);
+		io_post_aux_cqe(data->ctx, prsrc->tag, 0, 0);
+	data->do_put(data->ctx, prsrc);
 }
 
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
@@ -170,7 +170,7 @@ void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
 		list_del(&node->node);
 
 		if (likely(!node->empty))
-			io_rsrc_put_work(node->rsrc_data, &node->item);
+			io_rsrc_put_work(node);
 		io_rsrc_node_destroy(ctx, node);
 	}
 	if (list_empty(&ctx->rsrc_ref_list) && unlikely(ctx->rsrc_quiesce))
Signature · 1 line
2.40.0
#3 lore raw

PATCH 3/8 io_uring/rsrc: merge nodes and io_rsrc_put

In reply to Pavel Begunkov

struct io_rsrc_node carries a number of resources represented by struct io_rsrc_put. That was handy before for sync overhead ammortisation, but all complexity is gone and nodes are simple and lightweight. Let's allocate a separate node for each resource.

Nodes and io_rsrc_put and not much different in size, and former are cached, so node allocation should work better. That also removes some overhead for nested iteration in io_rsrc_node_ref_zero() / __io_rsrc_put_work().

Another reason for the patch is that it greatly reduces complexity by moving io_rsrc_node_switch[_start]() inside io_queue_rsrc_removal(), so users don't have to care about it.

---
 io_uring/filetable.c |  9 -----
 io_uring/rsrc.c      | 91 +++++++++++---------------------------------
 io_uring/rsrc.h      | 22 +----------
 3 files changed, 25 insertions(+), 97 deletions(-)
io_uring/filetable.c +0 -9
index 367a44a6c8c5..0f6fa791a47d 100644
@@ -80,10 +80,6 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 	if (file_slot->file_ptr) {
 		struct file *old_file;
 
-		ret = io_rsrc_node_switch_start(ctx);
-		if (ret)
-			return ret;
-
 		old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
 		ret = io_queue_rsrc_removal(ctx->file_data, slot_index, old_file);
 		if (ret)
@@ -91,7 +87,6 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 
 		file_slot->file_ptr = 0;
 		io_file_bitmap_clear(&ctx->file_table, slot_index);
-		io_rsrc_node_switch(ctx, ctx->file_data);
 	}
 
 	ret = io_scm_file_account(ctx, file);
@@ -152,9 +147,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
 		return -ENXIO;
 	if (offset >= ctx->nr_user_files)
 		return -EINVAL;
-	ret = io_rsrc_node_switch_start(ctx);
-	if (ret)
-		return ret;
 
 	offset = array_index_nospec(offset, ctx->nr_user_files);
 	file_slot = io_fixed_file_slot(&ctx->file_table, offset);
@@ -168,7 +160,6 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, unsigned int offset)
 
 	file_slot->file_ptr = 0;
 	io_file_bitmap_clear(&ctx->file_table, offset);
-	io_rsrc_node_switch(ctx, ctx->file_data);
 	return 0;
 }
 
io_uring/rsrc.c +23 -68

174 lines not on this page · raw message

io_uring/rsrc.h +2 -20
index 8ed3e6a65cf6..bad7103f5033 100644
@@ -18,7 +18,6 @@ enum {
 };
 
 struct io_rsrc_put {
-	struct list_head list;
 	u64 tag;
 	union {
 		void *rsrc;
@@ -43,17 +42,10 @@ struct io_rsrc_node {
 		struct io_cache_entry		cache;
 		struct io_rsrc_data		*rsrc_data;
 	};
-	struct list_head		node;
 	int				refs;
-
-	/*
-	 * Keeps a list of struct io_rsrc_put to be completed. Each entry
-	 * represents one rsrc (e.g. file or buffer), but all of them should've
-	 * came from the same table and so are of the same type.
-	 */
-	struct list_head		item_list;
-	struct io_rsrc_put		item;
 	int				inline_items;
+	struct list_head		node;
+	struct io_rsrc_put		item;
 };
 
 struct io_mapped_ubuf {
@@ -68,11 +60,8 @@ void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_put_work(struct work_struct *work);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
-int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc);
-void io_rsrc_node_switch(struct io_ring_ctx *ctx,
-			 struct io_rsrc_data *data_to_kill);
 
 int io_import_fixed(int ddir, struct iov_iter *iter,
 			   struct io_mapped_ubuf *imu,
@@ -109,13 +98,6 @@ static inline int io_scm_file_account(struct io_ring_ctx *ctx,
 	return __io_scm_file_account(ctx, file);
 }
 
-static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
-{
-	if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache)))
-		return __io_rsrc_node_switch_start(ctx);
-	return 0;
-}
-
 int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
 			     unsigned nr_args);
 int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Signature · 1 line
2.40.0
#7 lore raw

PATCH 7/8 io_uring/rsrc: devirtualise rsrc put callbacks

In reply to Pavel Begunkov

We only have two rsrc types, buffers and files, replace virtual callbacks for putting resources down with a switch..case.

---
 io_uring/rsrc.c | 25 +++++++++++++++++++------
 io_uring/rsrc.h |  2 +-
 2 files changed, 20 insertions(+), 7 deletions(-)
io_uring/rsrc.c +19 -6
index 9378691d49f5..62988b3aa927 100644
@@ -23,6 +23,8 @@ struct io_rsrc_update {
 	u32				offset;
 };
 
+static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
+static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
 				  struct io_mapped_ubuf **pimu,
 				  struct page **last_hpage);
@@ -147,7 +149,18 @@ static void io_rsrc_put_work(struct io_rsrc_node *node)
 
 	if (prsrc->tag)
 		io_post_aux_cqe(data->ctx, prsrc->tag, 0, 0);
-	data->do_put(data->ctx, prsrc);
+
+	switch (data->rsrc_type) {
+	case IORING_RSRC_FILE:
+		io_rsrc_file_put(data->ctx, prsrc);
+		break;
+	case IORING_RSRC_BUFFER:
+		io_rsrc_buf_put(data->ctx, prsrc);
+		break;
+	default:
+		WARN_ON_ONCE(1);
+		break;
+	}
 }
 
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
@@ -297,8 +310,8 @@ static __cold void **io_alloc_page_table(size_t size)
 	return table;
 }
 
-__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
-				     rsrc_put_fn *do_put, u64 __user *utags,
+__cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, int type,
+				     u64 __user *utags,
 				     unsigned nr, struct io_rsrc_data **pdata)
 {
 	struct io_rsrc_data *data;
@@ -316,7 +329,7 @@ __cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
 
 	data->nr = nr;
 	data->ctx = ctx;
-	data->do_put = do_put;
+	data->rsrc_type = type;
 	if (utags) {
 		ret = -EFAULT;
 		for (i = 0; i < nr; i++) {
@@ -849,7 +862,7 @@ int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
 		return -EMFILE;
 	if (nr_args > rlimit(RLIMIT_NOFILE))
 		return -EMFILE;
-	ret = io_rsrc_data_alloc(ctx, io_rsrc_file_put, tags, nr_args,
+	ret = io_rsrc_data_alloc(ctx, IORING_RSRC_FILE, tags, nr_args,
 				 &ctx->file_data);
 	if (ret)
 		return ret;
@@ -1184,7 +1197,7 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
 		return -EBUSY;
 	if (!nr_args || nr_args > IORING_MAX_REG_BUFFERS)
 		return -EINVAL;
-	ret = io_rsrc_data_alloc(ctx, io_rsrc_buf_put, tags, nr_args, &data);
+	ret = io_rsrc_data_alloc(ctx, IORING_RSRC_BUFFER, tags, nr_args, &data);
 	if (ret)
 		return ret;
 	ret = io_buffers_map_alloc(ctx, nr_args);
io_uring/rsrc.h +1 -1
index 232079363f6a..5d0733c4c08d 100644
@@ -33,7 +33,7 @@ struct io_rsrc_data {
 
 	u64				**tags;
 	unsigned int			nr;
-	rsrc_put_fn			*do_put;
+	u16				rsrc_type;
 	bool				quiesce;
 };
 
Signature · 1 line
2.40.0
#8 lore raw

PATCH 8/8 io_uring/rsrc: disassociate nodes and rsrc_data

In reply to Pavel Begunkov

Make rsrc nodes independent from rsrd_data, for that we keep ctx and rsrc type in nodes.

---
 io_uring/rsrc.c | 20 +++++++++-----------
 io_uring/rsrc.h |  3 ++-
 2 files changed, 11 insertions(+), 12 deletions(-)
io_uring/rsrc.c +9 -11
index 62988b3aa927..20dcc7668cb0 100644
@@ -144,18 +144,17 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
 
 static void io_rsrc_put_work(struct io_rsrc_node *node)
 {
-	struct io_rsrc_data *data = node->rsrc_data;
 	struct io_rsrc_put *prsrc = &node->item;
 
 	if (prsrc->tag)
-		io_post_aux_cqe(data->ctx, prsrc->tag, 0, 0);
+		io_post_aux_cqe(node->ctx, prsrc->tag, 0, 0);
 
-	switch (data->rsrc_type) {
+	switch (node->type) {
 	case IORING_RSRC_FILE:
-		io_rsrc_file_put(data->ctx, prsrc);
+		io_rsrc_file_put(node->ctx, prsrc);
 		break;
 	case IORING_RSRC_BUFFER:
-		io_rsrc_buf_put(data->ctx, prsrc);
+		io_rsrc_buf_put(node->ctx, prsrc);
 		break;
 	default:
 		WARN_ON_ONCE(1);
@@ -170,9 +169,9 @@ void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
 }
 
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
-	__must_hold(&node->rsrc_data->ctx->uring_lock)
+	__must_hold(&node->ctx->uring_lock)
 {
-	struct io_ring_ctx *ctx = node->rsrc_data->ctx;
+	struct io_ring_ctx *ctx = node->ctx;
 
 	while (!list_empty(&ctx->rsrc_ref_list)) {
 		node = list_first_entry(&ctx->rsrc_ref_list,
@@ -204,7 +203,7 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
 			return NULL;
 	}
 
-	ref_node->rsrc_data = NULL;
+	ref_node->ctx = ctx;
 	ref_node->empty = 0;
 	ref_node->refs = 1;
 	return ref_node;
@@ -225,7 +224,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 	if (!backup)
 		return -ENOMEM;
 	ctx->rsrc_node->empty = true;
-	ctx->rsrc_node->rsrc_data = data;
+	ctx->rsrc_node->type = -1;
 	list_add_tail(&ctx->rsrc_node->node, &ctx->rsrc_ref_list);
 	io_put_rsrc_node(ctx, ctx->rsrc_node);
 	ctx->rsrc_node = backup;
@@ -655,10 +654,9 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, void *rsrc)
 	}
 
 	node->item.rsrc = rsrc;
+	node->type = data->rsrc_type;
 	node->item.tag = *tag_slot;
 	*tag_slot = 0;
-
-	node->rsrc_data = data;
 	list_add_tail(&node->node, &ctx->rsrc_ref_list);
 	io_put_rsrc_node(ctx, node);
 	return 0;
io_uring/rsrc.h +2 -1
index 5d0733c4c08d..0a8a95e9b99e 100644
@@ -40,10 +40,11 @@ struct io_rsrc_data {
 struct io_rsrc_node {
 	union {
 		struct io_cache_entry		cache;
-		struct io_rsrc_data		*rsrc_data;
+		struct io_ring_ctx		*ctx;
 	};
 	int				refs;
 	bool				empty;
+	u16				type;
 	struct list_head		node;
 	struct io_rsrc_put		item;
 };
Signature · 1 line
2.40.0
maintainer Applied
#9 lore raw
In reply to Pavel Begunkov
Quoted 11 lines · Pavel Begunkov

Further simplify rsrc infrastructure, and make it a little bit faster.

The main part is Patch 3, which establishes 1:1 relation between struct io_rsrc_put and nodes, which removes io_rsrc_node_switch() / io_rsrc_node_switch_start() and all the additional complexity with pre allocations. Note, it doesn't change any guarantees as io_queue_rsrc_removal() was doing allocations anyway and could always fail.

[...]

Applied, thanks!

[1/8] io_uring/rsrc: remove unused io_rsrc_node::llist

      commit: 2e6f45ac0e640bbd49296adfa0982c84f85fa342
[2/8] io_uring/rsrc: infer node from ctx on io_queue_rsrc_removal
      commit: 63fea89027ff4fd4f350b471ad5b9220d373eec5
[3/8] io_uring/rsrc: merge nodes and io_rsrc_put
      commit: c376644fb915fbdea8c4a04f859d032a8be352fd
[4/8] io_uring/rsrc: add empty flag in rsrc_node
      commit: 26147da37f3e52041d9deba189d39f27ce78a84f
[5/8] io_uring/rsrc: inline io_rsrc_put_work()
      commit: 4130b49991d6b8ca0ea44cb256e710c4e48d7f01
[6/8] io_uring/rsrc: pass node to io_rsrc_put_work()
      commit: 29b26c556e7439b1370ac6a59fce83a9d1521de1
[7/8] io_uring/rsrc: devirtualise rsrc put callbacks
      commit: fc7f3a8d3a78503c4f3e108155fb9a233dc307a4
[8/8] io_uring/rsrc: disassociate nodes and rsrc_data
      commit: 2236b3905b4d4e9cd4d149ab35767858c02bb79b

Best regards,

Signature · 1 line
Jens Axboe