Skip to contentSkip to conversation

some rsrc fixes and clean ups

PATCH Applied Pavel Begunkov 2 participants 12 messages Apr 13, 2023 – Apr 15, 2023 io_uring

lore
Share & tools
lore
#0 lore raw

cover some rsrc fixes and clean ups

Patch 1 is a simple fix for using indexes w/o array_index_nospec() protection.

Patches 2-5 are fixing a file / buffer table unregistration issue when the ring is configured with DEFER_TASKRUN.

The rest are clean ups on top.

Pavel Begunkov (10):
io_uring/rsrc: use nospec'ed indexes io_uring/rsrc: remove io_rsrc_node::done io_uring/rsrc: refactor io_rsrc_ref_quiesce io_uring/rsrc: use wq for quiescing io_uring/rsrc: fix DEFER_TASKRUN rsrc quiesce io_uring/rsrc: remove rsrc_data refs io_uring/rsrc: inline switch_start fast path io_uring/rsrc: clean up __io_sqe_buffers_update() io_uring/rsrc: simplify single file node switching io_uring/rsrc: refactor io_queue_rsrc_removal

 include/linux/io_uring_types.h |  2 +
 io_uring/filetable.c           | 11 ++---
 io_uring/io_uring.c            |  5 +-
 io_uring/rsrc.c                | 90 ++++++++++++++--------------------
 io_uring/rsrc.h                | 13 +++--
 5 files changed, 53 insertions(+), 68 deletions(-)
Signature · 1 line
2.40.0
#1 lore raw

PATCH 1/10 io_uring/rsrc: use nospec'ed indexes

We use array_index_nospec() for registered buffer indexes, but don't use it while poking into rsrc tags, fix that.

Fixes 634d00df5e1cf ("io_uring: add full-fledged dynamic buffers support")
---
 io_uring/rsrc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/rsrc.c +1 -1
index 11058e20bdcc..3c1538b8c8f4 100644
@@ -517,7 +517,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		}
 
 		ctx->user_bufs[i] = imu;
-		*io_get_tag_slot(ctx->buf_data, offset) = tag;
+		*io_get_tag_slot(ctx->buf_data, i) = tag;
 	}
 
 	if (needs_switch)
Signature · 1 line
2.40.0
#2 lore raw

PATCH 2/10 io_uring/rsrc: remove io_rsrc_node::done

In reply to Pavel Begunkov

Kill io_rsrc_node::node and check refs instead, it's set when the nodes refcount hits zero, and it won't change afterwards.

---
 io_uring/rsrc.c | 5 +----
 io_uring/rsrc.h | 1 -
 2 files changed, 1 insertion(+), 5 deletions(-)
io_uring/rsrc.c +1 -4
index 3c1538b8c8f4..5fc9d10743e0 100644
@@ -191,14 +191,12 @@ void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
 {
 	struct io_ring_ctx *ctx = node->rsrc_data->ctx;
 
-	node->done = true;
 	while (!list_empty(&ctx->rsrc_ref_list)) {
 		node = list_first_entry(&ctx->rsrc_ref_list,
 					    struct io_rsrc_node, node);
 		/* recycle ref nodes in order */
-		if (!node->done)
+		if (node->refs)
 			break;
-
 		list_del(&node->node);
 		__io_rsrc_put_work(node);
 	}
@@ -222,7 +220,6 @@ struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
 	ref_node->refs = 1;
 	INIT_LIST_HEAD(&ref_node->node);
 	INIT_LIST_HEAD(&ref_node->item_list);
-	ref_node->done = false;
 	ref_node->inline_items = 0;
 	return ref_node;
 }
io_uring/rsrc.h +0 -1
index 17dfe180208f..88adcb0b7963 100644
@@ -48,7 +48,6 @@ struct io_rsrc_node {
 	struct list_head		node;
 	struct llist_node		llist;
 	int				refs;
-	bool				done;
 
 	/*
 	 * Keeps a list of struct io_rsrc_put to be completed. Each entry
Signature · 1 line
2.40.0
#3 lore raw

PATCH 3/10 io_uring/rsrc: refactor io_rsrc_ref_quiesce

In reply to Pavel Begunkov

Refactor io_rsrc_ref_quiesce() by moving the first mutex_unlock(), so we don't have to have a second mutex_unlock() further in the loop. It prepares us to the next patch.

---
 io_uring/rsrc.c | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)
io_uring/rsrc.c +5 -13
index 5fc9d10743e0..d7e7528f7159 100644
@@ -272,8 +272,8 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 		return 0;
 
 	data->quiesce = true;
-	mutex_unlock(&ctx->uring_lock);
 	do {
+		mutex_unlock(&ctx->uring_lock);
 		ret = io_run_task_work_sig(ctx);
 		if (ret < 0) {
 			mutex_lock(&ctx->uring_lock);
@@ -285,18 +285,10 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 			}
 			break;
 		}
-		ret = wait_for_completion_interruptible(&data->done);
-		if (!ret) {
-			mutex_lock(&ctx->uring_lock);
-			if (!data->refs)
-				break;
-			/*
-			 * it has been revived by another thread while
-			 * we were unlocked
-			 */
-			mutex_unlock(&ctx->uring_lock);
-		}
-	} while (1);
+		wait_for_completion_interruptible(&data->done);
+		mutex_lock(&ctx->uring_lock);
+		ret = 0;
+	} while (data->refs);
 	data->quiesce = false;
 
 	return ret;
Signature · 1 line
2.40.0
#4 lore raw

PATCH 4/10 io_uring/rsrc: use wq for quiescing

In reply to Pavel Begunkov

Replace completions with waitqueues for rsrc data quiesce, the main wakeup condition is when data refs hit zero. Note that data refs are only changes under ->uring_lock, so we prepare before mutex_unlock() reacquire it after taking the lock back. This change will be needed in the next patch.

---
 include/linux/io_uring_types.h |  1 +
 io_uring/io_uring.c            |  1 +
 io_uring/rsrc.c                | 18 ++++++++++++------
 io_uring/rsrc.h                |  1 -
 4 files changed, 14 insertions(+), 7 deletions(-)
include/linux/io_uring_types.h +1 -0
index 40cab420b1bd..5c9645319770 100644
@@ -333,6 +333,7 @@ struct io_ring_ctx {
 	/* protected by ->uring_lock */
 	struct list_head		rsrc_ref_list;
 	struct io_alloc_cache		rsrc_node_cache;
+	struct wait_queue_head		rsrc_quiesce_wq;
 
 	struct list_head		io_buffers_pages;
 
io_uring/io_uring.c +1 -0
index 9083a8466ebf..3c1c8c788b7b 100644
@@ -321,6 +321,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	mutex_init(&ctx->uring_lock);
 	init_waitqueue_head(&ctx->cq_wait);
 	init_waitqueue_head(&ctx->poll_wq);
+	init_waitqueue_head(&ctx->rsrc_quiesce_wq);
 	spin_lock_init(&ctx->completion_lock);
 	spin_lock_init(&ctx->timeout_lock);
 	INIT_WQ_LIST(&ctx->iopoll_list);
io_uring/rsrc.c +12 -6
index d7e7528f7159..f9ce4076c73d 100644
@@ -158,6 +158,7 @@ static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
 static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 {
 	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
+	struct io_ring_ctx *ctx = rsrc_data->ctx;
 	struct io_rsrc_put *prsrc, *tmp;
 
 	if (ref_node->inline_items)
@@ -171,13 +172,13 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 
 	io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
 	if (io_put_rsrc_data_ref(rsrc_data))
-		complete(&rsrc_data->done);
+		wake_up_all(&ctx->rsrc_quiesce_wq);
 }
 
 void io_wait_rsrc_data(struct io_rsrc_data *data)
 {
-	if (data && !io_put_rsrc_data_ref(data))
-		wait_for_completion(&data->done);
+	if (data)
+		WARN_ON_ONCE(!io_put_rsrc_data_ref(data));
 }
 
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
@@ -257,6 +258,7 @@ int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
 __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 				      struct io_ring_ctx *ctx)
 {
+	DEFINE_WAIT(we);
 	int ret;
 
 	/* As we may drop ->uring_lock, other task may have started quiesce */
@@ -273,7 +275,9 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 
 	data->quiesce = true;
 	do {
+		prepare_to_wait(&ctx->rsrc_quiesce_wq, &we, TASK_INTERRUPTIBLE);
 		mutex_unlock(&ctx->uring_lock);
+
 		ret = io_run_task_work_sig(ctx);
 		if (ret < 0) {
 			mutex_lock(&ctx->uring_lock);
@@ -285,12 +289,15 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 			}
 			break;
 		}
-		wait_for_completion_interruptible(&data->done);
+
+		schedule();
+		__set_current_state(TASK_RUNNING);
 		mutex_lock(&ctx->uring_lock);
 		ret = 0;
 	} while (data->refs);
-	data->quiesce = false;
 
+	finish_wait(&ctx->rsrc_quiesce_wq, &we);
+	data->quiesce = false;
 	return ret;
 }
 
@@ -366,7 +373,6 @@ __cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
 				goto fail;
 		}
 	}
-	init_completion(&data->done);
 	*pdata = data;
 	return 0;
 fail:
io_uring/rsrc.h +0 -1
index 88adcb0b7963..d93ba4e9742a 100644
@@ -35,7 +35,6 @@ struct io_rsrc_data {
 	u64				**tags;
 	unsigned int			nr;
 	rsrc_put_fn			*do_put;
-	struct completion		done;
 	int				refs;
 	bool				quiesce;
 };
Signature · 1 line
2.40.0
#5 lore raw

PATCH 5/10 io_uring/rsrc: fix DEFER_TASKRUN rsrc quiesce

In reply to Pavel Begunkov

For io_rsrc_ref_quiesce() to progress it should execute all task_work items, including deferred ones. However, currently nobody would wake us, and so let's set ctx->cq_wait_nr, so io_req_local_work_add() would wake us up.

Fixes c0e0d6ba25f18 ("io_uring: add IORING_SETUP_DEFER_TASKRUN")
---
 io_uring/rsrc.c | 9 +++++++++
 1 file changed, 9 insertions(+)
io_uring/rsrc.c +9 -0
index f9ce4076c73d..e634ef384724 100644
@@ -273,6 +273,11 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 	if (io_put_rsrc_data_ref(data))
 		return 0;
 
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
+		atomic_set(&ctx->cq_wait_nr, 1);
+		smp_mb();
+	}
+
 	data->quiesce = true;
 	do {
 		prepare_to_wait(&ctx->rsrc_quiesce_wq, &we, TASK_INTERRUPTIBLE);
@@ -298,6 +303,10 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 
 	finish_wait(&ctx->rsrc_quiesce_wq, &we);
 	data->quiesce = false;
+	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
+		atomic_set(&ctx->cq_wait_nr, 0);
+		smp_mb();
+	}
 	return ret;
 }
 
Signature · 1 line
2.40.0
#6 lore raw

PATCH 6/10 io_uring/rsrc: remove rsrc_data refs

In reply to Pavel Begunkov

Instead of waiting for rsrc_data->refs to be downed to zero, check whether there are rsrc nodes queued for completion, that's easier then maintaining references.

---
 include/linux/io_uring_types.h |  1 +
 io_uring/io_uring.c            |  4 ++--
 io_uring/rsrc.c                | 32 ++++++++------------------------
 io_uring/rsrc.h                |  2 --
 4 files changed, 11 insertions(+), 28 deletions(-)
include/linux/io_uring_types.h +1 -0
index 5c9645319770..1b2a20a42413 100644
@@ -334,6 +334,7 @@ struct io_ring_ctx {
 	struct list_head		rsrc_ref_list;
 	struct io_alloc_cache		rsrc_node_cache;
 	struct wait_queue_head		rsrc_quiesce_wq;
+	unsigned			rsrc_quiesce;
 
 	struct list_head		io_buffers_pages;
 
io_uring/io_uring.c +2 -2
index 3c1c8c788b7b..3d43df8f1e4e 100644
@@ -2831,8 +2831,8 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
 	io_sq_thread_finish(ctx);
 	/* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
-	io_wait_rsrc_data(ctx->buf_data);
-	io_wait_rsrc_data(ctx->file_data);
+	if (WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)))
+		return;
 
 	mutex_lock(&ctx->uring_lock);
 	if (ctx->buf_data)
io_uring/rsrc.c +8 -24
index e634ef384724..5415a18844e0 100644
@@ -31,11 +31,6 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
 #define IORING_MAX_FIXED_FILES	(1U << 20)
 #define IORING_MAX_REG_BUFFERS	(1U << 14)
 
-static inline bool io_put_rsrc_data_ref(struct io_rsrc_data *rsrc_data)
-{
-	return !--rsrc_data->refs;
-}
-
 int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
 {
 	unsigned long page_limit, cur_pages, new_pages;
@@ -158,7 +153,6 @@ static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
 static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 {
 	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
-	struct io_ring_ctx *ctx = rsrc_data->ctx;
 	struct io_rsrc_put *prsrc, *tmp;
 
 	if (ref_node->inline_items)
@@ -171,14 +165,6 @@ static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
 	}
 
 	io_rsrc_node_destroy(rsrc_data->ctx, ref_node);
-	if (io_put_rsrc_data_ref(rsrc_data))
-		wake_up_all(&ctx->rsrc_quiesce_wq);
-}
-
-void io_wait_rsrc_data(struct io_rsrc_data *data)
-{
-	if (data)
-		WARN_ON_ONCE(!io_put_rsrc_data_ref(data));
 }
 
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
@@ -201,6 +187,8 @@ void io_rsrc_node_ref_zero(struct io_rsrc_node *node)
 		list_del(&node->node);
 		__io_rsrc_put_work(node);
 	}
+	if (list_empty(&ctx->rsrc_ref_list) && unlikely(ctx->rsrc_quiesce))
+		wake_up_all(&ctx->rsrc_quiesce_wq);
 }
 
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
@@ -235,7 +223,6 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 	if (WARN_ON_ONCE(!backup))
 		return;
 
-	data_to_kill->refs++;
 	node->rsrc_data = data_to_kill;
 	list_add_tail(&node->node, &ctx->rsrc_ref_list);
 	/* put master ref */
@@ -269,8 +256,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 		return ret;
 	io_rsrc_node_switch(ctx, data);
 
-	/* kill initial ref */
-	if (io_put_rsrc_data_ref(data))
+	if (list_empty(&ctx->rsrc_ref_list))
 		return 0;
 
 	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
@@ -278,6 +264,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 		smp_mb();
 	}
 
+	ctx->rsrc_quiesce++;
 	data->quiesce = true;
 	do {
 		prepare_to_wait(&ctx->rsrc_quiesce_wq, &we, TASK_INTERRUPTIBLE);
@@ -286,12 +273,8 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 		ret = io_run_task_work_sig(ctx);
 		if (ret < 0) {
 			mutex_lock(&ctx->uring_lock);
-			if (!data->refs) {
+			if (list_empty(&ctx->rsrc_ref_list))
 				ret = 0;
-			} else {
-				/* restore the master reference */
-				data->refs++;
-			}
 			break;
 		}
 
@@ -299,10 +282,12 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data,
 		__set_current_state(TASK_RUNNING);
 		mutex_lock(&ctx->uring_lock);
 		ret = 0;
-	} while (data->refs);
+	} while (!list_empty(&ctx->rsrc_ref_list));
 
 	finish_wait(&ctx->rsrc_quiesce_wq, &we);
 	data->quiesce = false;
+	ctx->rsrc_quiesce--;
+
 	if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
 		atomic_set(&ctx->cq_wait_nr, 0);
 		smp_mb();
@@ -371,7 +356,6 @@ __cold static int io_rsrc_data_alloc(struct io_ring_ctx *ctx,
 	data->nr = nr;
 	data->ctx = ctx;
 	data->do_put = do_put;
-	data->refs = 1;
 	if (utags) {
 		ret = -EFAULT;
 		for (i = 0; i < nr; i++) {
io_uring/rsrc.h +0 -2
index d93ba4e9742a..5dd2fcb28069 100644
@@ -35,7 +35,6 @@ struct io_rsrc_data {
 	u64				**tags;
 	unsigned int			nr;
 	rsrc_put_fn			*do_put;
-	int				refs;
 	bool				quiesce;
 };
 
@@ -69,7 +68,6 @@ struct io_mapped_ubuf {
 void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_put_work(struct work_struct *work);
-void io_wait_rsrc_data(struct io_rsrc_data *data);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
 int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
Signature · 1 line
2.40.0
#7 lore raw

PATCH 7/10 io_uring/rsrc: inline switch_start fast path

In reply to Pavel Begunkov

Inline the part of io_rsrc_node_switch_start() that checks whether the cache is empty or not, as most of the times it will have some number of entries in there.

---
 io_uring/rsrc.c | 12 +++++-------
 io_uring/rsrc.h |  9 ++++++++-
 2 files changed, 13 insertions(+), 8 deletions(-)
io_uring/rsrc.c +5 -7
index 5415a18844e0..bfa0b382c6c6 100644
@@ -230,15 +230,13 @@ void io_rsrc_node_switch(struct io_ring_ctx *ctx,
 	ctx->rsrc_node = backup;
 }
 
-int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
+int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
 {
-	if (io_alloc_cache_empty(&ctx->rsrc_node_cache)) {
-		struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
+	struct io_rsrc_node *node = kzalloc(sizeof(*node), GFP_KERNEL);
 
-		if (!node)
-			return -ENOMEM;
-		io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
-	}
+	if (!node)
+		return -ENOMEM;
+	io_alloc_cache_put(&ctx->rsrc_node_cache, &node->cache);
 	return 0;
 }
 
io_uring/rsrc.h +8 -1
index 5dd2fcb28069..732496afed4c 100644
@@ -69,7 +69,7 @@ void io_rsrc_put_tw(struct callback_head *cb);
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_put_work(struct work_struct *work);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
-int io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
+int __io_rsrc_node_switch_start(struct io_ring_ctx *ctx);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
 int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 			  struct io_rsrc_node *node, void *rsrc);
@@ -111,6 +111,13 @@ static inline int io_scm_file_account(struct io_ring_ctx *ctx,
 	return __io_scm_file_account(ctx, file);
 }
 
+static inline int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
+{
+	if (unlikely(io_alloc_cache_empty(&ctx->rsrc_node_cache)))
+		return __io_rsrc_node_switch_start(ctx);
+	return 0;
+}
+
 int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
 			     unsigned nr_args);
 int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
Signature · 1 line
2.40.0
#8 lore raw

PATCH 8/10 io_uring/rsrc: clean up __io_sqe_buffers_update()

In reply to Pavel Begunkov

Inline offset variable, so we don't use it without subjecting it to array_index_nospec() first.

---
 io_uring/rsrc.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
io_uring/rsrc.c +1 -2
index bfa0b382c6c6..38f0c9ce67a7 100644
@@ -469,7 +469,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 
 	for (done = 0; done < nr_args; done++) {
 		struct io_mapped_ubuf *imu;
-		int offset = up->offset + done;
 		u64 tag = 0;
 
 		err = io_copy_iov(ctx, &iov, iovs, done);
@@ -490,7 +489,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		if (err)
 			break;
 
-		i = array_index_nospec(offset, ctx->nr_user_bufs);
+		i = array_index_nospec(up->offset + done, ctx->nr_user_bufs);
 		if (ctx->user_bufs[i] != ctx->dummy_ubuf) {
 			err = io_queue_rsrc_removal(ctx->buf_data, i,
 						    ctx->rsrc_node, ctx->user_bufs[i]);
Signature · 1 line
2.40.0
#9 lore raw

PATCH 9/10 io_uring/rsrc: simplify single file node switching

In reply to Pavel Begunkov

At maximum io_install_fixed_file() removes only one file, so no need to keep needs_switch state and we can call io_rsrc_node_switch() right after removal.

---
 io_uring/filetable.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)
io_uring/filetable.c +4 -7
index b80614e7d605..6255fa255ae2 100644
@@ -64,7 +64,6 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 				 u32 slot_index)
 	__must_hold(&req->ctx->uring_lock)
 {
-	bool needs_switch = false;
 	struct io_fixed_file *file_slot;
 	int ret;
 
@@ -83,16 +82,17 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 
 		ret = io_rsrc_node_switch_start(ctx);
 		if (ret)
-			goto err;
+			return ret;
 
 		old_file = (struct file *)(file_slot->file_ptr & FFS_MASK);
 		ret = io_queue_rsrc_removal(ctx->file_data, slot_index,
 					    ctx->rsrc_node, old_file);
 		if (ret)
-			goto err;
+			return ret;
+
 		file_slot->file_ptr = 0;
 		io_file_bitmap_clear(&ctx->file_table, slot_index);
-		needs_switch = true;
+		io_rsrc_node_switch(ctx, ctx->file_data);
 	}
 
 	ret = io_scm_file_account(ctx, file);
@@ -101,9 +101,6 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 		io_fixed_file_set(file_slot, file);
 		io_file_bitmap_set(&ctx->file_table, slot_index);
 	}
-err:
-	if (needs_switch)
-		io_rsrc_node_switch(ctx, ctx->file_data);
 	return ret;
 }
 
Signature · 1 line
2.40.0
#10 lore raw

PATCH 10/10 io_uring/rsrc: refactor io_queue_rsrc_removal

In reply to Pavel Begunkov

We can queue up a rsrc into a list in io_queue_rsrc_removal() while allocating io_rsrc_put and so simplify the function.

---
 io_uring/rsrc.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)
io_uring/rsrc.c +1 -4
index 38f0c9ce67a7..db58a51d19da 100644
@@ -685,7 +685,6 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 {
 	u64 *tag_slot = io_get_tag_slot(data, idx);
 	struct io_rsrc_put *prsrc;
-	bool inline_item = true;
 
 	if (!node->inline_items) {
 		prsrc = &node->item;
@@ -694,14 +693,12 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
 		prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
 		if (!prsrc)
 			return -ENOMEM;
-		inline_item = false;
+		list_add(&prsrc->list, &node->item_list);
 	}
 
 	prsrc->tag = *tag_slot;
 	*tag_slot = 0;
 	prsrc->rsrc = rsrc;
-	if (!inline_item)
-		list_add(&prsrc->list, &node->item_list);
 	return 0;
 }
 
Signature · 1 line
2.40.0
maintainer Applied
#11 lore raw
In reply to Pavel Begunkov
Quoted 9 lines · Pavel Begunkov

Patch 1 is a simple fix for using indexes w/o array_index_nospec() protection.

Patches 2-5 are fixing a file / buffer table unregistration issue when the ring is configured with DEFER_TASKRUN.

The rest are clean ups on top.

[...]

Applied, thanks!

[01/10] io_uring/rsrc: use nospec'ed indexes

        commit: 953c37e066f05a3dca2d74643574b8dfe8a83983
[02/10] io_uring/rsrc: remove io_rsrc_node::done
        commit: c732ea242d565c8281c4b017929fc62a246d81b9
[03/10] io_uring/rsrc: refactor io_rsrc_ref_quiesce
        commit: eef81fcaa61e1bc6b7735be65f41bbf1a8efd133
[04/10] io_uring/rsrc: use wq for quiescing
        commit: 4ea15b56f0810f0d8795d475db1bb74b3a7c1b2f
[05/10] io_uring/rsrc: fix DEFER_TASKRUN rsrc quiesce
        commit: 7d481e0356334eb2de254414769b4bed4b2a8827
[06/10] io_uring/rsrc: remove rsrc_data refs
        commit: 0b222eeb6514ba6c3457b667fa4f3645032e1fc9
[07/10] io_uring/rsrc: inline switch_start fast path
        commit: 2f2af35f8e5a1ed552ed02e47277d50092a2b9f6
[08/10] io_uring/rsrc: clean up __io_sqe_buffers_update()
        commit: 9a57fffedc0ee078418a7793ab29cd3864205340
[09/10] io_uring/rsrc: simplify single file node switching
        commit: c87fd583f3b5ef770af33893394ea37c7a10b5b8
[10/10] io_uring/rsrc: refactor io_queue_rsrc_removal
        commit: c899a5d7d0eca054546b63e95c94b1e609516f84

Best regards,

Signature · 1 line
Jens Axboe