Skip to contentSkip to conversation

io_uring: protect remaining lockless ctx->rings accesses with RCU

PATCH In review Jens Axboe 2 participants 3 messages Mar 31 – Apr 1 io_uring

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

PATCH v2 1/1 io_uring: protect remaining lockless ctx->rings accesses with RCU

Commit 96189080265e addressed one case of ctx->rings being potentially accessed while a resize is happening on the ring, but there are still a few others that need handling. Add a helper for retrieving the rings associated with an io_uring context, and add some sanity checking to that to catch bad uses. ->rings_rcu is always valid, as long as it's used within RCU read lock. Any use of ->rings_rcu or ->rings inside either ->uring_lock or ->completion_lock is sane as well.

Do the minimum fix for the current kernel, but set it up such that this basic infra can be extended for later kernels to make this harder to mess up in the future.

Thanks to Junxi Qian for finding and debugging this issue.

Fixes 79cfe9e59c2a ("io_uring/register: add IORING_REGISTER_RESIZE_RINGS")

---

io_uring/io_uring.c +5 -2
index 20ec8fdafcae..48f2f627319d 100644
@@ -2015,7 +2015,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 	if (ctx->flags & IORING_SETUP_SQ_REWIND)
 		entries = ctx->sq_entries;
 	else
-		entries = io_sqring_entries(ctx);
+		entries = __io_sqring_entries(ctx);
 
 	entries = min(nr, entries);
 	if (unlikely(!entries))
@@ -2250,7 +2250,9 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)
 	 */
 	poll_wait(file, &ctx->poll_wq, wait);
 
-	if (!io_sqring_full(ctx))
+	rcu_read_lock();
+
+	if (!__io_sqring_full(ctx))
 		mask |= EPOLLOUT | EPOLLWRNORM;
 
 	/*
@@ -2270,6 +2272,7 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait)
 	if (__io_cqring_events_user(ctx) || io_has_work(ctx))
 		mask |= EPOLLIN | EPOLLRDNORM;
 
+	rcu_read_unlock();
 	return mask;
 }
 
io_uring/io_uring.h +29 -5
index 0fa844faf287..ee24bc5d77b3 100644
@@ -142,16 +142,28 @@ struct io_wait_queue {
 #endif
 };
 
+static inline struct io_rings *io_get_rings(struct io_ring_ctx *ctx)
+{
+	return rcu_dereference_check(ctx->rings_rcu,
+			lockdep_is_held(&ctx->uring_lock) ||
+			lockdep_is_held(&ctx->completion_lock));
+}
+
 static inline bool io_should_wake(struct io_wait_queue *iowq)
 {
 	struct io_ring_ctx *ctx = iowq->ctx;
-	int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;
+	struct io_rings *rings;
+	int dist;
+
+	guard(rcu)();
+	rings = io_get_rings(ctx);
 
 	/*
 	 * Wake up if we have enough events, or if a timeout occurred since we
 	 * started waiting. For timeouts, we always want to return to userspace,
 	 * regardless of event count.
 	 */
+	dist = READ_ONCE(rings->cq.tail) - (int) iowq->cq_tail;
 	return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
 }
 
@@ -431,9 +443,9 @@ static inline void io_cqring_wake(struct io_ring_ctx *ctx)
 	__io_wq_wake(&ctx->cq_wait);
 }
 
-static inline bool io_sqring_full(struct io_ring_ctx *ctx)
+static inline bool __io_sqring_full(struct io_ring_ctx *ctx)
 {
-	struct io_rings *r = ctx->rings;
+	struct io_rings *r = io_get_rings(ctx);
 
 	/*
 	 * SQPOLL must use the actual sqring head, as using the cached_sq_head
@@ -445,9 +457,15 @@ static inline bool io_sqring_full(struct io_ring_ctx *ctx)
 	return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries;
 }
 
-static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
+static inline bool io_sqring_full(struct io_ring_ctx *ctx)
 {
-	struct io_rings *rings = ctx->rings;
+	guard(rcu)();
+	return __io_sqring_full(ctx);
+}
+
+static inline unsigned int __io_sqring_entries(struct io_ring_ctx *ctx)
+{
+	struct io_rings *rings = io_get_rings(ctx);
 	unsigned int entries;
 
 	/* make sure SQ entry isn't read before tail */
@@ -455,6 +473,12 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
 	return min(entries, ctx->sq_entries);
 }
 
+static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
+{
+	guard(rcu)();
+	return __io_sqring_entries(ctx);
+}
+
 /*
  * Don't complete immediately but use deferred completion infrastructure.
  * Protected by ->uring_lock and can only be used either with
io_uring/wait.c +31 -19

92 lines not on this page · raw message

io_uring/wait.h +5 -2
index 5e236f74e1af..3a145fcfd3dd 100644
@@ -28,12 +28,15 @@ void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx);
 
 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
 {
-	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
+	struct io_rings *rings = io_get_rings(ctx);
+	return ctx->cached_cq_tail - READ_ONCE(rings->cq.head);
 }
 
 static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
 {
-	return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
+	struct io_rings *rings = io_get_rings(ctx);
+
+	return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);
 }
 
 /*
Signature · 1 line
Jens Axboe
#1 lore raw

Hi Jens,

The kernel test robot reported sparse warnings on your commit (9d0a7bda72c5):

io_uring/wait.c:309:49: sparse: cast removes address space '__rcu' of expression io_uring/wait.c:319:16: sparse: cast removes address space '__rcu' of expression io_uring/wait.c:319:54: sparse: cast removes address space '__rcu' of expression

The issue is that wait.c lines 309 and 319 access ctx->rings_rcu directly without going through rcu_dereference(), which sparse flags as an __rcu address space violation.

The fix would be to replace the raw ctx->rings_rcu dereferences with io_get_rings(ctx), e.g.:

READ_ONCE(ctx->rings_rcu->cq.tail)

READ_ONCE(io_get_rings(ctx)->cq.tail)

Apologies for not catching this during my review of v2.

Junxi Qian

author maintainer
#2 lore raw
Quoted 12 lines · junxi qian

Hi Jens,

The kernel test robot reported sparse warnings on your commit (9d0a7bda72c5):

io_uring/wait.c:309:49: sparse: cast removes address space '__rcu' of expression io_uring/wait.c:319:16: sparse: cast removes address space '__rcu' of expression io_uring/wait.c:319:54: sparse: cast removes address space '__rcu' of expression

The issue is that wait.c lines 309 and 319 access ctx->rings_rcu directly without going through rcu_dereference(), which sparse flags as an __rcu address space violation.

Quoted 8 lines

The fix would be to replace the raw ctx->rings_rcu dereferences with io_get_rings(ctx), e.g.:

READ_ONCE(ctx->rings_rcu->cq.tail)

READ_ONCE(io_get_rings(ctx)->cq.tail)

Apologies for not catching this during my review of v2.

Indeed, might just fold that in. It's just a sparse complaint, but might as well get it sorted before it goes upstream.

Signature · 1 line
Jens Axboe