commit/posted fixes
cover commit/posted fixes
Looks like I accidentially didn't send this series.
Regarding mb in 3/3, I have a for-next patch removing it (one mb less in total), just prefer it to settle a bit so there is more time to be reported if anything goes wrong.
Pavel Begunkov (3):
io_uring: trigger eventfd for IOPOLL
io_uring: dont kill fasync under completion_lock
io_uring: synchronise ev_posted() with waitqueues
fs/io_uring.c | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-)
Signature · 1 line
2.24.0
PATCH 1/3 io_uring: trigger eventfd for IOPOLL
Make sure io_iopoll_complete() tries to wake up eventfd, which currently is skipped together with io_cqring_ev_posted() for non-SQPOLL IOPOLL.
Add an iopoll version of io_cqring_ev_posted(), duplicates a bit of code, but they actually use different sets of wait queues may be for better.
--- fs/io_uring.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-)
fs/io_uring.c +11 -2
@@ -1713,6 +1713,16 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx)eventfd_signal (ctx->cq_ev_fd,1 ); } +static void io_cqring_ev_posted_iopoll (struct io_ring_ctx *ctx) +{ +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (waitqueue_active (&ctx->wait)) +wake_up (&ctx->wait); + } +if (io_should_trigger_evfd (ctx)) +eventfd_signal (ctx->cq_ev_fd,1 ); +} +/* Returns true if there are no backlogged entries after the flush */ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force, struct task_struct *tsk,
@@ -2428,8 +2438,7 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, }io_commit_cqring (ctx); -if (ctx->flags &IORING_SETUP_SQPOLL ) -io_cqring_ev_posted (ctx); +io_cqring_ev_posted_iopoll (ctx);io_req_free_batch_finish (ctx, &rb);if (!list_empty (&again))
Signature · 1 line
2.24.0
PATCH 2/3 io_uring: dont kill fasync under completion_lock
In reply to Pavel Begunkov CPU0 CPU1
---- ----
lock(&new->fa_lock);
local_irq_disable();
lock(&ctx->completion_lock);
lock(&new->fa_lock);
<Interrupt>
lock(&ctx->completion_lock);*** DEADLOCK ***
Move kill_fasync() out of io_commit_cqring() to io_cqring_ev_posted(), so it doesn't hold completion_lock while doing it. That saves from the reported deadlock, and it's just nice to shorten the locking time and untangle nested locks (compl_lock -> wq_head::lock).
--- fs/io_uring.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
fs/io_uring.c +8 -5
@@ -1345,11 +1345,6 @@ static void __io_commit_cqring(struct io_ring_ctx *ctx)/* order cqe stores with ring update */ smp_store_release (&rings->cq.tail, ctx->cached_cq_tail); - -if (wq_has_sleeper (&ctx->cq_wait)) { -wake_up_interruptible (&ctx->cq_wait); -kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); - } }static void io_put_identity (struct io_uring_task *tctx,struct io_kiocb *req)
@@ -1711,6 +1706,10 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx)wake_up (&ctx->sq_data->wait);if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); +if (wq_has_sleeper (&ctx->cq_wait)) { +wake_up_interruptible (&ctx->cq_wait); +kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); + } }static void io_cqring_ev_posted_iopoll (struct io_ring_ctx *ctx)
@@ -1721,6 +1720,10 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) }if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); +if (wq_has_sleeper (&ctx->cq_wait)) { +wake_up_interruptible (&ctx->cq_wait); +kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); + } }/* Returns true if there are no backlogged entries after the flush */
Signature · 1 line
2.24.0
PATCH 3/3 io_uring: synchronise ev_posted() with waitqueues
In reply to Pavel Begunkovwaitqueue_active() needs smp_mb() to be in sync with waitqueues modification, but we miss it in io_cqring_ev_posted*() apart from cq_wait() case.
Take an smb_mb() out of wq_has_sleeper() making it waitqueue_active(), and place it a few lines before, so it can synchronise other waitqueue_active() as well.
The patch doesn't add any additional overhead, so even if there are no problems currently, it's just safer to have it this way.
--- fs/io_uring.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-)
fs/io_uring.c +8 -2
@@ -1700,13 +1700,16 @@ static inline unsigned __io_cqring_events(struct io_ring_ctx *ctx)static void io_cqring_ev_posted (struct io_ring_ctx *ctx) { +/* see waitqueue_active() comment */ +smp_mb (); +if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait);if (ctx->sq_data &&waitqueue_active (&ctx->sq_data->wait))wake_up (&ctx->sq_data->wait);if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); -if (wq_has_sleeper (&ctx->cq_wait)) { +if (waitqueue_active (&ctx->cq_wait)) {wake_up_interruptible (&ctx->cq_wait);kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); }
@@ -1714,13 +1717,16 @@ static void io_cqring_ev_posted(struct io_ring_ctx *ctx)static void io_cqring_ev_posted_iopoll (struct io_ring_ctx *ctx) { +/* see waitqueue_active() comment */ +smp_mb (); +if (ctx->flags &IORING_SETUP_SQPOLL ) {if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait); }if (io_should_trigger_evfd (ctx))eventfd_signal (ctx->cq_ev_fd,1 ); -if (wq_has_sleeper (&ctx->cq_wait)) { +if (waitqueue_active (&ctx->cq_wait)) {wake_up_interruptible (&ctx->cq_wait);kill_fasync (&ctx->cq_fasync,SIGIO ,POLL_IN ); }
Signature · 1 line
2.24.0
Quoted 6 lines · Pavel Begunkov
Looks like I accidentially didn't send this series.
Regarding mb in 3/3, I have a for-next patch removing it (one mb less in total), just prefer it to settle a bit so there is more time to be reported if anything goes wrong.
Applied, thanks.
Signature · 1 line
Jens Axboe