poll fixes
cover poll fixes
Two fixes about poll.
Hao Xu (2):
io_uring: fix tw list mess-up by adding tw while it's already in tw
list
io_uring: fix race between poll completion and cancel_hash insertion
fs/io_uring.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)
Signature · 1 line
2.24.4
PATCH 1/2 io_uring: fix tw list mess-up by adding tw while it's already in tw list
For multishot mode, there may be cases like:io_poll_task_func()
-> add_wait_queue()
async_wake()
->io_req_task_work_add()
this one mess up the running task_work list
since req->io_task_work.node is in use.similar situation for req->io_task_work.fallback_node.
Fix it by set node->next = NULL before we run the tw, so that when we add req back to the wait queue in middle of tw running, we can safely re-add it to the tw list.
---
fs/io_uring.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)
fs/io_uring.c +8 -3
@@ -1216,13 +1216,17 @@ static void io_fallback_req_func(struct work_struct *work) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work);struct llist_node *node =llist_del_all (&ctx->fallback_llist); -struct io_kiocb *req, *tmp; +struct io_kiocb *req;bool locked = false;percpu_ref_get (&ctx->refs); -llist_for_each_entry_safe (req, tmp, node, io_task_work.fallback_node) + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); +while (member_address_is_nonnull (req, io_task_work.fallback_node)) { + node = req->io_task_work.fallback_node.next; + req->io_task_work.fallback_node.next =NULL ; req->io_task_work.func (req, &locked); - + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + }if (locked) {if (ctx->submit_state.compl_nr)io_submit_flush_completions (ctx);
@@ -2126,6 +2130,7 @@ static void tctx_task_work(struct callback_head *cb) locked =mutex_trylock (&ctx->uring_lock);percpu_ref_get (&ctx->refs); } + node->next =NULL ; req->io_task_work.func (req, &locked); node = next; }while (node);
Signature · 1 line
2.24.4
Re: [PATCH 1/2] io_uring: fix tw list mess-up by adding tw while it's already in tw list
Quoted 8 lines · Hao Xu
For multishot mode, there may be cases like:
io_poll_task_func()
->add_wait_queue()async_wake() ->io_req_task_work_add() this one mess up the running task_work list since req->io_task_work.node is in use.
Quoted 4 lines
similar situation for
req->io_task_work.fallback_node.
Fix it by setnode->next= NULL before we run the tw, so that when we add req back to the wait queue in middle of tw running, we can safely re-add it to the tw list.
It may get screwed before we get to "node->next = NULL;",
-> async_wake()
-> io_req_task_work_add()
-> async_wake()
-> io_req_task_work_add()tctx_task_work()
Quoted 41 lines
Fixes 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work")Signed-off-by Hao Xu <[email protected]>---
fs/io_uring.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)fs/io_uring.c +8
-3@@ -1216,13 +1216,17 @@ static void io_fallback_req_func(struct work_struct *work) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work);struct llist_node *node =llist_del_all (&ctx->fallback_llist); -struct io_kiocb *req, *tmp; +struct io_kiocb *req;bool locked = false;percpu_ref_get (&ctx->refs); -llist_for_each_entry_safe (req, tmp, node, io_task_work.fallback_node) + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); +while (member_address_is_nonnull (req, io_task_work.fallback_node)) { + node = req->io_task_work.fallback_node.next; + req->io_task_work.fallback_node.next =NULL ; req->io_task_work.func (req, &locked); - + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + }if (locked) {if (ctx->submit_state.compl_nr)io_submit_flush_completions (ctx);@@ -2126,6 +2130,7 @@ static void tctx_task_work(struct callback_head *cb) locked =mutex_trylock (&ctx->uring_lock);percpu_ref_get (&ctx->refs); } + node->next =NULL ; req->io_task_work.func (req, &locked); node = next; }while (node);
Signature · 1 line
Pavel Begunkov
Re: [PATCH 1/2] io_uring: fix tw list mess-up by adding tw while it's already in tw list
Quoted 14 lines · Pavel Begunkov
Quoted 8 lines · Hao Xu
For multishot mode, there may be cases like:
io_poll_task_func()
->add_wait_queue()async_wake() ->io_req_task_work_add() this one mess up the running task_work list since req->io_task_work.node is in use.Quoted 4 lines
similar situation for
req->io_task_work.fallback_node.
Fix it by setnode->next= NULL before we run the tw, so that when we add req back to the wait queue in middle of tw running, we can safely re-add it to the tw list.
Quoted 7 lines
It may get screwed before we get to "
node->next= NULL;",->
async_wake()
->io_req_task_work_add()
->async_wake()
->io_req_task_work_add()tctx_task_work()
True, this may happen if there is second poll wait entry. This pacth is for single wait entry only..
I'm thinking about the second poll entry issue, would be in a separate patch.
Quoted 44 lines
Quoted 43 lines
Fixes 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work")Signed-off-by Hao Xu <[email protected]>---
fs/io_uring.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)fs/io_uring.c +8
-3@@ -1216,13 +1216,17 @@ static void io_fallback_req_func(struct work_struct *work) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work);struct llist_node *node =llist_del_all (&ctx->fallback_llist); -struct io_kiocb *req, *tmp; +struct io_kiocb *req;bool locked = false;percpu_ref_get (&ctx->refs); -llist_for_each_entry_safe (req, tmp, node, io_task_work.fallback_node) + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); +while (member_address_is_nonnull (req, io_task_work.fallback_node)) { + node = req->io_task_work.fallback_node.next; + req->io_task_work.fallback_node.next =NULL ; req->io_task_work.func (req, &locked); - + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + }if (locked) {if (ctx->submit_state.compl_nr)io_submit_flush_completions (ctx);@@ -2126,6 +2130,7 @@ static void tctx_task_work(struct callback_head *cb) locked =mutex_trylock (&ctx->uring_lock);percpu_ref_get (&ctx->refs); } + node->next =NULL ; req->io_task_work.func (req, &locked); node = next; }while (node);
Re: [PATCH 1/2] io_uring: fix tw list mess-up by adding tw while it's already in tw list
Quoted 22 lines · Hao Xu
Quoted 14 lines · Pavel Begunkov
Quoted 8 lines · Hao Xu
For multishot mode, there may be cases like:
io_poll_task_func()
->add_wait_queue()
async_wake()->io_req_task_work_add()
this one mess up the runningtask_worklist sincereq->io_task_work.nodeis in use.Quoted 4 lines
similar situation for
req->io_task_work.fallback_node.
Fix it by setnode->next= NULL before we run the tw, so that when we add req back to the wait queue in middle of tw running, we can safely re-add it to the tw list.Quoted 7 lines
It may get screwed before we get to "
node->next= NULL;",->
async_wake()
->io_req_task_work_add()
->async_wake()
->io_req_task_work_add()tctx_task_work()
Quoted 4 lines
True, this may happen if there is second poll wait entry. This pacth is for single wait entry only..
I'm thinking about the second poll entry issue, would be in a separate patch.
hmm, reviewed this email again and now I think I got what you were saying, do you mean the second async_wake() triggered before we removed the wait entry in the first async_wake(), like
async_wake
async_wake
->del wait entry
Quoted 49 lines
Quoted 49 lines
Quoted 48 lines
Fixes 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work")Signed-off-by Hao Xu <[email protected]>---
fs/io_uring.c| 11 ++++++++---1 file changed, 8 insertions(+), 3 deletions(-)fs/io_uring.c +6
-3@@ -1216,13 +1216,17 @@ static void io_fallback_req_func(struct work_struct *work) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work); struct llist_node *node = llist_del_all(&ctx->fallback_llist); - struct io_kiocb *req, *tmp; + struct io_kiocb *req; bool locked = false; percpu_ref_get(&ctx->refs); - llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node) + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + while (member_address_is_nonnull(req, io_task_work.fallback_node)) { + node = req->io_task_work.fallback_node.next; + req->io_task_work.fallback_node.next = NULL; req->io_task_work.func(req, &locked); - + req = llist_entry(node, struct io_kiocb,io_task_work.fallback_node);
+ }
if (locked) {
if (ctx->submit_state.compl_nr)
io_submit_flush_completions(ctx);fs/io_uring.c +1
-0@@ -2126,6 +2130,7 @@ static void tctx_task_work(struct callback_head *cb) locked = mutex_trylock(&ctx->uring_lock); percpu_ref_get(&ctx->refs); } + node->next = NULL; req->io_task_work.func(req, &locked); node = next;} while (node);
Re: [PATCH 1/2] io_uring: fix tw list mess-up by adding tw while it's already in tw list
Quoted 27 lines · Hao Xu
Quoted 22 lines · Hao Xu
Quoted 14 lines · Pavel Begunkov
Quoted 8 lines · Hao Xu
For multishot mode, there may be cases like:
io_poll_task_func()
->add_wait_queue()
async_wake()->io_req_task_work_add()
this one mess up the runningtask_worklist sincereq->io_task_work.nodeis in use.Quoted 4 lines
similar situation for
req->io_task_work.fallback_node.
Fix it by setnode->next= NULL before we run the tw, so that when we add req back to the wait queue in middle of tw running, we can safely re-add it to the tw list.Quoted 7 lines
It may get screwed before we get to "
node->next= NULL;",->
async_wake()
->io_req_task_work_add()
->async_wake()
->io_req_task_work_add()tctx_task_work()Quoted 4 lines
True, this may happen if there is second poll wait entry. This pacth is for single wait entry only..
I'm thinking about the second poll entry issue, would be in a separate patch.
Quoted 7 lines
hmm, reviewed this email again and now I think I got what you were saying, do you mean the second
async_wake()triggered before we removed the wait entry in the firstasync_wake(), like
async_wake
async_wake
->del wait entry
Looks we had different problems in mind, let's move the conversation to the new thread with resent patches
Quoted 42 lines
Quoted 41 lines
Quoted 41 lines
Quoted 40 lines
Fixes 7cbf1722d5fc ("io_uring: provide FIFO ordering for task_work")Signed-off-by Hao Xu <[email protected]>---
fs/io_uring.c| 11 ++++++++---1 file changed, 8 insertions(+), 3 deletions(-)fs/io_uring.c +8
-3@@ -1216,13 +1216,17 @@ static void io_fallback_req_func(struct work_struct *work) struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, fallback_work.work); struct llist_node *node = llist_del_all(&ctx->fallback_llist); - struct io_kiocb *req, *tmp; + struct io_kiocb *req; bool locked = false; percpu_ref_get(&ctx->refs); - llist_for_each_entry_safe(req, tmp, node, io_task_work.fallback_node) + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + while (member_address_is_nonnull(req, io_task_work.fallback_node)) { + node = req->io_task_work.fallback_node.next; + req->io_task_work.fallback_node.next = NULL; req->io_task_work.func(req, &locked); - + req = llist_entry(node, struct io_kiocb, io_task_work.fallback_node); + } if (locked) { if (ctx->submit_state.compl_nr) io_submit_flush_completions(ctx);@@ -2126,6 +2130,7 @@ static void tctx_task_work(struct callback_head *cb) locked = mutex_trylock(&ctx->uring_lock); percpu_ref_get(&ctx->refs); } + node->next = NULL; req->io_task_work.func(req, &locked); node = next; } while (node);
Signature · 1 line
Pavel Begunkov
PATCH 2/2 io_uring: fix race between poll completion and cancel_hash insertion
In reply to Hao XuIf poll arming and poll completion runs parallelly, there maybe races. For instance, run io_poll_add in iowq and io_poll_task_func in original context, then:
iowq original context
io_poll_add
vfs_poll
(interruption happens
tw queued to original
context) io_poll_task_func
generate cqe
del from cancel_hash[]
if !poll.done
insert to cancel_hash[]The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
---
Didn't find the exact commit to add Fixes: for..
fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
fs/io_uring.c +3 -3
@@ -5118,10 +5118,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;
@@ -5152,6 +5150,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);
@@ -5289,6 +5288,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + req->poll.done = true;spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))
Signature · 1 line
2.24.4
Re: [PATCH 2/2] io_uring: fix race between poll completion and cancel_hash insertion
Quoted 14 lines · Hao Xu
If poll arming and poll completion runs parallelly, there maybe races. For instance, run
io_poll_addin iowq andio_poll_task_funcin original context, then:
iowq original context
io_poll_addvfs_poll (interruption happens tw queued to original context) io_poll_task_func generate cqe del from cancel_hash[] if !poll.done insert to cancel_hash[]
Quoted 2 lines
The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
Sounds like a valid concern. And the code looks good, but one of two patches crashed the kernel somewhere in io_read().
./232c93d07b74-test
will be retesting
Quoted 4 lines
Signed-off-by Hao Xu <[email protected]>---
Didn't find the exact commit to add Fixes: for..
That's ok. Maybe you can find which kernel versions are affected? So we can add stable and specify where it should be backported? E.g.
Quoted 36 lines
fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)fs/io_uring.c +3
-3@@ -5118,10 +5118,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;@@ -5152,6 +5150,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);@@ -5289,6 +5288,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + req->poll.done = true;spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/2] io_uring: fix race between poll completion and cancel_hash insertion
Quoted 15 lines · Pavel Begunkov
Quoted 14 lines · Hao Xu
If poll arming and poll completion runs parallelly, there maybe races. For instance, run
io_poll_addin iowq andio_poll_task_funcin original context, then:
iowq original context
io_poll_addvfs_poll (interruption happens tw queued to original context) io_poll_task_func generate cqe del from cancel_hash[] if !poll.done insert to cancel_hash[]
Quoted 7 lines
Quoted 2 lines
The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
Sounds like a valid concern. And the code looks good, but one of two patches crashed the kernel somewhere in
io_read()../232c93d07b74-test
I'll run it too, didn't meet that when I did the test.
Quoted 4 lines
will be retesting
Quoted 8 lines
Quoted 4 lines
Signed-off-by Hao Xu <[email protected]>---
Didn't find the exact commit to add Fixes: for..
That's ok. Maybe you can find which kernel versions are affected? So we can add stable and specify where it should be backported? E.g.
Sure, will do that.
Quoted 41 lines
Cc [email protected] # 5.12+Quoted 36 lines
fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)fs/io_uring.c +3
-3@@ -5118,10 +5118,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;@@ -5152,6 +5150,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);@@ -5289,6 +5288,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + req->poll.done = true;spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))
Re: [PATCH 2/2] io_uring: fix race between poll completion and cancel_hash insertion
In reply to Hao XuQuoted 50 lines · Hao Xu
If poll arming and poll completion runs parallelly, there maybe races. For instance, run
io_poll_addin iowq andio_poll_task_funcin original context, then:
iowq original context
io_poll_addvfs_poll (interruption happens tw queued to original context) io_poll_task_func generate cqe del from cancel_hash[] if !poll.done insert to cancel_hash[]The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
Signed-off-by Hao Xu <[email protected]>---
Didn't find the exact commit to add Fixes: for..
fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)fs/io_uring.c +2
-3@@ -5118,10 +5118,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;@@ -5152,6 +5150,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);
fs/io_uring.c static void io_async_task_func(struct io_kiocb *req, bool *locked) · line 5291
Quoted 4 lines
fs/io_uring.c +1
-0@@ -5289,6 +5288,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + req->poll.done = true;
Only poll request has req->poll. E.g. it overwrites parts of req->rw.kiocb, I guess .ki_complete in particular.
struct async_poll *apoll = req->apoll;apoll->poll.done = true;
Quoted 4 lines
fs/io_uring.c +0
-0spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))
Signature · 1 line
Pavel Begunkov
Re: [PATCH 2/2] io_uring: fix race between poll completion and cancel_hash insertion
Quoted 59 lines · Pavel Begunkov
Quoted 50 lines · Hao Xu
If poll arming and poll completion runs parallelly, there maybe races. For instance, run
io_poll_addin iowq andio_poll_task_funcin original context, then:
iowq original context
io_poll_addvfs_poll (interruption happens tw queued to original context) io_poll_task_func generate cqe del from cancel_hash[] if !poll.done insert to cancel_hash[]The entry left in cancel_hash[], similar case for fast poll. Fix it by set poll.done = true when del from cancel_hash[].
Signed-off-by Hao Xu <[email protected]>---
Didn't find the exact commit to add Fixes: for..
fs/io_uring.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)fs/io_uring.c +2
-3@@ -5118,10 +5118,8 @@ static bool __io_poll_complete(struct io_kiocb *req, __poll_t mask) }if (req->poll.events &EPOLLONESHOT ) flags =0 ; -if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) { - req->poll.done = true; +if (!io_cqring_fill_event (ctx, req->user_data, error, flags)) flags =0 ; - }if (flags &IORING_CQE_F_MORE ) ctx->cq_extra++;@@ -5152,6 +5150,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)if (done) {io_poll_remove_double (req);hash_del (&req->hash_node); + req->poll.done = true; }else { req->result =0 ;add_wait_queue (req->poll.head, &req->poll.wait);fs/io_uring.c static void io_async_task_func(struct io_kiocb *req, bool *locked) · line 5291
Quoted 4 lines
fs/io_uring.c +1
-0@@ -5289,6 +5288,7 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)hash_del (&req->hash_node);io_poll_remove_double (req); + req->poll.done = true;Only poll request has
req->poll. E.g. it overwrites parts ofreq->rw.kiocb, I guess .ki_complete in particular.
Quoted 2 lines
struct
async_poll*apoll =req->apoll;apoll->poll.done= true;
Thanks!
Quoted 7 lines
Quoted 6 lines
fs/io_uring.c +0
-0spin_unlock (&ctx->completion_lock);if (!READ_ONCE (apoll->poll.canceled))