Skip to contentSkip to conversation

io_uring: simplify the SQPOLL thread check when cancelling requests

PATCH Superseded Bui Quang Minh 3 participants 7 messages Jan 12, 2025 – Jan 13, 2025 io_uring

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 io_uring: simplify the SQPOLL thread check when cancelling requests

In io_uring_try_cancel_requests, we check whether sq_data->thread == current to determine if the function is called by the SQPOLL thread to do iopoll when IORING_SETUP_SQPOLL is set. This check can race with the SQPOLL thread termination.

io_uring_cancel_generic is used in 2 places: io_uring_cancel_generic and io_ring_exit_work. In io_uring_cancel_generic, we have the information whether the current is SQPOLL thread already. In io_ring_exit_work, in case the SQPOLL thread reaches this path, we don't need to iopoll and leave that for io_uring_cancel_generic to handle.

So to avoid the racy check, this commit adds a boolean flag to io_uring_try_cancel_requests to determine if we need to do iopoll inside the function and only sets this flag in io_uring_cancel_generic when the current is SQPOLL thread.

---
 io_uring/io_uring.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)
io_uring/io_uring.c +15 -6
index ff691f37462c..f28ea1254143 100644
@@ -143,7 +143,8 @@ struct io_defer_entry {
 
 static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct io_uring_task *tctx,
-					 bool cancel_all);
+					 bool cancel_all,
+					 bool force_iopoll);
 
 static void io_queue_sqe(struct io_kiocb *req);
 
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct work_struct *work)
 		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
 			io_move_task_work_from_local(ctx);
 
-		while (io_uring_try_cancel_requests(ctx, NULL, true))
+		/*
+		 * Even if SQPOLL thread reaches this path, don't force
+		 * iopoll here, let the io_uring_cancel_generic handle
+		 * it.
+		 */
+		while (io_uring_try_cancel_requests(ctx, NULL, true, false))
 			cond_resched();
 
 		if (ctx->sq_data) {
@@ -3066,7 +3072,8 @@ static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
 
 static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 						struct io_uring_task *tctx,
-						bool cancel_all)
+						bool cancel_all,
+						bool force_iopoll)
 {
 	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
 	enum io_wq_cancel cret;
@@ -3096,7 +3103,7 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 
 	/* SQPOLL thread does its own polling */
 	if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
-	    (ctx->sq_data && ctx->sq_data->thread == current)) {
+	    force_iopoll) {
 		while (!wq_list_empty(&ctx->iopoll_list)) {
 			io_iopoll_try_reap_events(ctx);
 			ret = true;
@@ -3169,13 +3176,15 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 					continue;
 				loop |= io_uring_try_cancel_requests(node->ctx,
 							current->io_uring,
-							cancel_all);
+							cancel_all,
+							false);
 			}
 		} else {
 			list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
 				loop |= io_uring_try_cancel_requests(ctx,
 								     current->io_uring,
-								     cancel_all);
+								     cancel_all,
+								     true);
 		}
 
 		if (loop) {
Signature · 1 line
2.43.0
#1 lore raw

Hi,

Quoted 50 lines

-----Original Message-----
From: Bui Quang Minh <[email protected]>
Sent: Sunday, January 12, 2025 10:34 PM
To: [email protected]
Cc: Bui Quang Minh <[email protected]>; Jens Axboe <[email protected]>; Pavel Begunkov <[email protected]>; io- [email protected]; [email protected]; lizetao <[email protected]>
Subject: [PATCH] io_uring: simplify the SQPOLL thread check when cancelling requests

In io_uring_try_cancel_requests, we check whether sq_data->thread == current to determine if the function is called by the SQPOLL thread to do iopoll when IORING_SETUP_SQPOLL is set. This check can race with the SQPOLL thread termination.

io_uring_cancel_generic is used in 2 places: io_uring_cancel_generic and io_ring_exit_work. In io_uring_cancel_generic, we have the information whether the current is SQPOLL thread already. In io_ring_exit_work, in case the SQPOLL thread reaches this path, we don't need to iopoll and leave that for io_uring_cancel_generic to handle.

So to avoid the racy check, this commit adds a boolean flag to io_uring_try_cancel_requests to determine if we need to do iopoll inside the function and only sets this flag in io_uring_cancel_generic when the current is SQPOLL thread.

---
 io_uring/io_uring.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)
io_uring/io_uring.c index renamed from io_uring/io_uring.c +0 -0

ff691f37462c..f28ea1254143 100644

io_uring/io_uring.c +2 -1
@@ -143,7 +143,8 @@ struct io_defer_entry {
 
 static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct io_uring_task *tctx,
-					 bool cancel_all);
+					 bool cancel_all,
+					 bool force_iopoll);
 
 static void io_queue_sqe(struct io_kiocb *req);
 

io_uring/io_uring.c static __cold void io_ring_exit_work(struct · line 2906

Quoted 9 lines
io_uring/io_uring.c +4 -1
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
 		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
 			io_move_task_work_from_local(ctx);
 
-		while (io_uring_try_cancel_requests(ctx, NULL, true))
+		/*
+		 * Even if SQPOLL thread reaches this path, don't force
+		 * iopoll here, let the io_uring_cancel_generic handle
+		 * it.

Just curious, will sq_thread enter this io_ring_exit_work path?

Quoted 48 lines
io_uring/io_uring.c +2 -0
+		 */
+		while (io_uring_try_cancel_requests(ctx, NULL, true, false))
 			cond_resched();
 
io_uring/io_uring.c +2 -1
 		if (ctx->sq_data) {
@@ -3066,7 +3072,8 @@ static __cold bool io_uring_try_cancel_iowq(struct
 io_ring_ctx *ctx)
 
 static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 						struct io_uring_task *tctx,
-						bool cancel_all)
+						bool cancel_all,
+						bool force_iopoll)
 {
 	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
io_uring/io_uring.c +1 -1
 	enum io_wq_cancel cret;
@@ -3096,7 +3103,7 @@ static __cold bool
 io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 
 	/* SQPOLL thread does its own polling */
 	if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
-	    (ctx->sq_data && ctx->sq_data->thread == current)) {
+	    force_iopoll) {
 		while (!wq_list_empty(&ctx->iopoll_list)) {
 			io_iopoll_try_reap_events(ctx);
io_uring/io_uring.c +4 -2
 			ret = true;
@@ -3169,13 +3176,15 @@ __cold void io_uring_cancel_generic(bool
 cancel_all, struct io_sq_data *sqd)
 					continue;
 				loop |= io_uring_try_cancel_requests(node-
 >ctx,
 							current->io_uring,
-							cancel_all);
+							cancel_all,
+							false);
 			}
 		} else {
 			list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
 				loop |= io_uring_try_cancel_requests(ctx,
 								     current-
 >io_uring,
-								     cancel_all);
+								     cancel_all,
+								     true);
io_uring/io_uring.c +0 -1
 		}
 
 		if (loop) {
--
Quoted 2 lines

2.43.0

Maybe you miss something, just like Begunkov mentioned in your last version patch:

  io_uring_cancel_generic
    WARN_ON_ONCE(sqd && sqd->thread != current);

This WARN_ON_ONCE will never be triggered, so you could remove it.

---
Li Zetao

#2 lore raw
Quoted 56 lines · lizetao

Hi,

Quoted 54 lines

-----Original Message-----
From: Bui Quang Minh <[email protected]>
Sent: Sunday, January 12, 2025 10:34 PM
To: [email protected]
Cc: Bui Quang Minh <[email protected]>; Jens Axboe <[email protected]>; Pavel Begunkov <[email protected]>; io- [email protected]; [email protected]; lizetao <[email protected]>
Subject: [PATCH] io_uring: simplify the SQPOLL thread check when cancelling requests

In io_uring_try_cancel_requests, we check whether sq_data->thread == current to determine if the function is called by the SQPOLL thread to do iopoll when IORING_SETUP_SQPOLL is set. This check can race with the SQPOLL thread termination.

io_uring_cancel_generic is used in 2 places: io_uring_cancel_generic and io_ring_exit_work. In io_uring_cancel_generic, we have the information whether the current is SQPOLL thread already. In io_ring_exit_work, in case the SQPOLL thread reaches this path, we don't need to iopoll and leave that for io_uring_cancel_generic to handle.

So to avoid the racy check, this commit adds a boolean flag to io_uring_try_cancel_requests to determine if we need to do iopoll inside the function and only sets this flag in io_uring_cancel_generic when the current is SQPOLL thread.

---
  io_uring/io_uring.c | 21 +++++++++++++++------
  1 file changed, 15 insertions(+), 6 deletions(-)
io_uring/io_uring.c index renamed from io_uring/io_uring.c +0 -0

ff691f37462c..f28ea1254143 100644

io_uring/io_uring.c +2 -1
@@ -143,7 +143,8 @@ struct io_defer_entry {
 
  static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
  					 struct io_uring_task *tctx,
-					 bool cancel_all);
+					 bool cancel_all,
+					 bool force_iopoll);
 
  static void io_queue_sqe(struct io_kiocb *req);
 
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
  		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
  			io_move_task_work_from_local(ctx);
 
Quoted 7 lines

io_uring/io_uring.c static __cold void io_ring_exit_work(struct work_struct *work) · line 2905

Quoted 5 lines
io_uring/io_uring.c +4 -1
-		while (io_uring_try_cancel_requests(ctx, NULL, true))
+		/*
+		 * Even if SQPOLL thread reaches this path, don't force
+		 * iopoll here, let the io_uring_cancel_generic handle
+		 * it.

Just curious, will sq_thread enter this io_ring_exit_work path?

AFAIK, yes. The SQPOLL thread is created with create_io_thread, this function creates a new task with CLONE_FILES. So all the open files is shared. There will be case that the parent closes its io_uring file and SQPOLL thread become the only owner of that file. So it can reach this path when terminating.

Quoted 53 lines
Quoted 46 lines
io_uring/io_uring.c +2 -0
+		 */
+		while (io_uring_try_cancel_requests(ctx, NULL, true, false))
  			cond_resched();
 
  		if (ctx->sq_data) {
io_uring/io_uring.c +2 -1
@@ -3066,7 +3072,8 @@ static __cold bool io_uring_try_cancel_iowq(struct
 io_ring_ctx *ctx)
 
  static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
  						struct io_uring_task *tctx,
-						bool cancel_all)
+						bool cancel_all,
+						bool force_iopoll)
  {
  	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };

enum io_wq_cancel cret;

io_uring/io_uring.c +1 -1
@@ -3096,7 +3103,7 @@ static __cold bool
 io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 
  	/* SQPOLL thread does its own polling */
  	if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
-	    (ctx->sq_data && ctx->sq_data->thread == current)) {
+	    force_iopoll) {
  		while (!wq_list_empty(&ctx->iopoll_list)) {
  			io_iopoll_try_reap_events(ctx);
  			ret = true;
io_uring/io_uring.c +4 -2
@@ -3169,13 +3176,15 @@ __cold void io_uring_cancel_generic(bool
 cancel_all, struct io_sq_data *sqd)
  					continue;
  				loop |= io_uring_try_cancel_requests(node-
 > ctx,
  							current->io_uring,
-							cancel_all);
+							cancel_all,
+							false);
  			}
  		} else {
  			list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
  				loop |= io_uring_try_cancel_requests(ctx,
  								     current-
 > io_uring,
-								     cancel_all);
+								     cancel_all,
+								     true);
  		}
Quoted 4 lines

if (loop) {
--
2.43.0

Maybe you miss something, just like Begunkov mentioned in your last version patch:

Quoted 4 lines
   io_uring_cancel_generic
     WARN_ON_ONCE(sqd && sqd->thread != current);

This WARN_ON_ONCE will never be triggered, so you could remove it.

He meant that we don't need to annotate sqd->thread access in this debug check. The io_uring_cancel_generic function has assumption that the sgd is not NULL only when it's called by a SQPOLL thread. So the check means to ensure this assumption. A data race happens only when this function is called by other tasks than the SQPOLL thread, so it can race with the SQPOLL termination. However, the sgd is not NULL only when this function is called by SQPOLL thread. In normal situation following the io_uring_cancel_generic's assumption, the data race cannot happen. And in case the assumption is broken, the warning almost always is triggered even if data race happens. So we can ignore the race here.

Thanks,
Quang Minh.

#3 lore raw

On 1/12/25 16:14, Bui Quang Minh wrote:
...

Quoted 13 lines
Quoted 5 lines
Quoted 5 lines
io_uring/io_uring.c +0 -0
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
           if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
               io_move_task_work_from_local(ctx);
 
Quoted 7 lines

io_uring/io_uring.c static __cold void io_ring_exit_work(struct work_struct *work) · line 2905

Quoted 5 lines
io_uring/io_uring.c +4 -1
-        while (io_uring_try_cancel_requests(ctx, NULL, true))
+        /*
+         * Even if SQPOLL thread reaches this path, don't force
+         * iopoll here, let the io_uring_cancel_generic handle
+         * it.

Just curious, will sq_thread enter this io_ring_exit_work path?

Quoted 1 line

AFAIK, yes. The SQPOLL thread is created with create_io_thread, this function creates a new task with CLONE_FILES. So all the open files is shared. There will be case that the parent closes its io_uring file and SQPOLL thread become the only owner of that file. So it can reach this path when terminating.

The function is run by a separate kthread, the sqpoll task doesn't call it directly.

[...]

Quoted 11 lines
Quoted 11 lines
Quoted 10 lines
io_uring/io_uring.c +2 -1
 > io_uring,
-                                     cancel_all);
+                                     cancel_all,
+                                     true);

          }

          if (loop) {

Signature · 1 line
2.43.0
Quoted 8 lines
Quoted 6 lines

Maybe you miss something, just like Begunkov mentioned in your last version patch:

   io_uring_cancel_generic
     WARN_ON_ONCE(sqd && sqd->thread != current);

This WARN_ON_ONCE will never be triggered, so you could remove it.

He meant that we don't need to annotate sqd->thread access in this debug check. The io_uring_cancel_generic function has assumption that the sgd is not NULL only when it's called by a SQPOLL thread. So the check means to ensure this assumption. A data race happens only when this function is called by other tasks than the SQPOLL thread, so it can race with the SQPOLL termination. However, the sgd is not NULL only when this function is called by SQPOLL thread. In normal situation following the io_uring_cancel_generic's assumption, the data race cannot happen. And in case the assumption is broken, the warning almost always is triggered even if data race happens. So we can ignore the race here.

Right. And that's the point of warnings, they're supposed to be untriggerable, otherwise there is a problem with the code that needs to be fixed.

Signature · 1 line
Pavel Begunkov
#4 lore raw

Hi,

Quoted 25 lines

-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, January 13, 2025 5:16 AM
To: Bui Quang Minh <[email protected]>; lizetao <[email protected]>; [email protected]
Cc: Jens Axboe <[email protected]>; [email protected]; [email protected]
Subject: Re: [PATCH] io_uring: simplify the SQPOLL thread check when cancelling requests

On 1/12/25 16:14, Bui Quang Minh wrote:
...

Quoted 13 lines
Quoted 12 lines

io_uring/io_uring.c static __cold void io_ring_exit_work(struct · line 2906

Quoted 10 lines
io_uring/io_uring.c +4 -1
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
           if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
               io_move_task_work_from_local(ctx);
 
-        while (io_uring_try_cancel_requests(ctx, NULL, true))
+        /*
+         * Even if SQPOLL thread reaches this path, don't force
+         * iopoll here, let the io_uring_cancel_generic handle
+         * it.

Just curious, will sq_thread enter this io_ring_exit_work path?

Quoted 6 lines
Quoted 1 line

AFAIK, yes. The SQPOLL thread is created with create_io_thread, this function

creates a new task with CLONE_FILES. So all the open files is shared. There will be case that the parent closes its io_uring file and SQPOLL thread become the only owner of that file. So it can reach this path when terminating.

The function is run by a separate kthread, the sqpoll task doesn't call it directly.

I also think so, the sqpoll task may not call io_ring_exit_work().

Quoted 32 lines

[...]

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line

io_uring,

-                                     cancel_all); +                                     cancel_all, +                                     true); }

Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

          if (loop) {
--
2.43.0

Maybe you miss something, just like Begunkov mentioned in your last

version patch:

Quoted 7 lines
Quoted 5 lines

   io_uring_cancel_generic
     WARN_ON_ONCE(sqd && sqd->thread != current);

This WARN_ON_ONCE will never be triggered, so you could remove it.

He meant that we don't need to annotate sqd->thread access in this debug

check. The io_uring_cancel_generic function has assumption that the sgd is not NULL only when it's called by a SQPOLL thread. So the check means to ensure this assumption. A data race happens only when this function is called by other tasks than the SQPOLL thread, so it can race with the SQPOLL termination. However, the sgd is not NULL only when this function is called by SQPOLL thread. In normal situation following the io_uring_cancel_generic's assumption, the data race cannot happen. And in case the assumption is broken, the warning almost always is triggered even if data race happens. So we can ignore the race here.

Quoted 2 lines

Right. And that's the point of warnings, they're supposed to be untriggerable, otherwise there is a problem with the code that needs to be fixed.

Okay, I understand the meaning of this WARN.

Quoted 3 lines

--
Pavel Begunkov

---
Li Zetao

#5 lore raw
In reply to Pavel Begunkov
Quoted 15 lines · Pavel Begunkov

On 1/12/25 16:14, Bui Quang Minh wrote:
...

Quoted 13 lines
Quoted 12 lines

io_uring/io_uring.c static __cold void io_ring_exit_work(struct · line 2906

Quoted 10 lines
io_uring/io_uring.c +4 -1
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
           if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
               io_move_task_work_from_local(ctx);
 
-        while (io_uring_try_cancel_requests(ctx, NULL, true))
+        /*
+         * Even if SQPOLL thread reaches this path, don't force
+         * iopoll here, let the io_uring_cancel_generic handle
+         * it.

Just curious, will sq_thread enter this io_ring_exit_work path?

Quoted 4 lines
Quoted 1 line

AFAIK, yes. The SQPOLL thread is created with create_io_thread, this function creates a new task with CLONE_FILES. So all the open files is shared. There will be case that the parent closes its io_uring file and SQPOLL thread become the only owner of that file. So it can reach this path when terminating.

The function is run by a separate kthread, the sqpoll task doesn't call it directly.

Yeah, the io_uring_release can be called in sqpoll thread but the io_ring_exit_work is queued in the io_uring workqueue so that function is executed in a kthread worker.

I will update the comment and commit message.

Thanks,
Quang Minh.

Reviewed-by on v1 1/1
#6 lore raw
In reply to Bui Quang Minh
Quoted 109 lines

-----Original Message-----
From: Bui Quang Minh <[email protected]>
Sent: Sunday, January 12, 2025 10:34 PM
To: [email protected]
Cc: Bui Quang Minh <[email protected]>; Jens Axboe <[email protected]>; Pavel Begunkov <[email protected]>; io- [email protected]; [email protected]; lizetao <[email protected]>
Subject: [PATCH] io_uring: simplify the SQPOLL thread check when cancelling requests

In io_uring_try_cancel_requests, we check whether sq_data->thread == current to determine if the function is called by the SQPOLL thread to do iopoll when IORING_SETUP_SQPOLL is set. This check can race with the SQPOLL thread termination.

io_uring_cancel_generic is used in 2 places: io_uring_cancel_generic and io_ring_exit_work. In io_uring_cancel_generic, we have the information whether the current is SQPOLL thread already. In io_ring_exit_work, in case the SQPOLL thread reaches this path, we don't need to iopoll and leave that for io_uring_cancel_generic to handle.

So to avoid the racy check, this commit adds a boolean flag to io_uring_try_cancel_requests to determine if we need to do iopoll inside the function and only sets this flag in io_uring_cancel_generic when the current is SQPOLL thread.

---
 io_uring/io_uring.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)
io_uring/io_uring.c index renamed from io_uring/io_uring.c +0 -0

ff691f37462c..f28ea1254143 100644

io_uring/io_uring.c +8 -2
@@ -143,7 +143,8 @@ struct io_defer_entry {
 
 static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 					 struct io_uring_task *tctx,
-					 bool cancel_all);
+					 bool cancel_all,
+					 bool force_iopoll);
 
 static void io_queue_sqe(struct io_kiocb *req);
 
@@ -2898,7 +2899,12 @@ static __cold void io_ring_exit_work(struct
 work_struct *work)
 		if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
 			io_move_task_work_from_local(ctx);
 
-		while (io_uring_try_cancel_requests(ctx, NULL, true))
+		/*
+		 * Even if SQPOLL thread reaches this path, don't force
+		 * iopoll here, let the io_uring_cancel_generic handle
+		 * it.
+		 */
+		while (io_uring_try_cancel_requests(ctx, NULL, true, false))
 			cond_resched();
 
io_uring/io_uring.c +2 -1
 		if (ctx->sq_data) {
@@ -3066,7 +3072,8 @@ static __cold bool io_uring_try_cancel_iowq(struct
 io_ring_ctx *ctx)
 
 static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 						struct io_uring_task *tctx,
-						bool cancel_all)
+						bool cancel_all,
+						bool force_iopoll)
 {
 	struct io_task_cancel cancel = { .tctx = tctx, .all = cancel_all, };
io_uring/io_uring.c +1 -1
 	enum io_wq_cancel cret;
@@ -3096,7 +3103,7 @@ static __cold bool
 io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 
 	/* SQPOLL thread does its own polling */
 	if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
-	    (ctx->sq_data && ctx->sq_data->thread == current)) {
+	    force_iopoll) {
 		while (!wq_list_empty(&ctx->iopoll_list)) {
 			io_iopoll_try_reap_events(ctx);
io_uring/io_uring.c +4 -2
 			ret = true;
@@ -3169,13 +3176,15 @@ __cold void io_uring_cancel_generic(bool
 cancel_all, struct io_sq_data *sqd)
 					continue;
 				loop |= io_uring_try_cancel_requests(node-
 >ctx,
 							current->io_uring,
-							cancel_all);
+							cancel_all,
+							false);
 			}
 		} else {
 			list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
 				loop |= io_uring_try_cancel_requests(ctx,
 								     current-
 >io_uring,
-								     cancel_all);
+								     cancel_all,
+								     true);
io_uring/io_uring.c +0 -1
 		}
 
 		if (loop) {
--
Quoted 2 lines

2.43.0

---
Li Zetao