io_uring/napi: ensure napi polling is aborted when work is available
Jens Axboe
maintainerPATCH 1/1 io_uring/napi: ensure napi polling is aborted when work is available
While testing io_uring NAPI with DEFER_TASKRUN, I ran into slowdowns and stalls in packet delivery. Turns out that while io_napi_busy_loop_should_end() aborts appropriately on regular task_work, it does not abort if we have local task_work pending.
Move io_has_work() into the private io_uring.h header, and gate whether we should continue polling on that as well. This makes NAPI polling on send/receive work as designed with IORING_SETUP_DEFER_TASKRUN as well.
Fixes 8d0c12a80cde ("io-uring: add napi busy poll support")
Signed-off-by Jens Axboe <[email protected]>
---
io_uring/io_uring.c +0 -11
@@ -122,11 +122,6 @@#define IO_COMPL_BATCH 32#define IO_REQ_ALLOC_BATCH 8 -enum { -IO_CHECK_CQ_OVERFLOW_BIT , -IO_CHECK_CQ_DROPPED_BIT , -}; -struct io_defer_entry {struct list_head list;struct io_kiocb *req;
@@ -2479,12 +2474,6 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)return ret; } -static inline bool io_has_work (struct io_ring_ctx *ctx) -{ -return test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq) || - !llist_empty (&ctx->work_llist); -} -static int io_wake_function (struct wait_queue_entry *curr,unsigned int mode,int wake_flags,void *key) {
io_uring/io_uring.h +11 -0
@@ -448,4 +448,15 @@ static inline bool io_file_can_poll(struct io_kiocb *req) }return false; } + +enum { +IO_CHECK_CQ_OVERFLOW_BIT , +IO_CHECK_CQ_DROPPED_BIT , +}; + +static inline bool io_has_work (struct io_ring_ctx *ctx) +{ +return test_bit (IO_CHECK_CQ_OVERFLOW_BIT , &ctx->check_cq) || + !llist_empty (&ctx->work_llist); +}#endif
io_uring/napi.c +1 -1
@@ -122,7 +122,7 @@ static bool io_napi_busy_loop_should_end(void *data,if (signal_pending (current))return true; -if (io_should_wake (iowq)) +if (io_should_wake (iowq) ||io_has_work (iowq->ctx))return true;if (io_napi_busy_loop_timeout (start_time, iowq->napi_busy_poll_to))return true;
Signature · 1 line
Jens Axboe