io_uring: defer eventfd signaling when queued from a wakeup handler
Jens Axboe
maintainerPATCH 1/1 io_uring: defer eventfd signaling when queued from a wakeup handler
io_req_local_work_add() signals the CQ ring eventfd inline when it is the one to push the first entry onto ->work_list. For DEFER_TASKRUN rings that add is frequently done from a waitqueue wakeup handler, where an arbitrary waitqueue lock is held.
eventfd_signal_mask() only refuses to recurse when current->in_eventfd is set, but that bit is set by eventfd_signal_mask() itself. If the wake chain starts somewhere else, signal goes out inline and can feed back into epoll.
Add IOU_F_TWQ_IN_WAKE, set it on the task_work add done from the three waitqueue callbacks, and use it to force io_eventfd_signal() down the existing call_rcu_hurry() deferral instead of signaling inline.
---
include/linux/io_uring_types.h +8 -0
@@ -20,6 +20,14 @@ enum {* It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. */ IOU_F_TWQ_LAZY_WAKE =1 , + +/* +* Set when task_work is queued from a waitqueue wakeup handler, where +* an arbitrary provider waitqueue lock is held. Signaling the CQ ring +* eventfd inline from there can recurse back into that lock through +* epoll, so the eventfd signal must be deferred. +*/ + IOU_F_TWQ_IN_WAKE = 2, }; enum io_uring_cmd_flags {
io_uring/eventfd.c +4 -4
@@ -51,9 +51,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu)/* * Returns true if the caller should put the ev_fd reference, false if not. */ -static bool __io_eventfd_signal (struct io_ev_fd *ev_fd) +static bool __io_eventfd_signal (struct io_ev_fd *ev_fd,bool defer) { -if (eventfd_signal_allowed ()) { +if (!defer &&eventfd_signal_allowed ()) {eventfd_signal_mask (ev_fd->cq_ev_fd,EPOLL_URING_WAKE );return true; }
@@ -73,7 +73,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd)return !ev_fd->eventfd_async ||io_wq_current_is_worker (); } -void io_eventfd_signal (struct io_ring_ctx *ctx,bool cqe_event) +void io_eventfd_signal (struct io_ring_ctx *ctx,bool cqe_event,bool defer) {bool skip = false;struct io_ev_fd *ev_fd;
@@ -113,7 +113,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event)spin_unlock (&ctx->completion_lock); } -if (skip ||__io_eventfd_signal (ev_fd)) +if (skip ||__io_eventfd_signal (ev_fd, defer))io_eventfd_put (ev_fd); }
io_uring/eventfd.h +1 -1
@@ -5,4 +5,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int eventfd_async);int io_eventfd_unregister (struct io_ring_ctx *ctx); -void io_eventfd_signal (struct io_ring_ctx *ctx,bool cqe_event); +void io_eventfd_signal (struct io_ring_ctx *ctx,bool cqe_event,bool defer);
io_uring/futex.c +2 -2
@@ -181,7 +181,7 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q)io_req_set_res (req,0 ,0 ); req->io_task_work.func = io_futexv_complete; -io_req_task_work_add (req); +__io_req_task_work_add (req,IOU_F_TWQ_IN_WAKE ); }int io_futexv_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe)
@@ -237,7 +237,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q)io_req_set_res (req,0 ,0 ); req->io_task_work.func = io_futex_complete; -io_req_task_work_add (req); +__io_req_task_work_add (req,IOU_F_TWQ_IN_WAKE ); }int io_futexv_wait (struct io_kiocb *req,unsigned int issue_flags)
io_uring/io_uring.c +1 -1
@@ -475,7 +475,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx)if (ctx->int_flags &IO_RING_F_OFF_TIMEOUT_USED )io_flush_timeouts (ctx);if (ctx->int_flags &IO_RING_F_HAS_EVFD ) -io_eventfd_signal (ctx, true); +io_eventfd_signal (ctx, true, false); }static inline void __io_cq_lock (struct io_ring_ctx *ctx)
io_uring/poll.c +12 -11
@@ -208,9 +208,9 @@ enum { IOU_POLL_REQUEUE = 4, }; -static void __io_poll_execute (struct io_kiocb *req,int mask) +static void __io_poll_execute (struct io_kiocb *req,int mask,unsigned tw_flags) { -unsigned flags =0 ; +unsigned flags = tw_flags;io_req_set_res (req, mask,0 ); req->io_task_work.func = io_poll_task_func;
@@ -218,14 +218,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask)trace_io_uring_task_add (req, mask);if (!(req->flags &REQ_F_POLL_NO_LAZY )) - flags =IOU_F_TWQ_LAZY_WAKE ; + flags |=IOU_F_TWQ_LAZY_WAKE ;__io_req_task_work_add (req, flags); } -static inline void io_poll_execute (struct io_kiocb *req,int res) +static inline void io_poll_execute (struct io_kiocb *req,int res, +unsigned tw_flags) {if (io_poll_get_ownership (req)) -__io_poll_execute (req, res); +__io_poll_execute (req, res, tw_flags); }/*
@@ -344,7 +345,7 @@ void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw)if (ret ==IOU_POLL_NO_ACTION ) {return ; }else if (ret ==IOU_POLL_REQUEUE ) { -__io_poll_execute (req,0 ); +__io_poll_execute (req,0 ,0 );return ; }io_poll_remove_entries (req);
@@ -383,7 +384,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) {io_poll_mark_cancelled (req);/* kick tw, which should complete the request */ -io_poll_execute (req,0 ); +io_poll_execute (req,0 ,0 ); }#define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI)
@@ -392,7 +393,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) {io_poll_mark_cancelled (req);/* we have to kick tw in case it's not already */ -io_poll_execute (req,0 ); +io_poll_execute (req,0 ,IOU_F_TWQ_IN_WAKE );io_poll_remove_waitq (poll);return 1 ; }
@@ -430,7 +431,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,else req->flags &= ~REQ_F_SINGLE_POLL ; } -__io_poll_execute (req, mask); +__io_poll_execute (req, mask,IOU_F_TWQ_IN_WAKE ); }return 1 ; }
@@ -618,7 +619,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req,if (mask && (poll->events &EPOLLET ) &&io_poll_can_finish_inline (req, ipt)) { -__io_poll_execute (req, mask); +__io_poll_execute (req, mask,0 );return 0 ; }io_napi_add (req);
@@ -629,7 +630,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req,* poll was waken up, queue up a tw, it'll deal with it. */ if (atomic_cmpxchg (&req->poll_refs,1 ,0 ) !=1 ) -__io_poll_execute (req,0 ); +__io_poll_execute (req,0 ,0 ); }return 0 ; }
io_uring/tw.c +1 -1
@@ -170,7 +170,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags)if (mpscq_push (&ctx->work_list, &req->io_task_work.node)) {io_ctx_mark_taskrun (ctx);if (data_race (ctx->int_flags) &IO_RING_F_HAS_EVFD ) -io_eventfd_signal (ctx, false); +io_eventfd_signal (ctx, false, flags &IOU_F_TWQ_IN_WAKE ); }/*
io_uring/waitid.c +1 -1
@@ -253,7 +253,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode,return 1 ; req->io_task_work.func = io_waitid_cb; -io_req_task_work_add (req); +__io_req_task_work_add (req,IOU_F_TWQ_IN_WAKE );return 1 ; }
Signature · 1 line
Jens Axboe