io_uring: always arm linked timeouts prior to issue
Jens Axboe
maintainerPATCH 1/1 io_uring: always arm linked timeouts prior to issue
There are a few spots where linked timeouts are armed, and not all of them adhere to the pre-arm, attempt issue, post-arm pattern. This can be problematic if the linked request returns that it will trigger a callback later, and does so before the linked timeout is fully armed.
Consolidate all the linked timeout handling into __io_issue_sqe(), rather than have it spread throughout the various issue entry points.
Signed-off-by Jens Axboe <[email protected]>
---
io_uring/io_uring.c +15 -35
@@ -448,24 +448,6 @@ static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)return req->link; } -static inline struct io_kiocb *io_prep_linked_timeout (struct io_kiocb *req) -{ -if (likely (!(req->flags &REQ_F_ARM_LTIMEOUT ))) -return NULL ; -return __io_prep_linked_timeout (req); -} - -static noinline void __io_arm_ltimeout(struct io_kiocb *req) -{ -io_queue_linked_timeout (__io_prep_linked_timeout (req)); -} - -static inline void io_arm_ltimeout (struct io_kiocb *req) -{ -if (unlikely (req->flags &REQ_F_ARM_LTIMEOUT )) -__io_arm_ltimeout (req); -} -static void io_prep_async_work (struct io_kiocb *req) {const struct io_issue_def *def = &io_issue_defs[req->opcode];
@@ -518,7 +500,6 @@ static void io_prep_async_link(struct io_kiocb *req)static void io_queue_iowq (struct io_kiocb *req) { -struct io_kiocb *link =io_prep_linked_timeout (req);struct io_uring_task *tctx = req->tctx;BUG_ON (!tctx);
@@ -543,8 +524,6 @@ static void io_queue_iowq(struct io_kiocb *req)trace_io_uring_queue_async_work (req,io_wq_is_hashed (&req->work));io_wq_enqueue (tctx->io_wq, &req->work); -if (link) -io_queue_linked_timeout (link); }static void io_req_queue_iowq_tw (struct io_kiocb *req,io_tw_token_t tw)
@@ -1724,15 +1703,22 @@ static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,return !!req->file; } +#define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT) +static inline int __io_issue_sqe (struct io_kiocb *req,unsigned int issue_flags,const struct io_issue_def *def) {const struct cred *creds =NULL ; +struct io_kiocb *link =NULL ;int ret; -if (unlikely ((req->flags &REQ_F_CREDS ) && req->creds !=current_cred ())) - creds =override_creds (req->creds); +if (unlikely (req->flags &REQ_ISSUE_SLOW_FLAGS )) { +if ((req->flags &REQ_F_CREDS ) && req->creds !=current_cred ()) + creds =override_creds (req->creds); +if (req->flags &REQ_F_ARM_LTIMEOUT ) + link =__io_prep_linked_timeout (req); + }if (!def->audit_skip)audit_uring_entry (req->opcode);
@@ -1742,8 +1728,12 @@ static inline int __io_issue_sqe(struct io_kiocb *req,if (!def->audit_skip)audit_uring_exit (!ret, ret); -if (creds) -revert_creds (creds); +if (unlikely (creds || link)) { +if (creds) +revert_creds (creds); +if (link) +io_queue_linked_timeout (link); + }return ret; }
@@ -1769,7 +1759,6 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)if (ret ==IOU_ISSUE_SKIP_COMPLETE ) { ret =0 ; -io_arm_ltimeout (req);/* If the op doesn't have a file, we're not polling for it */ if ((req->ctx->flags &IORING_SETUP_IOPOLL ) && def->iopoll_queue)
@@ -1824,8 +1813,6 @@ void io_wq_submit_work(struct io_wq_work *work)else req_ref_get (req); -io_arm_ltimeout (req); -/* either cancelled or io-wq is dying, so don't touch tctx->iowq */ if (atomic_read (&work->flags) &IO_WQ_WORK_CANCEL ) { fail:
@@ -1941,15 +1928,11 @@ struct file *io_file_get_normal(struct io_kiocb *req, int fd)static void io_queue_async (struct io_kiocb *req,int ret) __must_hold(&req->ctx->uring_lock) { -struct io_kiocb *linked_timeout; -if (ret != -EAGAIN || (req->flags &REQ_F_NOWAIT )) {io_req_defer_failed (req, ret);return ; } - linked_timeout =io_prep_linked_timeout (req); -switch (io_arm_poll_handler (req,0 )) {case IO_APOLL_READY :io_kbuf_recycle (req,0 );
@@ -1962,9 +1945,6 @@ static void io_queue_async(struct io_kiocb *req, int ret)case IO_APOLL_OK :break ; } - -if (linked_timeout) -io_queue_linked_timeout (linked_timeout); }static inline void io_queue_sqe (struct io_kiocb *req)
Signature · 1 line
Jens Axboe