io_uring: close lookup gap for dependent work
Jens Axboe
maintainerv2 cover io_uring: close lookup gap for dependent work
(resend as v2, as v1 had some older patches mixed into the series)
As discussed earlier today on this list, there's a gap between finding dependent work and ensuring we can look it up for cancellation purposes. On top of that, we also currently NEVER find dependent work due to how we do lookups of it, so that is fixed in patch 1 while patch 2 implements the fix for the lookup gap.
Patches are against for-5.5/io_uring-post
fs/io-wq.c | 3 +++ fs/io-wq.h | 12 +++++++++++- fs/io_uring.c | 34 +++++++++++++++++++++++++++------- 3 files changed, 41 insertions(+), 8 deletions(-)
--
Jens Axboe
Jens Axboe
maintainerPATCH v2 1/2 io_uring: allow finding next link independent of req reference count
We currently try and start the next link when we put the request, and only if we were going to free it. This means that the optimization to continue executing requests from the same context often fails, as we're not putting the final reference.
Add REQ_F_LINK_NEXT to keep track of this, and allow io_uring to find the next request more efficiently.
--- fs/io_uring.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-)
fs/io_uring.c +15 -5
@@ -340,6 +340,7 @@ struct io_kiocb {#define REQ_F_NOWAIT 1/* must not punt to workers */ #define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_FIXED_FILE 4/* ctx owns file */ +#define REQ_F_LINK_NEXT 8/* already grabbed next link */ #define REQ_F_IO_DRAIN 16/* drain existing IO first */ #define REQ_F_IO_DRAINED 32/* drain done */ #define REQ_F_LINK 64/* linked sqes */
@@ -874,6 +875,10 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)struct io_kiocb *nxt;bool wake_ev = false; +/* Already got next link */ +if (req->flags &REQ_F_LINK_NEXT ) +return ; +/* * The list should never be empty when we are called here. But could * potentially happen if the chain is messed up, check to be on the
@@ -910,6 +915,7 @@ static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)break ; } + req->flags |=REQ_F_LINK_NEXT ;if (wake_ev)io_cqring_ev_posted (ctx); }
@@ -946,12 +952,10 @@ static void io_fail_links(struct io_kiocb *req)io_cqring_ev_posted (ctx); } -static void io_free_req_find_next (struct io_kiocb *req,struct io_kiocb **nxt) +static void io_req_find_next (struct io_kiocb *req,struct io_kiocb **nxt) { -if (likely (!(req->flags &REQ_F_LINK ))) { -__io_free_req (req); +if (likely (!(req->flags &REQ_F_LINK )))return ; - }/* * If LINK is set, we have dependent requests in this chain. If we
@@ -977,7 +981,11 @@ static void io_free_req_find_next(struct io_kiocb *req, struct io_kiocb **nxt) }else {io_req_link_next (req, nxt); } +} +static void io_free_req_find_next (struct io_kiocb *req,struct io_kiocb **nxt) +{ +io_req_find_next (req, nxt);__io_free_req (req); }
@@ -994,8 +1002,10 @@ static void io_put_req_find_next(struct io_kiocb *req, struct io_kiocb **nxtptr) {struct io_kiocb *nxt =NULL ; +io_req_find_next (req, &nxt); +if (refcount_dec_and_test (&req->refs)) -io_free_req_find_next (req, &nxt); +__io_free_req (req);if (nxt) {if (nxtptr)
Signature · 1 line
2.24.0
Jens Axboe
maintainerPATCH v2 2/2 io_uring: close lookup gap for dependent next work
In reply to Jens AxboeWhen we find new work to process within the work handler, we queue the linked timeout before we have issued the new work. This can be problematic for very short timeouts, as we have a window where the new work isn't visible.
Allow the work handler to store a callback function for this in the work item, and flag it with IO_WQ_WORK_CB if the caller has done so. If that is set, then io-wq will call the callback when it has setup the new work item.
--- fs/io-wq.c | 3 +++ fs/io-wq.h | 12 +++++++++++- fs/io_uring.c | 14 ++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-)
fs/io-wq.c +3 -0
@@ -427,6 +427,9 @@ static void io_worker_handle_work(struct io_worker *worker) worker->cur_work = work;spin_unlock_irq (&worker->lock); +if (work->flags &IO_WQ_WORK_CB ) + work->cb.fn (work->cb.data); +if ((work->flags &IO_WQ_WORK_NEEDS_FILES ) && current->files != work->files) {task_lock (current);
fs/io-wq.h +11 -1
@@ -11,6 +11,7 @@ enum {IO_WQ_WORK_NEEDS_FILES =16 ,IO_WQ_WORK_UNBOUND =32 ,IO_WQ_WORK_INTERNAL =64 , +IO_WQ_WORK_CB =128 , IO_WQ_HASH_SHIFT = 24, /* upper 8 bits are used for hash key */ };
@@ -21,8 +22,17 @@ enum io_wq_cancel { IO_WQ_CANCEL_NOTFOUND, /* work not found */ }; +struct io_wq_work ; +struct io_wq_work_cb { +void (*fn)(void *data); +void *data; +}; +struct io_wq_work { -struct list_head list; +union { +struct list_head list; +struct io_wq_work_cb cb; + };void (*func)(struct io_wq_work **);unsigned flags;struct files_struct *files;
fs/io_uring.c +12 -2
@@ -2664,6 +2664,13 @@ static int __io_submit_sqe(struct io_kiocb *req, struct io_kiocb **nxt,return 0 ; } +static void io_link_work_cb (void *data) +{ +struct io_kiocb *link = data; + +io_queue_linked_timeout (link); +} +static void io_wq_submit_work (struct io_wq_work **workptr) {struct io_wq_work *work = *workptr;
@@ -2710,8 +2717,11 @@ static void io_wq_submit_work(struct io_wq_work **workptr)io_prep_async_work (nxt, &link); *workptr = &nxt->work; -if (link) -io_queue_linked_timeout (link); +if (link) { + nxt->work.flags |=IO_WQ_WORK_CB ; + nxt->work.cb.fn = io_link_work_cb; + nxt->work.cb.data = link; + } } }
Signature · 1 line
2.24.0