io_uring fixes for 5.7
Jens Axboe
maintainercover io_uring fixes for 5.7
A few fixes for the current branch, that came out of memcached testing on roughly 320K sockets with the revised poll scheme.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/3 io_uring: retry poll if we got woken with non-matching mask
If we get woken and the poll doesn't match our mask, re-add the task to the poll waitqueue and try again instead of completing the request with a mask of 0.
--- fs/io_uring.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
fs/io_uring.c +12 -0
@@ -4412,8 +4412,20 @@ static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)static void io_poll_task_handler (struct io_kiocb *req,struct io_kiocb **nxt) {struct io_ring_ctx *ctx = req->ctx; +struct io_poll_iocb *poll = &req->poll; + +if (!req->result && !READ_ONCE (poll->canceled)) { +struct poll_table_struct pt = { ._key = poll->events }; + + req->result =vfs_poll (req->file, &pt) & poll->events; + }spin_lock_irq (&ctx->completion_lock); +if (!req->result && !READ_ONCE (poll->canceled)) { +add_wait_queue (poll->head, &poll->wait); +spin_unlock_irq (&ctx->completion_lock); +return ; + }hash_del (&req->hash_node);io_poll_complete (req, req->result,0 ); req->flags |=REQ_F_COMP_LOCKED ;
Signature · 1 line
2.26.0
Jens Axboe
maintainerPATCH 2/3 io_uring: grab task reference for poll requests
In reply to Jens AxboeWe can have a task exit if it's not the owner of the ring. Be safe and grab an actual reference to it, to avoid a potential use-after-free.
--- fs/io_uring.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-)
fs/io_uring.c +7 -12
@@ -615,10 +615,8 @@ struct io_kiocb {struct list_head list;unsigned int flags;refcount_t refs; -union { -struct task_struct *task; -unsigned long fsize; - }; +struct task_struct *task; +unsigned long fsize;u64 user_data;u32 result;u32 sequence;
@@ -1336,6 +1334,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, req->flags =0 ;/* one is dropped after submission, the other at completion */ refcount_set (&req->refs,2 ); + req->task =NULL ; req->result =0 ;INIT_IO_WORK (&req->work, io_wq_submit_work);return req;
@@ -1372,6 +1371,8 @@ static void __io_req_aux_free(struct io_kiocb *req)kfree (req->io);if (req->file)io_put_file (req, req->file, (req->flags &REQ_F_FIXED_FILE )); +if (req->task) +put_task_struct (req->task);io_req_work_drop_env (req); }
@@ -4256,10 +4257,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) req->flags |=REQ_F_POLLED ;memcpy (&apoll->work, &req->work,sizeof (req->work)); -/* -* Don't need a reference here, as we're adding it to the task -* task_works list. If the task exits, the list is pruned. -*/ +get_task_struct (current); req->task = current; req->apoll = apoll;INIT_HLIST_NODE (&req->hash_node);
@@ -4482,10 +4480,7 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe events =READ_ONCE (sqe->poll_events); poll->events =demangle_poll (events) |EPOLLERR |EPOLLHUP ; -/* -* Don't need a reference here, as we're adding it to the task -* task_works list. If the task exits, the list is pruned. -*/ +get_task_struct (current); req->task = current;return 0 ; }
Signature · 1 line
2.26.0
Jens Axboe
maintainerPATCH 3/3 io_uring: use io-wq manager as backup task if task is exiting
In reply to Jens AxboeIf the original task is (or has) exited, then the task work will not get queued properly. Allow for using the io-wq manager task to queue this work for execution, and ensure that the io-wq manager notices and runs this work if woken up (or exiting).
--- fs/io-wq.c | 12 ++++++++++++ fs/io-wq.h | 2 ++ fs/io_uring.c | 13 +++++++++---- 3 files changed, 23 insertions(+), 4 deletions(-)
fs/io-wq.c +12 -0
@@ -17,6 +17,7 @@#include <linux/kthread.h> #include <linux/rculist_nulls.h> #include <linux/fs_struct.h> +#include <linux/task_work.h> #include "io-wq.h"
@@ -716,6 +717,9 @@ static int io_wq_manager(void *data)complete (&wq->done);while (!kthread_should_stop ()) { +if (current->task_works) +task_work_run (); +for_each_node (node) {struct io_wqe *wqe = wq->wqes[node];bool fork_worker[2 ] = { false, false };
@@ -738,6 +742,9 @@ static int io_wq_manager(void *data)schedule_timeout (HZ ); } +if (current->task_works) +task_work_run (); +return 0 ; err:set_bit (IO_WQ_BIT_ERROR , &wq->state);
@@ -1124,3 +1131,8 @@ void io_wq_destroy(struct io_wq *wq)if (refcount_dec_and_test (&wq->use_refs))__io_wq_destroy (wq); } + +struct task_struct *io_wq_get_task (struct io_wq *wq) +{ +return wq->manager; +}
fs/io-wq.h +2 -0
@@ -136,6 +136,8 @@ typedef bool (work_cancel_fn)(struct io_wq_work *, void *);enum io_wq_cancel io_wq_cancel_cb (struct io_wq *wq,work_cancel_fn *cancel,void *data); +struct task_struct *io_wq_get_task (struct io_wq *wq); +#if defined(CONFIG_IO_WQ )extern void io_wq_worker_sleeping (struct task_struct *);extern void io_wq_worker_running (struct task_struct *);
fs/io_uring.c +9 -4
@@ -4120,6 +4120,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, __poll_t mask, task_work_func_t func) {struct task_struct *tsk; +int ret;/* for instances that support it check for an event match first: */ if (mask && !(mask & poll->events))
@@ -4133,11 +4134,15 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, req->result = mask;init_task_work (&req->task_work, func);/* -* If this fails, then the task is exiting. If that is the case, then -* the exit check will ultimately cancel these work items. Hence we -* don't need to check here and handle it specifically. +* If this fails, then the task is exiting. Punt to one of the io-wq +* threads to ensure the work gets run, we can't always rely on exit +* cancelation taking care of this. */ -task_work_add (tsk, &req->task_work, true); + ret =task_work_add (tsk, &req->task_work, true); +if (unlikely (ret)) { + tsk =io_wq_get_task (req->ctx->io_wq); +task_work_add (tsk, &req->task_work, true); + }wake_up_process (tsk);return 1 ; }
Signature · 1 line
2.26.0