Extend cancelation support
Jens Axboe
maintainerv3 cover Extend cancelation support
Hi,
We currently only support looking up and canceling requests based on the user_data of the original request. Sometimes it can be useful to instead key off the fd used in the original request, eg if a socket goes away.
Patch 1 is just a cleanup spotted while doing this, 2 is a prep patch, patch 3 adds support for IORING_ASYNC_CANCEL_ALL, and finally patch 4 adds support for IORING_ASYNC_CANCEL_FD.
If IORING_ASYNC_CANCEL_ALL is set, all requests matching the given criteria are canceled. Return value is number of requests canceled, and 0 if none were found, or any error encountered canceling requests.
If IORING_ASYNC_CANCEL_FD is set, requests matching sqe->fd are canceled rather than matching on sqe->addr for user_data.
v3:
- Fixup issues with CANCEL_ALL
- Combine two prep patches, and extent io_cancel_data propagation further.
- Get rid of buggy task_work running, we don't need it anymore.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 1/4 io_uring: remove dead 'poll_only' argument to io_poll_cancel()
It's only called from one location, and it always passes in 'false'. Kill the argument, and just pass in 'false' to io_poll_find().
--- fs/io_uring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)
fs/io_uring.c +3 -4
@@ -6316,11 +6316,10 @@ static bool io_poll_disarm(struct io_kiocb *req)return true; } -static int io_poll_cancel (struct io_ring_ctx *ctx,__u64 sqe_addr, -bool poll_only) +static int io_poll_cancel (struct io_ring_ctx *ctx,__u64 sqe_addr) __must_hold(&ctx->completion_lock) { -struct io_kiocb *req =io_poll_find (ctx, sqe_addr, poll_only); +struct io_kiocb *req =io_poll_find (ctx, sqe_addr, false);if (!req)return -ENOENT ;
@@ -6808,7 +6807,7 @@ static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)return 0 ;spin_lock (&ctx->completion_lock); - ret =io_poll_cancel (ctx, sqe_addr, false); + ret =io_poll_cancel (ctx, sqe_addr);if (ret != -ENOENT ) goto out;
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH v3 2/4 io_uring: pass in struct io_cancel_data consistently
In reply to Jens AxboeIn preparation for being able to not only key cancel off the user_data, pass in the io_cancel_data struct for the various functions that deal with request cancelation.
--- fs/io_uring.c | 76 +++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 32 deletions(-)
fs/io_uring.c +44 -32
@@ -971,6 +971,11 @@ struct io_defer_entry {u32 seq; }; +struct io_cancel_data { +struct io_ring_ctx *ctx; +u64 data; +}; +struct io_op_def {/* needs req->file assigned */ unsigned needs_file :1 ;
@@ -6288,16 +6293,16 @@ static __cold bool io_poll_remove_all(struct io_ring_ctx *ctx,return found; } -static struct io_kiocb *io_poll_find (struct io_ring_ctx *ctx,__u64 sqe_addr, -bool poll_only) +static struct io_kiocb *io_poll_find (struct io_ring_ctx *ctx,bool poll_only, +struct io_cancel_data *cd) __must_hold(&ctx->completion_lock) {struct hlist_head *list;struct io_kiocb *req; - list = &ctx->cancel_hash[hash_long (sqe_addr, ctx->cancel_hash_bits)]; + list = &ctx->cancel_hash[hash_long (cd->data, ctx->cancel_hash_bits)];hlist_for_each_entry (req, list, hash_node) { -if (sqe_addr != req->cqe.user_data) +if (cd->data != req->cqe.user_data)continue ;if (poll_only && req->opcode !=IORING_OP_POLL_ADD )continue ;
@@ -6316,10 +6321,10 @@ static bool io_poll_disarm(struct io_kiocb *req)return true; } -static int io_poll_cancel (struct io_ring_ctx *ctx,__u64 sqe_addr) +static int io_poll_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd) __must_hold(&ctx->completion_lock) { -struct io_kiocb *req =io_poll_find (ctx, sqe_addr, false); +struct io_kiocb *req =io_poll_find (ctx, false, cd);if (!req)return -ENOENT ;
@@ -6411,13 +6416,14 @@ static int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)static int io_poll_update (struct io_kiocb *req,unsigned int issue_flags) { +struct io_cancel_data cd = { .data = req->poll_update.old_user_data, };struct io_ring_ctx *ctx = req->ctx;struct io_kiocb *preq;int ret2, ret =0 ;bool locked;spin_lock (&ctx->completion_lock); - preq =io_poll_find (ctx, req->poll_update.old_user_data, true); + preq =io_poll_find (ctx, true, &cd);if (!preq || !io_poll_disarm (preq)) {spin_unlock (&ctx->completion_lock); ret = preq ? -EALREADY : -ENOENT ;
@@ -6477,7 +6483,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) }static struct io_kiocb *io_timeout_extract (struct io_ring_ctx *ctx, -__u64 user_data) +struct io_cancel_data *cd) __must_hold(&ctx->timeout_lock) {struct io_timeout_data *io;
@@ -6485,7 +6491,7 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,bool found = false;list_for_each_entry (req, &ctx->timeout_list, timeout.list) { - found = user_data == req->cqe.user_data; + found = cd->data == req->cqe.user_data;if (found)break ; }
@@ -6499,11 +6505,11 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,return req; } -static int io_timeout_cancel (struct io_ring_ctx *ctx,__u64 user_data) +static int io_timeout_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd) __must_hold(&ctx->completion_lock) __must_hold(&ctx->timeout_lock) { -struct io_kiocb *req =io_timeout_extract (ctx, user_data); +struct io_kiocb *req =io_timeout_extract (ctx, cd);if (IS_ERR (req))return PTR_ERR (req);
@@ -6556,7 +6562,8 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, struct timespec64 *ts, enum hrtimer_mode mode) __must_hold(&ctx->timeout_lock) { - struct io_kiocb *req = io_timeout_extract(ctx, user_data); + struct io_cancel_data cd = { .data = user_data, }; +struct io_kiocb *req =io_timeout_extract (ctx, &cd);struct io_timeout_data *data;if (IS_ERR (req))
@@ -6621,9 +6628,11 @@ static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)int ret;if (!(req->timeout_rem.flags &IORING_TIMEOUT_UPDATE )) { +struct io_cancel_data cd = { .data = tr->addr, }; +spin_lock (&ctx->completion_lock);spin_lock_irq (&ctx->timeout_lock); - ret =io_timeout_cancel (ctx, tr->addr); + ret =io_timeout_cancel (ctx, &cd);spin_unlock_irq (&ctx->timeout_lock);spin_unlock (&ctx->completion_lock); }else {
@@ -6752,30 +6761,24 @@ static int io_timeout(struct io_kiocb *req, unsigned int issue_flags)return 0 ; } -struct io_cancel_data { -struct io_ring_ctx *ctx; -u64 user_data; -}; -static bool io_cancel_cb (struct io_wq_work *work,void *data) { struct io_kiocb *req = container_of(work, struct io_kiocb, work);struct io_cancel_data *cd = data; -return req->ctx == cd->ctx && req->cqe.user_data == cd->user_data; +return req->ctx == cd->ctx && req->cqe.user_data == cd->data; } -static int io_async_cancel_one (struct io_uring_task *tctx,u64 user_data, -struct io_ring_ctx *ctx) +static int io_async_cancel_one (struct io_uring_task *tctx, +struct io_cancel_data *cd) { -struct io_cancel_data data = { .ctx = ctx, .user_data = user_data, };enum io_wq_cancel cancel_ret;int ret =0 ;if (!tctx || !tctx->io_wq)return -ENOENT ; - cancel_ret =io_wq_cancel_cb (tctx->io_wq, io_cancel_cb, &data, false); + cancel_ret =io_wq_cancel_cb (tctx->io_wq, io_cancel_cb, cd, false);switch (cancel_ret) {case IO_WQ_CANCEL_OK : ret =0 ;
@@ -6791,14 +6794,14 @@ static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,return ret; } -static int io_try_cancel_userdata (struct io_kiocb *req,u64 sqe_addr) +static int io_try_cancel (struct io_kiocb *req,struct io_cancel_data *cd) {struct io_ring_ctx *ctx = req->ctx;int ret;WARN_ON_ONCE (!io_wq_current_is_worker () && req->task != current); - ret =io_async_cancel_one (req->task->io_uring, sqe_addr, ctx); + ret =io_async_cancel_one (req->task->io_uring, cd);/* * Fall-through even for -EALREADY, as we may have poll armed * that need unarming.
@@ -6807,12 +6810,12 @@ static int io_try_cancel_userdata(struct io_kiocb *req, u64 sqe_addr)return 0 ;spin_lock (&ctx->completion_lock); - ret =io_poll_cancel (ctx, sqe_addr); + ret =io_poll_cancel (ctx, cd);if (ret != -ENOENT ) goto out;spin_lock_irq (&ctx->timeout_lock); - ret =io_timeout_cancel (ctx, sqe_addr); + ret =io_timeout_cancel (ctx, cd);spin_unlock_irq (&ctx->timeout_lock); out:spin_unlock (&ctx->completion_lock);
@@ -6837,11 +6840,14 @@ static int io_async_cancel_prep(struct io_kiocb *req,static int io_async_cancel (struct io_kiocb *req,unsigned int issue_flags) {struct io_ring_ctx *ctx = req->ctx; -u64 sqe_addr = req->cancel.addr; +struct io_cancel_data cd = { + .ctx = ctx, + .data = req->cancel.addr, + };struct io_tctx_node *node;int ret; - ret =io_try_cancel_userdata (req, sqe_addr); + ret =io_try_cancel (req, &cd);if (ret != -ENOENT ) goto done;
@@ -6851,7 +6857,7 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)list_for_each_entry (node, &ctx->tctx_list, ctx_node) {struct io_uring_task *tctx = node->task->io_uring; - ret =io_async_cancel_one (tctx, req->cancel.addr, ctx); + ret =io_async_cancel_one (tctx, &cd);if (ret != -ENOENT )break ; }
@@ -7455,8 +7461,14 @@ static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked)int ret = -ENOENT ;if (prev) { -if (!(req->task->flags &PF_EXITING )) - ret =io_try_cancel_userdata (req, prev->cqe.user_data); +if (!(req->task->flags &PF_EXITING )) { +struct io_cancel_data cd = { + .ctx = req->ctx, + .data = prev->cqe.user_data, + }; + + ret =io_try_cancel (req, &cd); + }io_req_complete_post (req, ret ?: -ETIME ,0 );io_put_req (prev); }else {
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH v3 3/4 io_uring: add support for IORING_ASYNC_CANCEL_ALL
In reply to Jens AxboeThe current cancelation will lookup and cancel the first request it finds based on the key passed in. Add a flag that allows to cancel any request that matches they key. It completes with the number of requests found and canceled, or res < 0 if an error occured.
--- fs/io-wq.h | 1 + fs/io_uring.c | 91 +++++++++++++++++++++++++++-------- include/uapi/linux/io_uring.h | 7 +++ 3 files changed, 78 insertions(+), 21 deletions(-)
fs/io-wq.h +1 -0
@@ -155,6 +155,7 @@ struct io_wq_work_node *wq_stack_extract(struct io_wq_work_node *stack)struct io_wq_work {struct io_wq_work_node list;unsigned flags; +int cancel_seq; };static inline struct io_wq_work *wq_next_work (struct io_wq_work *work)
fs/io_uring.c +70 -21
168 lines not on this page · raw message
include/uapi/linux/io_uring.h +7 -0
@@ -187,6 +187,13 @@ enum {#define IORING_POLL_UPDATE_EVENTS (1U << 1)#define IORING_POLL_UPDATE_USER_DATA (1U << 2) +/* +* ASYNC_CANCEL flags. +* +* IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key +*/ +#define IORING_ASYNC_CANCEL_ALL (1U << 0) +/* * IO completion data structure (Completion Queue Entry) */
Signature · 1 line
2.35.1
Jens Axboe
maintainerPATCH v3 4/4 io_uring: allow IORING_OP_ASYNC_CANCEL with 'fd' key
In reply to Jens AxboeCurrently sqe->addr must contain the user_data of the request being canceled. Introduce the IORING_ASYNC_CANCEL_FD flag, which tells the kernel that we're keying off the file fd instead for cancelation. This allows canceling any request that a) uses a file, and b) was assigned the file based on the value being passed in.
--- fs/io_uring.c | 71 ++++++++++++++++++++++++++++++----- include/uapi/linux/io_uring.h | 3 ++ 2 files changed, 65 insertions(+), 9 deletions(-)
fs/io_uring.c +62 -9
@@ -570,6 +570,7 @@ struct io_cancel {struct file *file;u64 addr;u32 flags; +s32 fd; };struct io_timeout {
@@ -975,7 +976,10 @@ struct io_defer_entry {struct io_cancel_data {struct io_ring_ctx *ctx; -u64 data; +union { +u64 data; +struct file *file; + };u32 flags;int seq; };
@@ -6322,6 +6326,29 @@ static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,return NULL ; } +static struct io_kiocb *io_poll_file_find (struct io_ring_ctx *ctx, +struct io_cancel_data *cd) + __must_hold(&ctx->completion_lock) +{ +struct io_kiocb *req; +int i; + +for (i =0 ; i < (1U << ctx->cancel_hash_bits); i++) { +struct hlist_head *list; + + list = &ctx->cancel_hash[i]; +hlist_for_each_entry (req, list, hash_node) { +if (req->file != cd->file) +continue ; +if (cd->seq == req->work.cancel_seq) +continue ; + req->work.cancel_seq = cd->seq; +return req; + } + } +return NULL ; +} +static bool io_poll_disarm (struct io_kiocb *req) __must_hold(&ctx->completion_lock) {
@@ -6335,8 +6362,12 @@ static bool io_poll_disarm(struct io_kiocb *req)static int io_poll_cancel (struct io_ring_ctx *ctx,struct io_cancel_data *cd) __must_hold(&ctx->completion_lock) { -struct io_kiocb *req =io_poll_find (ctx, false, cd); +struct io_kiocb *req; +if (cd->flags &IORING_ASYNC_CANCEL_FD ) + req =io_poll_file_find (ctx, cd); +else + req =io_poll_find (ctx, false, cd);if (!req)return -ENOENT ;io_poll_cancel_req (req);
@@ -6785,8 +6816,13 @@ static bool io_cancel_cb(struct io_wq_work *work, void *data)if (req->ctx != cd->ctx)return false; -if (req->cqe.user_data != cd->data) -return false; +if (cd->flags &IORING_ASYNC_CANCEL_FD ) { +if (req->file != cd->file) +return false; + }else { +if (req->cqe.user_data != cd->data) +return false; + }if (cd->flags &IORING_ASYNC_CANCEL_ALL ) {if (cd->seq == req->work.cancel_seq)return false;
@@ -6841,9 +6877,11 @@ static int io_try_cancel(struct io_kiocb *req, struct io_cancel_data *cd)if (ret != -ENOENT ) goto out; -spin_lock_irq (&ctx->timeout_lock); - ret =io_timeout_cancel (ctx, cd); -spin_unlock_irq (&ctx->timeout_lock); +if (!(cd->flags &IORING_ASYNC_CANCEL_FD )) { +spin_lock_irq (&ctx->timeout_lock); + ret =io_timeout_cancel (ctx, cd); +spin_unlock_irq (&ctx->timeout_lock); + } out:spin_unlock (&ctx->completion_lock);return ret;
@@ -6854,15 +6892,17 @@ static int io_async_cancel_prep(struct io_kiocb *req, {if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL ))return -EINVAL ; -if (unlikely (req->flags & (REQ_F_FIXED_FILE |REQ_F_BUFFER_SELECT ))) +if (unlikely (req->flags &REQ_F_BUFFER_SELECT ))return -EINVAL ;if (sqe->ioprio || sqe->off || sqe->len || sqe->splice_fd_in)return -EINVAL ; req->cancel.addr =READ_ONCE (sqe->addr); req->cancel.flags =READ_ONCE (sqe->cancel_flags); -if (req->cancel.flags & ~IORING_ASYNC_CANCEL_ALL ) +if (req->cancel.flags & ~(IORING_ASYNC_CANCEL_ALL |IORING_ASYNC_CANCEL_FD ))return -EINVAL ; +if (req->cancel.flags &IORING_ASYNC_CANCEL_FD ) + req->cancel.fd =READ_ONCE (sqe->fd);return 0 ; }
@@ -6911,7 +6951,20 @@ static int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) };int ret; +if (cd.flags &IORING_ASYNC_CANCEL_FD ) { +if (req->flags &REQ_F_FIXED_FILE ) + req->file =io_file_get_fixed (req, req->cancel.fd, issue_flags); +else + req->file =io_file_get_normal (req, req->cancel.fd); +if (!req->file) { + ret = -EBADF ; + goto done; + } + cd.file = req->file; + } + ret =__io_async_cancel (&cd, req, issue_flags); +done:if (ret <0 )req_set_fail (req);io_req_complete_post (req, ret,0 );
include/uapi/linux/io_uring.h +3 -0
@@ -191,8 +191,11 @@ enum {* ASYNC_CANCEL flags. * * IORING_ASYNC_CANCEL_ALL Cancel all requests that match the given key +* IORING_ASYNC_CANCEL_FD Key off 'fd' for cancelation rather than the +* request 'user_data' */ #define IORING_ASYNC_CANCEL_ALL (1U << 0) +#define IORING_ASYNC_CANCEL_FD (1U << 1)/* * IO completion data structure (Completion Queue Entry)
Signature · 1 line
2.35.1