Add support for IORING_ASYNC_CANCEL_OP
Jens Axboe
maintainercover Add support for IORING_ASYNC_CANCEL_OP
Hi,
We currently support matching on user_data OR file descriptor, in conjunction with the ANY/ALL matches we also have.
This series starts by cleaning up the cancelation support a bit, most notably using a common match handler to avoid duplicating the matching code in a few different spots.
Then it adds IORING_ASYNC_CANCEL_USERDATA, to explicitly ask for matching on user_data. This was the only original way to match, but we since added FD matching. To retain backwards compatability, we will always match on user_data IFF none of the other key matches are set (eg FD and OP).
This now allows matching on any set of criteria that the application wants. It can match on user_data AND fd, for example.
Finally we add support for IORING_ASYNC_CANCEL_OP, which allows applications to match on the original request opcode as well.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/8 io_uring/poll: always set 'ctx' in io_cancel_data
This isn't strictly necessary for this callsite, as it uses it's internal lookup for this cancelation purpose. But let's be consistent with how it's used in general and set ctx as well.
--- io_uring/poll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/poll.c +1 -1
@@ -972,8 +972,8 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)int io_poll_remove (struct io_kiocb *req,unsigned int issue_flags) { struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); -struct io_cancel_data cd = { .data = poll_update->old_user_data, };struct io_ring_ctx *ctx = req->ctx; +struct io_cancel_data cd = { .ctx = ctx, .data = poll_update->old_user_data, };struct io_hash_bucket *bucket;struct io_kiocb *preq;int ret2, ret =0 ;
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 2/8 io_uring/timeout: always set 'ctx' in io_cancel_data
In reply to Jens AxboeIn preparation for using a generic handler to match requests for cancelation purposes, ensure that ctx is set in io_cancel_data. The timeout handlers don't check for this as it'll always match, but we'll need it set going forward.
--- io_uring/timeout.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/timeout.c +2 -2
@@ -409,7 +409,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, struct timespec64 *ts, enum hrtimer_mode mode) __must_hold(&ctx->timeout_lock) { - struct io_cancel_data cd = { .data = user_data, }; + struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };struct io_kiocb *req =io_timeout_extract (ctx, &cd); struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);struct io_timeout_data *data;
@@ -473,7 +473,7 @@ int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)int ret;if (!(tr->flags &IORING_TIMEOUT_UPDATE )) { -struct io_cancel_data cd = { .data = tr->addr, }; +struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, };spin_lock (&ctx->completion_lock); ret =io_timeout_cancel (ctx, &cd);
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 3/8 io_uring/cancel: abstract out request match helper
In reply to Jens AxboeWe have different match code in a variety of spots. Start the cleanup of this by abstracting out a helper that can be used to check if a given request matches the cancelation criteria outlined in io_cancel_data.
--- io_uring/cancel.c | 17 +++++++++++++---- io_uring/cancel.h | 1 + 2 files changed, 14 insertions(+), 4 deletions(-)
io_uring/cancel.c +13 -4
@@ -27,11 +27,11 @@ struct io_cancel {#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \ IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED) -static bool io_cancel_cb (struct io_wq_work *work,void *data) +/* +* Returns true if the request matches the criteria outlined by 'cd'. +*/ +bool io_cancel_req_match (struct io_kiocb *req,struct io_cancel_data *cd) { - struct io_kiocb *req = container_of(work, struct io_kiocb, work); -struct io_cancel_data *cd = data; -if (req->ctx != cd->ctx)return false;if (cd->flags &IORING_ASYNC_CANCEL_ANY ) {
@@ -48,9 +48,18 @@ static bool io_cancel_cb(struct io_wq_work *work, void *data)return false; req->work.cancel_seq = cd->seq; } +return true; } +static bool io_cancel_cb (struct io_wq_work *work,void *data) +{ + struct io_kiocb *req = container_of(work, struct io_kiocb, work); +struct io_cancel_data *cd = data; + +return io_cancel_req_match (req, cd); +} +static int io_async_cancel_one (struct io_uring_task *tctx,struct io_cancel_data *cd) {
io_uring/cancel.h +1 -0
@@ -21,3 +21,4 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,void init_hash_table (struct io_hash_table *table,unsigned size);int io_sync_cancel (struct io_ring_ctx *ctx,void __user *arg); +bool io_cancel_req_match (struct io_kiocb *req,struct io_cancel_data *cd);
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 4/8 io_uring/cancel: fix sequence matching for IORING_ASYNC_CANCEL_ANY
In reply to Jens AxboeWe always need to check/update the cancel sequence if IORING_ASYNC_CANCEL_ALL is set. Also kill the redundant check for IORING_ASYNC_CANCEL_ANY at the end, if we get here we know it's not set as we would've matched it higher up.
--- io_uring/cancel.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
io_uring/cancel.c +3 -2
@@ -35,7 +35,7 @@ bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)if (req->ctx != cd->ctx)return false;if (cd->flags &IORING_ASYNC_CANCEL_ANY ) { - ; + goto check_seq; }else if (cd->flags &IORING_ASYNC_CANCEL_FD ) {if (req->file != cd->file)return false;
@@ -43,7 +43,8 @@ bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)if (req->cqe.user_data != cd->data)return false; } -if (cd->flags & (IORING_ASYNC_CANCEL_ALL |IORING_ASYNC_CANCEL_ANY )) { +if (cd->flags &IORING_ASYNC_CANCEL_ALL ) { +check_seq:if (cd->seq == req->work.cancel_seq)return false; req->work.cancel_seq = cd->seq;
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 5/8 io_uring: use cancelation match helper for poll and timeout requests
In reply to Jens AxboeGet rid of the request vs io_cancel_data checking and just use the exported helper for this.
--- io_uring/poll.c | 12 ++++-------- io_uring/timeout.c | 12 +++--------- 2 files changed, 7 insertions(+), 17 deletions(-)
io_uring/poll.c +4 -8
@@ -824,14 +824,10 @@ static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,spin_lock (&hb->lock);hlist_for_each_entry (req, &hb->list, hash_node) { -if (!(cd->flags &IORING_ASYNC_CANCEL_ANY ) && - req->file != cd->file) -continue ; -if (cd->seq == req->work.cancel_seq) -continue ; - req->work.cancel_seq = cd->seq; - *out_bucket = hb; -return req; +if (io_cancel_req_match (req, cd)) { + *out_bucket = hb; +return req; + } }spin_unlock (&hb->lock); }
io_uring/timeout.c +3 -9
@@ -268,16 +268,10 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,list_for_each_entry (timeout, &ctx->timeout_list, list) {struct io_kiocb *tmp =cmd_to_io_kiocb (timeout); -if (!(cd->flags &IORING_ASYNC_CANCEL_ANY ) && - cd->data != tmp->cqe.user_data) -continue ; -if (cd->flags & (IORING_ASYNC_CANCEL_ALL |IORING_ASYNC_CANCEL_ANY )) { -if (cd->seq == tmp->work.cancel_seq) -continue ; - tmp->work.cancel_seq = cd->seq; +if (io_cancel_req_match (tmp, cd)) { + req = tmp; +break ; } - req = tmp; -break ; }if (!req)return ERR_PTR (-ENOENT );
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 6/8 io_uring/cancel: add IORING_ASYNC_CANCEL_USERDATA
In reply to Jens AxboeAdd a flag to explicitly match on user_data in the request for cancelation purposes. This is the default behavior if none of the other match flags are set, but if we ALSO want to match on user_data, then this flag can be set.
--- include/uapi/linux/io_uring.h | 2 ++ io_uring/cancel.c | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-)
include/uapi/linux/io_uring.h +2 -0
@@ -297,11 +297,13 @@ enum io_uring_op {* request 'user_data' * IORING_ASYNC_CANCEL_ANY Match any request * IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor +* IORING_ASYNC_CANCEL_USERDATA Match on user_data, default for no other key */ #define IORING_ASYNC_CANCEL_ALL (1U << 0)#define IORING_ASYNC_CANCEL_FD (1U << 1)#define IORING_ASYNC_CANCEL_ANY (1U << 2)#define IORING_ASYNC_CANCEL_FD_FIXED (1U << 3) +#define IORING_ASYNC_CANCEL_USERDATA (1U << 4)/* * send/sendmsg and recv/recvmsg flags (sqe->ioprio)
io_uring/cancel.c +12 -6
@@ -25,24 +25,30 @@ struct io_cancel { };#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \ - IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED) + IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \ + IORING_ASYNC_CANCEL_USERDATA)/* * Returns true if the request matches the criteria outlined by 'cd'. */ bool io_cancel_req_match (struct io_kiocb *req,struct io_cancel_data *cd) { +bool match_user_data = cd->flags &IORING_ASYNC_CANCEL_USERDATA ; +if (req->ctx != cd->ctx)return false; -if (cd->flags &IORING_ASYNC_CANCEL_ANY ) { + +if (!(cd->flags & (IORING_ASYNC_CANCEL_FD ))) + match_user_data = true; + +if (cd->flags &IORING_ASYNC_CANCEL_ANY ) goto check_seq; - }else if (cd->flags &IORING_ASYNC_CANCEL_FD ) { +if (cd->flags &IORING_ASYNC_CANCEL_FD ) {if (req->file != cd->file)return false; - }else { -if (req->cqe.user_data != cd->data) -return false; } +if (match_user_data && req->cqe.user_data != cd->data) +return false;if (cd->flags &IORING_ASYNC_CANCEL_ALL ) { check_seq:if (cd->seq == req->work.cancel_seq)
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 7/8 io_uring/cancel: support opcode based lookup and cancelation
In reply to Jens AxboeAdd IORING_ASYNC_CANCEL_OP flag for cancelation, which allows the application to target cancelation based on the opcode of the original request.
--- include/uapi/linux/io_uring.h | 2 ++ io_uring/cancel.c | 17 ++++++++++++++--- io_uring/cancel.h | 2 +- io_uring/poll.c | 3 ++- 4 files changed, 19 insertions(+), 5 deletions(-)
include/uapi/linux/io_uring.h +2 -0
@@ -298,12 +298,14 @@ enum io_uring_op {* IORING_ASYNC_CANCEL_ANY Match any request * IORING_ASYNC_CANCEL_FD_FIXED 'fd' passed in is a fixed descriptor * IORING_ASYNC_CANCEL_USERDATA Match on user_data, default for no other key +* IORING_ASYNC_CANCEL_OP Match request based on opcode */ #define IORING_ASYNC_CANCEL_ALL (1U << 0)#define IORING_ASYNC_CANCEL_FD (1U << 1)#define IORING_ASYNC_CANCEL_ANY (1U << 2)#define IORING_ASYNC_CANCEL_FD_FIXED (1U << 3)#define IORING_ASYNC_CANCEL_USERDATA (1U << 4) +#define IORING_ASYNC_CANCEL_OP (1U << 5)/* * send/sendmsg and recv/recvmsg flags (sqe->ioprio)
io_uring/cancel.c +14 -3
@@ -22,11 +22,12 @@ struct io_cancel {u64 addr;u32 flags;s32 fd; +u8 opcode; };#define CANCEL_FLAGS (IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \ IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \ - IORING_ASYNC_CANCEL_USERDATA) + IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)/* * Returns true if the request matches the criteria outlined by 'cd'.
@@ -38,7 +39,7 @@ bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)if (req->ctx != cd->ctx)return false; -if (!(cd->flags & (IORING_ASYNC_CANCEL_FD ))) +if (!(cd->flags & (IORING_ASYNC_CANCEL_FD |IORING_ASYNC_CANCEL_OP ))) match_user_data = true;if (cd->flags &IORING_ASYNC_CANCEL_ANY )
@@ -47,6 +48,10 @@ bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)if (req->file != cd->file)return false; } +if (cd->flags &IORING_ASYNC_CANCEL_OP ) { +if (req->opcode != cd->opcode) +return false; + }if (match_user_data && req->cqe.user_data != cd->data)return false;if (cd->flags &IORING_ASYNC_CANCEL_ALL ) {
@@ -127,7 +132,7 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)if (unlikely (req->flags &REQ_F_BUFFER_SELECT ))return -EINVAL ; -if (sqe->off || sqe->len || sqe->splice_fd_in) +if (sqe->off || sqe->splice_fd_in)return -EINVAL ; cancel->addr =READ_ONCE (sqe->addr);
@@ -139,6 +144,11 @@ int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)return -EINVAL ; cancel->fd =READ_ONCE (sqe->fd); } +if (cancel->flags &IORING_ASYNC_CANCEL_OP ) { +if (cancel->flags &IORING_ASYNC_CANCEL_ANY ) +return -EINVAL ; + cancel->opcode =READ_ONCE (sqe->len); + }return 0 ; }
@@ -185,6 +195,7 @@ int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) .ctx = req->ctx, .data = cancel->addr, .flags = cancel->flags, + .opcode = cancel->opcode, .seq = atomic_inc_return(&req->ctx->cancel_seq), };struct io_uring_task *tctx = req->task->io_uring;
io_uring/cancel.h +1 -1
@@ -8,11 +8,11 @@ struct io_cancel_data {u64 data;struct file *file; }; +u8 opcode;u32 flags;int seq; }; -int io_async_cancel_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe);int io_async_cancel (struct io_kiocb *req,unsigned int issue_flags);
io_uring/poll.c +2 -1
@@ -851,7 +851,8 @@ static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,struct io_hash_bucket *bucket;struct io_kiocb *req; -if (cd->flags & (IORING_ASYNC_CANCEL_FD |IORING_ASYNC_CANCEL_ANY )) +if (cd->flags & (IORING_ASYNC_CANCEL_FD |IORING_ASYNC_CANCEL_OP | +IORING_ASYNC_CANCEL_ANY )) req =io_poll_file_find (ctx, cd, table, &bucket);else req =io_poll_find (ctx, false, cd, table, &bucket);
Signature · 1 line
2.40.1
Jens Axboe
maintainerPATCH 8/8 io_uring/cancel: wire up IORING_ASYNC_CANCEL_OP for sync cancel
In reply to Jens AxboeAllow usage of IORING_ASYNC_CANCEL_OP through the sync cancelation API as well.
--- include/uapi/linux/io_uring.h | 4 +++- io_uring/cancel.c | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-)
include/uapi/linux/io_uring.h +3 -1
@@ -699,7 +699,9 @@ struct io_uring_sync_cancel_reg {__s32 fd;__u32 flags;struct __kernel_timespec timeout; -__u64 pad[4 ]; +__u8 opcode; +__u8 pad[7 ]; +__u64 pad2[3 ]; };/*
io_uring/cancel.c +8 -3
@@ -265,17 +265,22 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)struct io_uring_sync_cancel_reg sc;struct fd f = { };DEFINE_WAIT (wait); -int ret; +int ret, i;if (copy_from_user (&sc, arg,sizeof (sc)))return -EFAULT ;if (sc.flags & ~CANCEL_FLAGS )return -EINVAL ; -if (sc.pad[0 ] || sc.pad[1 ] || sc.pad[2 ] || sc.pad[3 ]) -return -EINVAL ; +for (i =0 ; i <ARRAY_SIZE (sc.pad); i++) +if (sc.pad[i]) +return -EINVAL ; +for (i =0 ; i <ARRAY_SIZE (sc.pad2); i++) +if (sc.pad2[i]) +return -EINVAL ; cd.data = sc.addr; cd.flags = sc.flags; + cd.opcode = sc.opcode;/* we can grab a normal file descriptor upfront */ if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
Signature · 1 line
2.40.1