Get rid of io_kiocb->imu
Jens Axboe
maintainerv3 cover Get rid of io_kiocb->imu
Hi,
There's really no need to keep this one in the io_kiocb, so get rid of it.
Changes since v2:
- Don't grab uring_lock in io_uring_cmd_import_fixed(), but check if req->rsrc_node has been set.
- Verify fixed buffers work, the liburing test case was broken.
include/linux/io_uring_types.h | 3 --- io_uring/net.c | 29 ++++++++++++++++------------- io_uring/rw.c | 5 +++-- io_uring/uring_cmd.c | 23 ++++++++++++++++++----- 4 files changed, 37 insertions(+), 23 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 1/4 io_uring/uring_cmd: get rid of using req->imu
It's pretty pointless to use io_kiocb as intermediate storage for this, so split the validity check and the actual usage. The resource node is assigned upfront at prep time, to prevent it from going away. The actual import is never called with the ctx->uring_lock held, so grab it for the import.
--- io_uring/uring_cmd.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-)
io_uring/uring_cmd.c +18 -5
@@ -211,11 +211,15 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)struct io_ring_ctx *ctx = req->ctx;u16 index; - req->buf_index =READ_ONCE (sqe->buf_index); -if (unlikely (req->buf_index >= ctx->nr_user_bufs)) + index =READ_ONCE (sqe->buf_index); +if (unlikely (index >= ctx->nr_user_bufs))return -EFAULT ; - index =array_index_nospec (req->buf_index, ctx->nr_user_bufs); - req->imu = ctx->user_bufs[index]; + req->buf_index =array_index_nospec (index, ctx->nr_user_bufs); +/* +* Pi node upfront, prior to io_uring_cmd_import_fixed() +* being called. This prevents destruction of the mapped buffer +* we'll need at actual import time. +*/ io_req_set_rsrc_node (req, ctx,0 ); } ioucmd->cmd_op =READ_ONCE (sqe->cmd_op);
@@ -272,8 +276,17 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, void *ioucmd) { struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); +struct io_ring_ctx *ctx = req->ctx; + +/* Must have had rsrc_node assigned at prep time */ +if (req->rsrc_node) { +struct io_mapped_ubuf *imu; + + imu =READ_ONCE (ctx->user_bufs[req->buf_index]); +return io_import_fixed (rw, iter, imu, ubuf, len); + } -return io_import_fixed (rw, iter, req->imu, ubuf, len); +return -EFAULT ; }EXPORT_SYMBOL_GPL (io_uring_cmd_import_fixed);
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 2/4 io_uring/rw: get rid of using req->imu
In reply to Jens AxboeIt's assigned in the same function that it's being used, get rid of it. A local variable will do just fine.
--- io_uring/rw.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
io_uring/rw.c +3 -2
@@ -330,6 +330,7 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqe { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);struct io_ring_ctx *ctx = req->ctx; +struct io_mapped_ubuf *imu;struct io_async_rw *io;u16 index;int ret;
@@ -341,11 +342,11 @@ static int io_prep_rw_fixed(struct io_kiocb *req, const struct io_uring_sqe *sqeif (unlikely (req->buf_index >= ctx->nr_user_bufs))return -EFAULT ; index =array_index_nospec (req->buf_index, ctx->nr_user_bufs); - req->imu = ctx->user_bufs[index]; + imu = ctx->user_bufs[index];io_req_set_rsrc_node (req, ctx,0 ); io = req->async_data; - ret =io_import_fixed (ddir, &io->iter, req->imu, rw->addr, rw->len); + ret =io_import_fixed (ddir, &io->iter, imu, rw->addr, rw->len);iov_iter_save_state (&io->iter, &io->iter_state);return ret; }
Signature · 1 line
2.45.2
Jens Axboe
maintainerPATCH v3 3/4 io_uring/net: move send zc fixed buffer import to issue path
In reply to Jens AxboeLet's keep it close with the actual import, there's no reason to do this on the prep side. With that, we can drop one of the branches checking for whether or not IORING_RECVSEND_FIXED_BUF is set.
As a side-effect, get rid of req->imu usage.
--- io_uring/net.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)
io_uring/net.c +16 -13
@@ -76,6 +76,7 @@ struct io_sr_msg {/* initialised and used only by !msg send variants */ u16 addr_len;u16 buf_group; +u16 buf_index;void __user *addr;void __user *msg_control;/* used only for send zerocopy */
@@ -1254,16 +1255,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } -if (zc->flags &IORING_RECVSEND_FIXED_BUF ) { -unsigned idx =READ_ONCE (sqe->buf_index); - -if (unlikely (idx >= ctx->nr_user_bufs)) -return -EFAULT ; - idx =array_index_nospec (idx, ctx->nr_user_bufs); - req->imu =READ_ONCE (ctx->user_bufs[idx]); -io_req_set_rsrc_node (notif, ctx,0 ); - } -if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))return -EINVAL ;
@@ -1279,6 +1270,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); zc->len =READ_ONCE (sqe->len); zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ; + zc->buf_index =READ_ONCE (sqe->buf_index);if (zc->msg_flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;
@@ -1339,13 +1331,24 @@ static int io_sg_from_iter(struct sk_buff *skb,return ret; } -static int io_send_zc_import (struct io_kiocb *req,struct io_async_msghdr *kmsg) +static int io_send_zc_import (struct io_kiocb *req,unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *kmsg = req->async_data;int ret;if (sr->flags &IORING_RECVSEND_FIXED_BUF ) { - ret =io_import_fixed (ITER_SOURCE , &kmsg->msg.msg_iter, req->imu, +struct io_ring_ctx *ctx = req->ctx; +struct io_mapped_ubuf *imu; +int idx; + +if (unlikely (sr->buf_index >= ctx->nr_user_bufs)) +return -EFAULT ; + idx =array_index_nospec (sr->buf_index, ctx->nr_user_bufs); + imu =READ_ONCE (ctx->user_bufs[idx]); +io_req_set_rsrc_node (sr->notif, ctx, issue_flags); + + ret =io_import_fixed (ITER_SOURCE , &kmsg->msg.msg_iter, imu, (u64 )(uintptr_t )sr->buf, sr->len);if (unlikely (ret))return ret;
@@ -1382,7 +1385,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)return -EAGAIN ;if (!zc->done_io) { - ret =io_send_zc_import (req, kmsg); + ret =io_send_zc_import (req, issue_flags);if (unlikely (ret))return ret; }
Signature · 1 line
2.45.2
Re: [PATCH 3/4] io_uring/net: move send zc fixed buffer import to issue path
Quoted 16 lines · Jens Axboe
Let's keep it close with the actual import, there's no reason to do this on the prep side. With that, we can drop one of the branches checking for whether or not
IORING_RECVSEND_FIXED_BUFis set.As a side-effect, get rid of
req->imuusage.Signed-off-by Jens Axboe <[email protected]>--- io_uring/net.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)io_uring/net.c +0
-0
io_uring/net.c struct io_sr_msg { · line 79
Quoted 4 lines
io_uring/net.c +1
-0@@ -76,6 +76,7 @@ struct io_sr_msg {/* initialised and used only by !msg send variants */ u16 addr_len;u16 buf_group; +u16 buf_index;
There is req->buf_index we can use
Quoted 40 lines
io_uring/net.c +3
-11void __user *addr;void __user *msg_control;/* used only for send zerocopy */ @@ -1254,16 +1255,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } -if (zc->flags &IORING_RECVSEND_FIXED_BUF ) { -unsigned idx =READ_ONCE (sqe->buf_index); - -if (unlikely (idx >= ctx->nr_user_bufs)) -return -EFAULT ; - idx =array_index_nospec (idx, ctx->nr_user_bufs); - req->imu =READ_ONCE (ctx->user_bufs[idx]); -io_req_set_rsrc_node (notif, ctx,0 ); - } -if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))return -EINVAL ;@@ -1279,6 +1270,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); zc->len =READ_ONCE (sqe->len); zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ; + zc->buf_index =READ_ONCE (sqe->buf_index);if (zc->msg_flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ;@@ -1339,13 +1331,24 @@ static int io_sg_from_iter(struct sk_buff *skb,return ret; } -static int io_send_zc_import (struct io_kiocb *req,struct io_async_msghdr *kmsg) +static int io_send_zc_import (struct io_kiocb *req,unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *kmsg = req->async_data;int ret;if (sr->flags &IORING_RECVSEND_FIXED_BUF ) {
io_uring/net.c static int io_sg_from_iter(struct sk_buff *skb, · line 1349
Quoted 10 lines
io_uring/net.c +9
-1- ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, req->imu, +struct io_ring_ctx *ctx = req->ctx; +struct io_mapped_ubuf *imu; +int idx; + +if (unlikely (sr->buf_index >= ctx->nr_user_bufs)) +return -EFAULT ; + idx =array_index_nospec (sr->buf_index, ctx->nr_user_bufs); + imu =READ_ONCE (ctx->user_bufs[idx]); +io_req_set_rsrc_node (sr->notif, ctx, issue_flags);
This entire section should be under uring_lock. First, we're looking at a imu that can be freed at any moment because io_req_set_rsrc_node() is done after. And even if change the order nothing guarantees that the CPU sees the imu content right.
FWIW, seems nobody was passing non-zero flags to io_req_set_rsrc_node() before this series, we should just kill the parameter.
Quoted 14 lines
io_uring/net.c +3
-1+ + ret =io_import_fixed (ITER_SOURCE , &kmsg->msg.msg_iter, imu, (u64 )(uintptr_t )sr->buf, sr->len);if (unlikely (ret))return ret;@@ -1382,7 +1385,7 @@ int io_send_zc(struct io_kiocb *req, unsigned int issue_flags)return -EAGAIN ;if (!zc->done_io) { - ret =io_send_zc_import (req, kmsg); + ret =io_send_zc_import (req, issue_flags);if (unlikely (ret))return ret; }
Signature · 1 line
Pavel Begunkov
Jens Axboe
maintainerRe: [PATCH 3/4] io_uring/net: move send zc fixed buffer import to issue path
Quoted 12 lines · Pavel Begunkov
Quoted 11 lines · Jens Axboe
Let's keep it close with the actual import, there's no reason to do this on the prep side. With that, we can drop one of the branches checking for whether or not
IORING_RECVSEND_FIXED_BUFis set.As a side-effect, get rid of
req->imuusage.Signed-off-by Jens Axboe <[email protected]>--- io_uring/net.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)
Quoted 11 lines
io_uring/net.c struct io_sr_msg { · line 79
Quoted 9 lines
io_uring/net.c +1
-0@@ -76,6 +76,7 @@ struct io_sr_msg {/* initialised and used only by !msg send variants */ u16 addr_len;u16 buf_group; +u16 buf_index;There is
req->buf_indexwe can use
But that gets repurposed as the group ID for provided buffers, which is why I wanted to add a separate field for that.
Quoted 43 lines
Quoted 43 lines
io_uring/net.c +1
-9void __user *addr;void __user *msg_control;/* used only for send zerocopy */ @@ -1254,16 +1255,6 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } } - if (zc->flags & IORING_RECVSEND_FIXED_BUF) { -unsigned idx =READ_ONCE (sqe->buf_index); - -if (unlikely (idx >= ctx->nr_user_bufs)) -return -EFAULT ; - idx =array_index_nospec (idx, ctx->nr_user_bufs); - req->imu =READ_ONCE (ctx->user_bufs[idx]); -io_req_set_rsrc_node (notif, ctx,0 ); - } -if (req->opcode ==IORING_OP_SEND_ZC ) {if (READ_ONCE (sqe->__pad3[0 ]))return -EINVAL ;@@ -1279,6 +1270,7 @@ int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->buf =u64_to_user_ptr (READ_ONCE (sqe->addr)); zc->len =READ_ONCE (sqe->len); zc->msg_flags =READ_ONCE (sqe->msg_flags) |MSG_NOSIGNAL |MSG_ZEROCOPY ; + zc->buf_index =READ_ONCE (sqe->buf_index);if (zc->msg_flags &MSG_DONTWAIT ) req->flags |=REQ_F_NOWAIT ; @@ -1339,13 +1331,24 @@ static int io_sg_from_iter(struct sk_buff *skb,return ret;}
-static intio_send_zc_import(struct io_kiocb *req, struct io_async_msghdr *kmsg)io_uring/net.c +8
-1+static int io_send_zc_import (struct io_kiocb *req,unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); +struct io_async_msghdr *kmsg = req->async_data;int ret;if (sr->flags &IORING_RECVSEND_FIXED_BUF ) { - ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter, req->imu, +struct io_ring_ctx *ctx = req->ctx; +struct io_mapped_ubuf *imu; +int idx; + +if (unlikely (sr->buf_index >= ctx->nr_user_bufs)) +return -EFAULT ;
Quoted 8 lines
io_uring/net.c static int io_sg_from_iter(struct sk_buff *skb, · line 1349
Quoted 3 lines
io_uring/net.c +3
-0+ idx =array_index_nospec (sr->buf_index, ctx->nr_user_bufs); + imu =READ_ONCE (ctx->user_bufs[idx]); +io_req_set_rsrc_node (sr->notif, ctx, issue_flags);This entire section should be under
uring_lock. First, we're looking at a imu that can be freed at any moment becauseio_req_set_rsrc_node()is done after. And even if change the order nothing guarantees that the CPU sees the imu content right.
True, I'll lock around it instead.
Quoted 2 lines
FWIW, seems nobody was passing non-zero flags to
io_req_set_rsrc_node()before this series, we should just kill the parameter.
I did briefly look at that last week, but this got in the way. I'll take another look.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 4/4 io_uring: kill 'imu' from struct io_kiocb
In reply to Jens AxboeIt's no longer being used, remove it.
--- include/linux/io_uring_types.h | 3 --- 1 file changed, 3 deletions(-)
include/linux/io_uring_types.h +0 -3
@@ -613,9 +613,6 @@ struct io_kiocb {struct task_struct *task;union { -/* store used ubuf, so we can prevent reloading */ -struct io_mapped_ubuf *imu; -/* stores selected buf, valid IFF REQ_F_BUFFER_SELECTED is set */ struct io_buffer *kbuf;
Signature · 1 line
2.45.2