Skip to contentSkip to conversation

pre-mapped rw attributes

RFC In review Pavel Begunkov 2 participants 9 messages Dec 30, 2024 – Jan 6, 2025 io_uring

lore
Share & tools
lore
#0 lore raw

cover pre-mapped rw attributes

warning: not properly tested

Follow up on the discussion about optimising copy_from_user() for read/write attributes. The idea here is to use registered regions (see IORING_REGISTER_MEM_REGION) for that purpose pretty much in the same way registered wait arguments work.

Putting it simply, a region is a user provided chunk of memory that has been registered and pre-mapped into io_uring / kernel, but it has more modes like mmap'ing kernel memory. For attributes the user passes an offset into a region, and the kernel can read from it directly without copy_from_user().

The other alternative is to store attributes into the upper half of SQE128, but then we might run out of space in SQE for larger and/or compound attributes. It'd also require SQE128, which has a (perhaps minor) downside when other types of requests don't need it.

Pavel Begunkov (4):
io_uring: add structure for registered arguments
io_uring: add registered request arguments io_uring/rw: use READ_ONCE with rw attributes io_uring/rw: pre-mapped rw attributes

 include/linux/io_uring_types.h | 11 ++++++++--
 include/uapi/linux/io_uring.h  |  4 +++-
 io_uring/io_uring.c            | 23 +++------------------
 io_uring/io_uring.h            | 16 +++++++++++++++
 io_uring/register.c            |  7 +++++--
 io_uring/rw.c                  | 37 ++++++++++++++++++++++++----------
 6 files changed, 62 insertions(+), 36 deletions(-)
Signature · 1 line
2.47.1
#1 lore raw

PATCH 1/4 io_uring: add structure for registered arguments

A preparation patch making infra for wait arguments a bit more general to use in in following patches.

---
 include/linux/io_uring_types.h |  9 +++++++--
 io_uring/io_uring.c            | 23 +++--------------------
 io_uring/io_uring.h            | 16 ++++++++++++++++
 io_uring/register.c            |  4 ++--
 4 files changed, 28 insertions(+), 24 deletions(-)
include/linux/io_uring_types.h +7 -2
index 493a8f7fa8e4..49008f00d064 100644
@@ -83,6 +83,11 @@ struct io_mapped_region {
 	unsigned		flags;
 };
 
+struct io_reg_args {
+	void			*ptr;
+	size_t			size;
+};
+
 /*
  * Arbitrary limit, can be raised if need be
  */
@@ -332,8 +337,8 @@ struct io_ring_ctx {
 		struct io_ev_fd	__rcu	*io_ev_fd;
 		unsigned		cq_extra;
 
-		void			*cq_wait_arg;
-		size_t			cq_wait_size;
+		struct io_reg_args	wait_args;
+
 	} ____cacheline_aligned_in_smp;
 
 	/*
io_uring/io_uring.c +3 -20
index 5535a72b0ce1..e2b6b256fc9a 100644
@@ -3178,25 +3178,6 @@ void __io_uring_cancel(bool cancel_all)
 	io_uring_cancel_generic(cancel_all, NULL);
 }
 
-static struct io_uring_reg_wait *io_get_ext_arg_reg(struct io_ring_ctx *ctx,
-			const struct io_uring_getevents_arg __user *uarg)
-{
-	unsigned long size = sizeof(struct io_uring_reg_wait);
-	unsigned long offset = (uintptr_t)uarg;
-	unsigned long end;
-
-	if (unlikely(offset % sizeof(long)))
-		return ERR_PTR(-EFAULT);
-
-	/* also protects from NULL ->cq_wait_arg as the size would be 0 */
-	if (unlikely(check_add_overflow(offset, size, &end) ||
-		     end > ctx->cq_wait_size))
-		return ERR_PTR(-EFAULT);
-
-	offset = array_index_nospec(offset, ctx->cq_wait_size - size);
-	return ctx->cq_wait_arg + offset;
-}
-
 static int io_validate_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
 			       const void __user *argp, size_t argsz)
 {
@@ -3233,7 +3214,9 @@ static int io_get_ext_arg(struct io_ring_ctx *ctx, unsigned flags,
 
 		if (ext_arg->argsz != sizeof(struct io_uring_reg_wait))
 			return -EINVAL;
-		w = io_get_ext_arg_reg(ctx, argp);
+
+		w = io_args_get_ptr(&ctx->wait_args, (uintptr_t)argp,
+				    sizeof(struct io_uring_reg_wait));
 		if (IS_ERR(w))
 			return PTR_ERR(w);
 
io_uring/io_uring.h +16 -0
index 032758b28d78..a18da74f18e8 100644
@@ -514,4 +514,20 @@ static inline bool io_has_work(struct io_ring_ctx *ctx)
 	return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
 	       io_local_work_pending(ctx);
 }
+
+static inline void *io_args_get_ptr(struct io_reg_args *args,
+				    unsigned long offset, size_t size)
+{
+	unsigned long end;
+
+	if (unlikely(offset % sizeof(long)))
+		return ERR_PTR(-EFAULT);
+
+	/* also protects from NULL as the size would be 0 */
+	if (unlikely(check_add_overflow(offset, size, &end) || end > args->size))
+		return ERR_PTR(-EFAULT);
+
+	return args->ptr + array_index_nospec(offset, args->size - size);
+}
+
 #endif
io_uring/register.c +2 -2
index f1698c18c7cb..b926eb053408 100644
@@ -604,8 +604,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
 	}
 
 	if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) {
-		ctx->cq_wait_arg = io_region_get_ptr(&ctx->param_region);
-		ctx->cq_wait_size = rd.size;
+		ctx->wait_args.ptr = io_region_get_ptr(&ctx->param_region);
+		ctx->wait_args.size = rd.size;
 	}
 	return 0;
 }
Signature · 1 line
2.47.1
#2 lore raw

PATCH 2/4 io_uring: add registered request arguments

In reply to Pavel Begunkov

Similarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as ->wait_args has different lifetime rules, add a new instance of struct io_reg_args.

---
 include/linux/io_uring_types.h | 2 ++
 io_uring/register.c            | 3 +++
 2 files changed, 5 insertions(+)
include/linux/io_uring_types.h +2 -0
index 49008f00d064..cd6642855533 100644
@@ -299,6 +299,8 @@ struct io_ring_ctx {
 
 		struct io_submit_state	submit_state;
 
+		struct io_reg_args	sqe_args;
+
 		/*
 		 * Modifications are protected by ->uring_lock and ->mmap_lock.
 		 * The flags, buf_pages and buf_nr_pages fields should be stable
io_uring/register.c +3 -0
index b926eb053408..d2232b90a81d 100644
@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
 		ctx->wait_args.ptr = io_region_get_ptr(&ctx->param_region);
 		ctx->wait_args.size = rd.size;
 	}
+
+	ctx->sqe_args.ptr = io_region_get_ptr(&ctx->param_region);
+	ctx->sqe_args.size = rd.size;
 	return 0;
 }
 
Signature · 1 line
2.47.1
#3 lore raw

RE: [PATCH 2/4] io_uring: add registered request arguments

Hi,

Quoted 38 lines

-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 2/4] io_uring: add registered request arguments

Similarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as -

Quoted 1 line

wait_args has different lifetime rules, add a new instance of struct

io_reg_args.

---
 include/linux/io_uring_types.h | 2 ++
 io_uring/register.c            | 3 +++
 2 files changed, 5 insertions(+)
include/linux/io_uring_types.h +2 -0
index 49008f00d064..cd6642855533 100644
@@ -299,6 +299,8 @@ struct io_ring_ctx {
 
 		struct io_submit_state	submit_state;
 
+		struct io_reg_args	sqe_args;
+
 		/*
 		 * Modifications are protected by ->uring_lock and -
Quoted 1 line

mmap_lock.

* The flags, buf_pages and buf_nr_pages fields should be
stable diff --git a/io_uring/register.c b/io_uring/register.c index b926eb053408..d2232b90a81d 100644

io_uring/register.c static int io_register_mem_region(struct io_ring_ctx · line 613

Quoted 7 lines
io_uring/register.c +3 -0
@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx
 *ctx, void __user *uarg)
 		ctx->wait_args.ptr = io_region_get_ptr(&ctx->param_region);
 		ctx->wait_args.size = rd.size;
 	}
+
+	ctx->sqe_args.ptr = io_region_get_ptr(&ctx->param_region);
+	ctx->sqe_args.size = rd.size;

Why not add an enum value against sqe_args? Will mixing it with IORING_MEM_REGION_REG_WAIT_ARG cause management confusion?

Quoted 6 lines
io_uring/register.c +0 -1
 	return 0;
 }
 
--

2.47.1

---
Li Zetao

#4 lore raw

Re: [PATCH 2/4] io_uring: add registered request arguments

Quoted 34 lines · lizetao

Hi,

Quoted 32 lines

-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 2/4] io_uring: add registered request arguments

Similarly to registered wait arguments we want to have a pre-mapped space for various request arguments. Use the same parameter region, however as -

Quoted 1 line

wait_args has different lifetime rules, add a new instance of struct

io_reg_args.

---
  include/linux/io_uring_types.h | 2 ++
  io_uring/register.c            | 3 +++
  2 files changed, 5 insertions(+)
include/linux/io_uring_types.h +2 -0
index 49008f00d064..cd6642855533 100644
@@ -299,6 +299,8 @@ struct io_ring_ctx {
 
  		struct io_submit_state	submit_state;
 
+		struct io_reg_args	sqe_args;
+
  		/*
  		 * Modifications are protected by ->uring_lock and -
Quoted 1 line

mmap_lock.

Quoted 15 lines

io_uring/register.c static int io_register_mem_region(struct io_ring_ctx · line 613

Quoted 13 lines

* The flags, buf_pages and buf_nr_pages fields should be
stable diff --git a/io_uring/register.c b/io_uring/register.c index b926eb053408..d2232b90a81d 100644

io_uring/register.c +3 -0
@@ -607,6 +607,9 @@ static int io_register_mem_region(struct io_ring_ctx
 *ctx, void __user *uarg)
  		ctx->wait_args.ptr = io_region_get_ptr(&ctx->param_region);
  		ctx->wait_args.size = rd.size;
  	}
+
+	ctx->sqe_args.ptr = io_region_get_ptr(&ctx->param_region);
+	ctx->sqe_args.size = rd.size;

Why not add an enum value against sqe_args? Will mixing it with

Not sure what you mean

Quoted 1 line

IORING_MEM_REGION_REG_WAIT_ARG cause management confusion?

This specific region was added to be shared for multiple use cases, the rest is up to the userspace. liburing will need to have a clear API

Signature · 1 line
Pavel Begunkov
#5 lore raw

PATCH 3/4 io_uring/rw: use READ_ONCE with rw attributes

In reply to Pavel Begunkov

In preparation to pre-mapped attributes read struct io_uring_attr_pi with READ_ONCE and use an intermediate pointer.

---
 io_uring/rw.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
io_uring/rw.c +14 -8
index 75f70935ccf4..dc1acaf95db1 100644
@@ -264,23 +264,29 @@ static inline void io_meta_restore(struct io_async_rw *io, struct kiocb *kiocb)
 static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
 			 u64 attr_ptr, u64 attr_type_mask)
 {
-	struct io_uring_attr_pi pi_attr;
+	struct io_uring_attr_pi __pi_attr;
+	struct io_uring_attr_pi *pi_attr;
 	struct io_async_rw *io;
+	void __user *pi_addr;
+	size_t pi_len;
 	int ret;
 
-	if (copy_from_user(&pi_attr, u64_to_user_ptr(attr_ptr),
+	if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
 	    sizeof(pi_attr)))
 		return -EFAULT;
+	pi_attr = &__pi_attr;
 
-	if (pi_attr.rsvd)
+	if (pi_attr->rsvd)
 		return -EINVAL;
 
 	io = req->async_data;
-	io->meta.flags = pi_attr.flags;
-	io->meta.app_tag = pi_attr.app_tag;
-	io->meta.seed = pi_attr.seed;
-	ret = import_ubuf(ddir, u64_to_user_ptr(pi_attr.addr),
-			  pi_attr.len, &io->meta.iter);
+	io->meta.flags = READ_ONCE(pi_attr->flags);
+	io->meta.app_tag = READ_ONCE(pi_attr->app_tag);
+	io->meta.seed = READ_ONCE(pi_attr->seed);
+
+	pi_addr = u64_to_user_ptr(READ_ONCE(pi_attr->addr));
+	pi_len = READ_ONCE(pi_attr->len);
+	ret = import_ubuf(ddir, pi_addr, pi_len, &io->meta.iter);
 	if (unlikely(ret < 0))
 		return ret;
 	req->flags |= REQ_F_HAS_METADATA;
Signature · 1 line
2.47.1
#6 lore raw

PATCH 4/4 io_uring/rw: pre-mapped rw attributes

In reply to Pavel Begunkov

Instead of copy_from_user()'ing request attributes, allow it to be grabbwd from a registered pre-registered parameter region like we do with registered wait arguments.

---
 include/uapi/linux/io_uring.h |  4 +++-
 io_uring/rw.c                 | 19 ++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)
include/uapi/linux/io_uring.h +3 -1
index 38f0d6b10eaf..ec6e6fd37d1c 100644
@@ -112,7 +112,9 @@ struct io_uring_sqe {
 };
 
 /* sqe->attr_type_mask flags */
-#define IORING_RW_ATTR_FLAG_PI	(1U << 0)
+#define IORING_RW_ATTR_FLAG_PI		(1UL << 0)
+#define IORING_RW_ATTR_REGISTERED	(1UL << 63)
+
 /* PI attribute information */
 struct io_uring_attr_pi {
 		__u16	flags;
io_uring/rw.c +14 -5
index dc1acaf95db1..b1db4595788b 100644
@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
 	size_t pi_len;
 	int ret;
 
-	if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
-	    sizeof(pi_attr)))
-		return -EFAULT;
-	pi_attr = &__pi_attr;
+	if (attr_type_mask & IORING_RW_ATTR_REGISTERED) {
+		pi_attr = io_args_get_ptr(&req->ctx->sqe_args, attr_ptr,
+					  sizeof(pi_attr));
+		if (IS_ERR(pi_attr))
+			return PTR_ERR(pi_attr);
+	} else {
+		if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
+		    sizeof(pi_attr)))
+			return -EFAULT;
+		pi_attr = &__pi_attr;
+	}
 
 	if (pi_attr->rsvd)
 		return -EINVAL;
@@ -294,6 +301,8 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir,
 	return ret;
 }
 
+#define IO_RW_ATTR_ALLOWED_MASK (IORING_RW_ATTR_FLAG_PI | IORING_RW_ATTR_REGISTERED)
+
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		      int ddir, bool do_import)
 {
@@ -332,7 +341,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		u64 attr_ptr;
 
 		/* only PI attribute is supported currently */
-		if (attr_type_mask != IORING_RW_ATTR_FLAG_PI)
+		if (attr_type_mask & IO_RW_ATTR_ALLOWED_MASK)
 			return -EINVAL;
 
 		attr_ptr = READ_ONCE(sqe->attr_ptr);
Signature · 1 line
2.47.1
#7 lore raw

RE: [PATCH 4/4] io_uring/rw: pre-mapped rw attributes

Hi,

Quoted 24 lines

-----Original Message-----
From: Pavel Begunkov <[email protected]>
Sent: Monday, December 30, 2024 9:30 PM
To: [email protected]
Cc: [email protected]; Anuj Gupta <[email protected]>; Kanchan Joshi <[email protected]>
Subject: [PATCH 4/4] io_uring/rw: pre-mapped rw attributes

Instead of copy_from_user()'ing request attributes, allow it to be grabbwd from a registered pre-registered parameter region like we do with registered wait arguments.

---
 include/uapi/linux/io_uring.h |  4 +++-
 io_uring/rw.c                 | 19 ++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)
include/uapi/linux/io_uring.h index renamed from include/uapi/linux/io_uring.h +0 -0

38f0d6b10eaf..ec6e6fd37d1c 100644

include/uapi/linux/io_uring.h struct io_uring_sqe { }; · line 115

Quoted 5 lines
include/uapi/linux/io_uring.h +2 -1
@@ -112,7 +112,9 @@ struct io_uring_sqe {  };
 
 /* sqe->attr_type_mask flags */
-#define IORING_RW_ATTR_FLAG_PI	(1U << 0)
+#define IORING_RW_ATTR_FLAG_PI		(1UL << 0)
+#define IORING_RW_ATTR_REGISTERED	(1UL << 63)

Why use (1UL << 63) instead of (1UL << 1) here?

Quoted 10 lines
include/uapi/linux/io_uring.h +1 -0
+
 /* PI attribute information */
 struct io_uring_attr_pi {
 		__u16	flags;
io_uring/rw.c index dc1acaf95db1..b1db4595788b renamed from io_uring/rw.c +0 -0

100644

io_uring/rw.c +0 -0
@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct
 io_rw *rw, int ddir,

io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct · line 277

Quoted 10 lines
io_uring/rw.c +3 -4
 	size_t pi_len;
 	int ret;
 
-	if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
-	    sizeof(pi_attr)))
-		return -EFAULT;
-	pi_attr = &__pi_attr;
+	if (attr_type_mask & IORING_RW_ATTR_REGISTERED) {
+		pi_attr = io_args_get_ptr(&req->ctx->sqe_args, attr_ptr,
+					  sizeof(pi_attr));

Here pi_attr is just pointer, so maybe sizeof(__pi_attr) or sizeof(struct io_uring_attr_pi) will be better.

io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, · line 281

Quoted 5 lines
io_uring/rw.c +5 -0
+		if (IS_ERR(pi_attr))
+			return PTR_ERR(pi_attr);
+	} else {
+		if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
+		    sizeof(pi_attr)))

Just like above, shoule be sizeof(struct io_uring_attr_pi) here.

Quoted 18 lines
io_uring/rw.c +3 -0
+			return -EFAULT;
+		pi_attr = &__pi_attr;
+	}
 
 	if (pi_attr->rsvd)
io_uring/rw.c +3 -0
 		return -EINVAL;
@@ -294,6 +301,8 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct
 io_rw *rw, int ddir,
 	return ret;
 }
 
+#define IO_RW_ATTR_ALLOWED_MASK (IORING_RW_ATTR_FLAG_PI |
+IORING_RW_ATTR_REGISTERED)
+
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		      int ddir, bool do_import)
 {

io_uring/rw.c static int io_prep_rw(struct io_kiocb *req, const struct · line 336

Quoted 5 lines
io_uring/rw.c +0 -1
@@ -332,7 +341,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct
 io_uring_sqe *sqe,
 		u64 attr_ptr;
 
 		/* only PI attribute is supported currently */
-		if (attr_type_mask != IORING_RW_ATTR_FLAG_PI)

The comment needs to be adjusted.

io_uring/rw.c static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · line 344

Quoted 1 line
io_uring/rw.c +1 -0
+		if (attr_type_mask & IO_RW_ATTR_ALLOWED_MASK)

Here should be attr_type_mask & ~IO_RW_ATTR_ALLOWED_MASK ?

Quoted 6 lines
io_uring/rw.c +0 -0
 			return -EINVAL;
 

attr_ptr = READ_ONCE(sqe->attr_ptr);
--
2.47.1

--
Li Zetao

#8 lore raw

Re: [PATCH 4/4] io_uring/rw: pre-mapped rw attributes

On 1/6/25 11:13, lizetao wrote:
...

Quoted 5 lines

include/uapi/linux/io_uring.h struct io_uring_sqe { · line 116

Quoted 4 lines

/* sqe->attr_type_mask flags */

include/uapi/linux/io_uring.h +2 -1
-#define IORING_RW_ATTR_FLAG_PI	(1U << 0)
+#define IORING_RW_ATTR_FLAG_PI		(1UL << 0)
+#define IORING_RW_ATTR_REGISTERED	(1UL << 63)

Why use (1UL << 63) instead of (1UL << 1) here?

They serve different purposes, think how you'd be iterating more attribute types, calculating the total size, etc.

Quoted 9 lines
Quoted 9 lines
io_uring/rw.c index dc1acaf95db1..b1db4595788b renamed from io_uring/rw.c +0 -0

100644

io_uring/rw.c +0 -0
@@ -271,10 +271,17 @@ static int io_prep_rw_pi(struct io_kiocb *req, struct
 io_rw *rw, int ddir,
  	size_t pi_len;
  	int ret;
 
Quoted 8 lines

io_uring/rw.c static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, · line 276

Quoted 7 lines
io_uring/rw.c +3 -4
-	if (copy_from_user(&__pi_attr, u64_to_user_ptr(attr_ptr),
-	    sizeof(pi_attr)))
-		return -EFAULT;
-	pi_attr = &__pi_attr;
+	if (attr_type_mask & IORING_RW_ATTR_REGISTERED) {
+		pi_attr = io_args_get_ptr(&req->ctx->sqe_args, attr_ptr,
+					  sizeof(pi_attr));

Here pi_attr is just pointer, so maybe sizeof(__pi_attr) or sizeof(struct io_uring_attr_pi) will be better.

Good catch, and that's an issue from 3/4. As noted it's not even tested and posted to discuss the API. I'd even prefer them to be thrown away, and for Anuj / Kanchan to take over if that's interesting.

Signature · 1 line
Pavel Begunkov