Skip to contentSkip to conversation

io_uring IO interface

PATCH Superseded Jens Axboe 8 participants 69 messages Jan 28, 2019 – Feb 1, 2019

Subsystems (12)
Documentation x86 VFS Block layer +8 more subsystems

v8 of 15 Superseded Latest is v15 →

lore
Share & tools
lore

Jens Axboe

author maintainer
#0 lore raw

PATCH v8 1/1 io_uring IO interface

Here's v8 of the io_uring interface. Various little fixes all over the map, and addressing various review concerns.

No new features, but the io_uring_enter(2) system call grew arguments for a sigset_t so we can support poll properly.

For a general introduction to this patchset, see previous postings or the LWN writeup here:

https://lwn.net/Articles/776703/

No new changes in the liburing user side library, but as a reference, you can clone that here:

git://git.kernel.dk/liburing

We're still missing a man page for io_uring_enter(2), but the two other system calls are documented.

Patches are against 5.0-rc4, and can also be found in my io_uring branch here:

git://git.kernel.dk/linux-block io_uring

Changes since v7:
- Rebase on v5.0-rc4
- Add grace period control for SQ poll
- Add IORING_ENTER_SQ_WAKEUP instead of overloading 'to_submit'
- Address various minor review comments
- Use in_compat_syscall() instead of storing it in the ctx
- Remove now unneeded compat system call
- Ensure nops appropriately serialize the cq ring
- Add sigset_t support for wait side of io_uring_enter(2)
- Stop using page_frag_free()
- Remove duplicate include
- Make sure sq thread and application can't stomp on each other
- Add array_index_nospec() limiter for p->sq_thread_cpu

 Documentation/filesystems/vfs.txt      |    3 +
 arch/x86/entry/syscalls/syscall_32.tbl |    3 +
 arch/x86/entry/syscalls/syscall_64.tbl |    3 +
 block/bio.c                            |   59 +-
 fs/Makefile                            |    1 +
 fs/block_dev.c                         |   19 +-
 fs/file.c                              |   15 +-
 fs/file_table.c                        |    9 +-
 fs/gfs2/file.c                         |    2 +
 fs/io_uring.c                          | 2550 ++++++++++++++++++++++++
 fs/iomap.c                             |   48 +-
 fs/xfs/xfs_file.c                      |    1 +
 include/linux/bio.h                    |   14 +
 include/linux/blk_types.h              |    1 +
 include/linux/file.h                   |    2 +
 include/linux/fs.h                     |    6 +-
 include/linux/iomap.h                  |    1 +
 include/linux/sched/user.h             |    2 +-
 include/linux/syscalls.h               |    8 +
 include/uapi/asm-generic/unistd.h      |    8 +-
 include/uapi/linux/io_uring.h          |  143 ++
 init/Kconfig                           |    9 +
 kernel/sys_ni.c                        |    3 +
 23 files changed, 2869 insertions(+), 41 deletions(-)
Signature · 8 lines
Jens Axboe


--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#1 lore raw

PATCH 1/18 fs: add an iopoll method to struct file_operations

From: Christoph Hellwig <[email protected]>

This new methods is used to explicitly poll for I/O completion for an iocb. It must be called for any iocb submitted asynchronously (that is with a non-null ki_complete) which has the IOCB_HIPRI flag set.

The method is assisted by a new ki_cookie field in struct iocb to store the polling cookie.

---
 Documentation/filesystems/vfs.txt | 3 +++
 include/linux/fs.h                | 2 ++
 2 files changed, 5 insertions(+)
Documentation/filesystems/vfs.txt +3 -0
index 8dc8e9c2913f..761c6fd24a53 100644
@@ -857,6 +857,7 @@ struct file_operations {
 	ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
 	ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
 	ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
+	int (*iopoll)(struct kiocb *kiocb, bool spin);
 	int (*iterate) (struct file *, struct dir_context *);
 	int (*iterate_shared) (struct file *, struct dir_context *);
 	__poll_t (*poll) (struct file *, struct poll_table_struct *);
@@ -902,6 +903,8 @@ otherwise noted.
 
   write_iter: possibly asynchronous write with iov_iter as source
 
+  iopoll: called when aio wants to poll for completions on HIPRI iocbs
+
   iterate: called when the VFS needs to read the directory contents
 
   iterate_shared: called when the VFS needs to read the directory contents
include/linux/fs.h +2 -0
index 811c77743dad..ccb0b7a63aa5 100644
@@ -310,6 +310,7 @@ struct kiocb {
 	int			ki_flags;
 	u16			ki_hint;
 	u16			ki_ioprio; /* See linux/ioprio.h */
+	unsigned int		ki_cookie; /* for ->iopoll */
 } __randomize_layout;
 
 static inline bool is_sync_kiocb(struct kiocb *kiocb)
@@ -1786,6 +1787,7 @@ struct file_operations {
 	ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *);
 	ssize_t (*read_iter) (struct kiocb *, struct iov_iter *);
 	ssize_t (*write_iter) (struct kiocb *, struct iov_iter *);
+	int (*iopoll)(struct kiocb *kiocb, bool spin);
 	int (*iterate) (struct file *, struct dir_context *);
 	int (*iterate_shared) (struct file *, struct dir_context *);
 	__poll_t (*poll) (struct file *, struct poll_table_struct *);
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#2 lore raw

PATCH 2/18 block: wire up block device iopoll method

In reply to Jens Axboe

From: Christoph Hellwig <[email protected]>

Just call blk_poll on the iocb cookie, we can derive the block device from the inode trivially.

---
 fs/block_dev.c | 10 ++++++++++
 1 file changed, 10 insertions(+)
fs/block_dev.c +10 -0
index 58a4c1217fa8..f18d076a2596 100644
@@ -293,6 +293,14 @@ struct blkdev_dio {
 
 static struct bio_set blkdev_dio_pool;
 
+static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
+{
+	struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
+	struct request_queue *q = bdev_get_queue(bdev);
+
+	return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
+}
+
 static void blkdev_bio_end_io(struct bio *bio)
 {
 	struct blkdev_dio *dio = bio->bi_private;
@@ -410,6 +418,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
 				bio->bi_opf |= REQ_HIPRI;
 
 			qc = submit_bio(bio);
+			WRITE_ONCE(iocb->ki_cookie, qc);
 			break;
 		}
 
@@ -2076,6 +2085,7 @@ const struct file_operations def_blk_fops = {
 	.llseek		= block_llseek,
 	.read_iter	= blkdev_read_iter,
 	.write_iter	= blkdev_write_iter,
+	.iopoll		= blkdev_iopoll,
 	.mmap		= generic_file_mmap,
 	.fsync		= blkdev_fsync,
 	.unlocked_ioctl	= block_ioctl,
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#3 lore raw

PATCH 3/18 block: add bio_set_polled() helper

In reply to Jens Axboe

For the upcoming async polled IO, we can't sleep allocating requests. If we do, then we introduce a deadlock where the submitter already has async polled IO in-flight, but can't wait for them to complete since polled requests must be active found and reaped.

Utilize the helper in the blockdev DIRECT_IO code.

---
 fs/block_dev.c      |  4 ++--
 include/linux/bio.h | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)
fs/block_dev.c +2 -2
index f18d076a2596..392e2bfb636f 100644
@@ -247,7 +247,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
 		task_io_account_write(ret);
 	}
 	if (iocb->ki_flags & IOCB_HIPRI)
-		bio.bi_opf |= REQ_HIPRI;
+		bio_set_polled(&bio, iocb);
 
 	qc = submit_bio(&bio);
 	for (;;) {
@@ -415,7 +415,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
 		nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
 		if (!nr_pages) {
 			if (iocb->ki_flags & IOCB_HIPRI)
-				bio->bi_opf |= REQ_HIPRI;
+				bio_set_polled(bio, iocb);
 
 			qc = submit_bio(bio);
 			WRITE_ONCE(iocb->ki_cookie, qc);
include/linux/bio.h +14 -0
index 7380b094dcca..f6f0a2b3cbc8 100644
@@ -823,5 +823,19 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
 
 #endif /* CONFIG_BLK_DEV_INTEGRITY */
 
+/*
+ * Mark a bio as polled. Note that for async polled IO, the caller must
+ * expect -EWOULDBLOCK if we cannot allocate a request (or other resources).
+ * We cannot block waiting for requests on polled IO, as those completions
+ * must be found by the caller. This is different than IRQ driven IO, where
+ * it's safe to wait for IO to complete.
+ */
+static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
+{
+	bio->bi_opf |= REQ_HIPRI;
+	if (!is_sync_kiocb(kiocb))
+		bio->bi_opf |= REQ_NOWAIT;
+}
+
 #endif /* CONFIG_BLOCK */
 #endif /* __LINUX_BIO_H */
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#4 lore raw

PATCH 4/18 iomap: wire up the iopoll method

In reply to Jens Axboe

From: Christoph Hellwig <[email protected]>

Store the request queue the last bio was submitted to in the iocb private data in addition to the cookie so that we find the right block device. Also refactor the common direct I/O bio submission code into a nice little helper.

Modified to use bio_set_polled().

---
 fs/gfs2/file.c        |  2 ++
 fs/iomap.c            | 43 ++++++++++++++++++++++++++++---------------
 fs/xfs/xfs_file.c     |  1 +
 include/linux/iomap.h |  1 +
 4 files changed, 32 insertions(+), 15 deletions(-)
fs/gfs2/file.c +2 -0
index a2dea5bc0427..58a768e59712 100644
@@ -1280,6 +1280,7 @@ const struct file_operations gfs2_file_fops = {
 	.llseek		= gfs2_llseek,
 	.read_iter	= gfs2_file_read_iter,
 	.write_iter	= gfs2_file_write_iter,
+	.iopoll		= iomap_dio_iopoll,
 	.unlocked_ioctl	= gfs2_ioctl,
 	.mmap		= gfs2_mmap,
 	.open		= gfs2_open,
@@ -1310,6 +1311,7 @@ const struct file_operations gfs2_file_fops_nolock = {
 	.llseek		= gfs2_llseek,
 	.read_iter	= gfs2_file_read_iter,
 	.write_iter	= gfs2_file_write_iter,
+	.iopoll		= iomap_dio_iopoll,
 	.unlocked_ioctl	= gfs2_ioctl,
 	.mmap		= gfs2_mmap,
 	.open		= gfs2_open,
fs/iomap.c +28 -15

82 lines not on this page · raw message

fs/xfs/xfs_file.c +1 -0
index e47425071e65..60c2da41f0fc 100644
@@ -1203,6 +1203,7 @@ const struct file_operations xfs_file_operations = {
 	.write_iter	= xfs_file_write_iter,
 	.splice_read	= generic_file_splice_read,
 	.splice_write	= iter_file_splice_write,
+	.iopoll		= iomap_dio_iopoll,
 	.unlocked_ioctl	= xfs_file_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl	= xfs_file_compat_ioctl,
include/linux/iomap.h +1 -0
index 9a4258154b25..0fefb5455bda 100644
@@ -162,6 +162,7 @@ typedef int (iomap_dio_end_io_t)(struct kiocb *iocb, ssize_t ret,
 		unsigned flags);
 ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
 		const struct iomap_ops *ops, iomap_dio_end_io_t end_io);
+int iomap_dio_iopoll(struct kiocb *kiocb, bool spin);
 
 #ifdef CONFIG_SWAP
 struct file;
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#5 lore raw

PATCH 5/18 Add io_uring IO interface

In reply to Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

---
 arch/x86/entry/syscalls/syscall_32.tbl |    2 +
 arch/x86/entry/syscalls/syscall_64.tbl |    2 +
 fs/Makefile                            |    1 +
 fs/io_uring.c                          | 1090 ++++++++++++++++++++++++
 include/linux/syscalls.h               |    6 +
 include/uapi/asm-generic/unistd.h      |    6 +-
 include/uapi/linux/io_uring.h          |   96 +++
 init/Kconfig                           |    9 +
 kernel/sys_ni.c                        |    2 +
 9 files changed, 1213 insertions(+), 1 deletion(-)
 create mode 100644 fs/io_uring.c
 create mode 100644 include/uapi/linux/io_uring.h
arch/x86/entry/syscalls/syscall_32.tbl +2 -0
index 3cf7b533b3d1..481c126259e9 100644
@@ -398,3 +398,5 @@
 384	i386	arch_prctl		sys_arch_prctl			__ia32_compat_sys_arch_prctl
 385	i386	io_pgetevents		sys_io_pgetevents		__ia32_compat_sys_io_pgetevents
 386	i386	rseq			sys_rseq			__ia32_sys_rseq
+425	i386	io_uring_setup		sys_io_uring_setup		__ia32_sys_io_uring_setup
+426	i386	io_uring_enter		sys_io_uring_enter		__ia32_sys_io_uring_enter
arch/x86/entry/syscalls/syscall_64.tbl +2 -0
index f0b1709a5ffb..6a32a430c8e0 100644
@@ -343,6 +343,8 @@
 332	common	statx			__x64_sys_statx
 333	common	io_pgetevents		__x64_sys_io_pgetevents
 334	common	rseq			__x64_sys_rseq
+425	common	io_uring_setup		__x64_sys_io_uring_setup
+426	common	io_uring_enter		__x64_sys_io_uring_enter
 
 #
 # x32-specific system call numbers start at 512 to avoid cache impact
fs/Makefile +1 -0
index 293733f61594..8e15d6fc4340 100644
@@ -30,6 +30,7 @@ obj-$(CONFIG_TIMERFD)		+= timerfd.o
 obj-$(CONFIG_EVENTFD)		+= eventfd.o
 obj-$(CONFIG_USERFAULTFD)	+= userfaultfd.o
 obj-$(CONFIG_AIO)               += aio.o
+obj-$(CONFIG_IO_URING)		+= io_uring.o
 obj-$(CONFIG_FS_DAX)		+= dax.o
 obj-$(CONFIG_FS_ENCRYPTION)	+= crypto/
 obj-$(CONFIG_FILE_LOCKING)      += locks.o
fs/io_uring.c new file +1090 -0

1,090 lines not on this page · raw message

include/linux/syscalls.h +6 -0
index 257cccba3062..3072dbaa7869 100644
@@ -69,6 +69,7 @@ struct file_handle;
 struct sigaltstack;
 struct rseq;
 union bpf_attr;
+struct io_uring_params;
 
 #include <linux/types.h>
 #include <linux/aio_abi.h>
@@ -309,6 +310,11 @@ asmlinkage long sys_io_pgetevents_time32(aio_context_t ctx_id,
 				struct io_event __user *events,
 				struct old_timespec32 __user *timeout,
 				const struct __aio_sigset *sig);
+asmlinkage long sys_io_uring_setup(u32 entries,
+				struct io_uring_params __user *p);
+asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit,
+				u32 min_complete, u32 flags,
+				const sigset_t __user *sig, size_t sigsz);
 
 /* fs/xattr.c */
 asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
include/uapi/asm-generic/unistd.h +5 -1
index d90127298f12..87871e7b7ea7 100644
@@ -740,9 +740,13 @@ __SC_COMP(__NR_io_pgetevents, sys_io_pgetevents, compat_sys_io_pgetevents)
 __SYSCALL(__NR_rseq, sys_rseq)
 #define __NR_kexec_file_load 294
 __SYSCALL(__NR_kexec_file_load,     sys_kexec_file_load)
+#define __NR_io_uring_setup 425
+__SYSCALL(__NR_io_uring_setup, sys_io_uring_setup)
+#define __NR_io_uring_enter 426
+__SYSCALL(__NR_io_uring_enter, sys_io_uring_enter)
 
 #undef __NR_syscalls
-#define __NR_syscalls 295
+#define __NR_syscalls 427
 
 /*
  * 32 bit systems traditionally used different
include/uapi/linux/io_uring.h new file +96 -0

96 lines not on this page · raw message

init/Kconfig +9 -0
index 513fa544a134..0cf723867e69 100644
@@ -1403,6 +1403,15 @@ config AIO
 	  by some high performance threaded applications. Disabling
 	  this option saves about 7k.
 
+config IO_URING
+	bool "Enable IO uring support" if EXPERT
+	select ANON_INODES
+	default y
+	help
+	  This option enables support for the io_uring interface, enabling
+	  applications to submit and completion IO through submission and
+	  completion rings that are shared between the kernel and application.
+
 config ADVISE_SYSCALLS
 	bool "Enable madvise/fadvise syscalls" if EXPERT
 	default y
kernel/sys_ni.c +2 -0
index ab9d0e3c6d50..ee5e523564bb 100644
@@ -46,6 +46,8 @@ COND_SYSCALL(io_getevents);
 COND_SYSCALL(io_pgetevents);
 COND_SYSCALL_COMPAT(io_getevents);
 COND_SYSCALL_COMPAT(io_pgetevents);
+COND_SYSCALL(io_uring_setup);
+COND_SYSCALL(io_uring_enter);
 
 /* fs/xattr.c */
 
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#6 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

fs/io_uring.c line 702

Quoted 13 lines · Jens Axboe
fs/io_uring.c new file +13 -0
+static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
+			    unsigned min_complete, unsigned flags,
+			    const sigset_t __user *sig, size_t sigsz)
+{
+	int submitted, ret;
+
+	submitted = ret = 0;
+	if (to_submit) {
+		submitted = io_ring_submit(ctx, to_submit);
+		if (submitted < 0)
+			return submitted;
+	}
+	if (flags & IORING_ENTER_GETEVENTS) {

Do we want to disallow any unsupported flags?

-Jeff

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#7 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 17 lines · Jeff Moyer

fs/io_uring.c line 702

Quoted 13 lines · Jens Axboe
fs/io_uring.c new file +13 -0
+static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
+			    unsigned min_complete, unsigned flags,
+			    const sigset_t __user *sig, size_t sigsz)
+{
+	int submitted, ret;
+
+	submitted = ret = 0;
+	if (to_submit) {
+		submitted = io_ring_submit(ctx, to_submit);
+		if (submitted < 0)
+			return submitted;
+	}
+	if (flags & IORING_ENTER_GETEVENTS) {

Do we want to disallow any unsupported flags?

Yes good point, we do that in other spots. Fixed.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#8 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
fs/io_uring.c new file +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 34 lines
fs/io_uring.c new file +34 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;

fs/io_uring.c line 512

Quoted 10 lines
fs/io_uring.c new file +10 -0
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

Quoted 20 lines
fs/io_uring.c new file +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

fs/io_uring.c line 722

Quoted 5 lines
fs/io_uring.c new file +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

fs/io_uring.c line 732

Quoted 9 lines
fs/io_uring.c new file +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Quoted 15 lines
fs/io_uring.c new file +15 -0
+       /* Do QD, or 2 * CPUS, whatever is smallest */
+       ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
+                       min(ctx->sq_entries - 1, 2 * num_online_cpus()));
+       if (!ctx->sqo_wq) {
+               ret = -ENOMEM;
+               goto err;
+       }
+
+       return 0;
+err:
+       if (ctx->sqo_files)
+               ctx->sqo_files = NULL;
+       ctx->sqo_mm = NULL;
+       return ret;
+}

[...]

fs/io_uring.c line 932

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static const struct file_operations io_uring_fops = {
+       .release        = io_uring_release,
+       .mmap           = io_uring_mmap,
+       .poll           = io_uring_poll,
+       .fasync         = io_uring_fasync,
+};

[...]

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#9 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 172 lines · Jann Horn
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
fs/io_uring.c new file +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
Quoted 8 lines

fs/io_uring.c line 512

Quoted 5 lines
fs/io_uring.c new file +5 -0
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

'current' is the work queue item in this case, do we need to protect against anything else? I can add the locking around the assignments (both places).

Quoted 21 lines
Quoted 20 lines
fs/io_uring.c new file +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

Quoted 7 lines

fs/io_uring.c line 722

Quoted 5 lines
fs/io_uring.c new file +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

I think we're deadling with the same thing as the files below, I'll defer to that.

Quoted 12 lines

fs/io_uring.c line 732

Quoted 9 lines
fs/io_uring.c new file +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

Quoted 8 lines

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

The anonfd is CLOEXEC. The idea is exactly that it only runs when the last reference to the file has been dropped. Not sure why you think I need ->flush() here?

Quoted 4 lines

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Let me see if I can explain my intent better than that comment... We know the parent who set up the io_uring instance will be around for as long as io_uring instance persists. When we are tearing down the io_uring, then we wait for any async contexts (like the one above) to exit. Once they are exited, it's safe to proceed with the exit and teardown ->files[].

That's the idea... Not trying to be clever, some of this dates back to the aio weirdness where it was impossible to have cross references like this, since it would lead to teardown deadlocks with how exit_aio() is used. I can probably grab a struct files reference above, but currently I don't see why it's needed.

Quoted 24 lines
Quoted 15 lines
fs/io_uring.c new file +15 -0
+       /* Do QD, or 2 * CPUS, whatever is smallest */
+       ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
+                       min(ctx->sq_entries - 1, 2 * num_online_cpus()));
+       if (!ctx->sqo_wq) {
+               ret = -ENOMEM;
+               goto err;
+       }
+
+       return 0;
+err:
+       if (ctx->sqo_files)
+               ctx->sqo_files = NULL;
+       ctx->sqo_mm = NULL;
+       return ret;
+}

[...]

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static const struct file_operations io_uring_fops = {
+       .release        = io_uring_release,
+       .mmap           = io_uring_mmap,
+       .poll           = io_uring_poll,
+       .fasync         = io_uring_fasync,
+};

[...]

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#10 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 182 lines · Jens Axboe
Quoted 172 lines · Jann Horn
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
fs/io_uring.c new file +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
Quoted 8 lines

fs/io_uring.c line 512

Quoted 5 lines
fs/io_uring.c new file +5 -0
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

Quoted 3 lines

'current' is the work queue item in this case, do we need to protect against anything else? I can add the locking around the assignments (both places).

Stuff like proc_fd_link() uses get_files_struct(), which grabs a reference to your current files_struct protected only by task_lock(); and it doesn't use anything like READ_ONCE(), so even if the object lifetime is not a problem, get_files_struct() could potentially crash due to a double-read (reading task->files twice and assuming that the result will be the same). As far as I can tell, this procfs code also works on kernel threads.

Quoted 53 lines
Quoted 21 lines
Quoted 20 lines
fs/io_uring.c new file +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

Quoted 7 lines

fs/io_uring.c line 722

Quoted 5 lines
fs/io_uring.c new file +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

I think we're deadling with the same thing as the files below, I'll defer to that.

Quoted 12 lines

fs/io_uring.c line 732

Quoted 9 lines
fs/io_uring.c new file +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

Quoted 8 lines

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

Quoted 3 lines

The anonfd is CLOEXEC. The idea is exactly that it only runs when the last reference to the file has been dropped. Not sure why you think I need ->flush() here?

Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag? Or send the fd via SCM_RIGHTS?

Quoted 8 lines
Quoted 4 lines

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Let me see if I can explain my intent better than that comment... We know the parent who set up the io_uring instance will be around for as long as io_uring instance persists.

That's the part that I think is wrong: As far as I can tell, the parent can go away and you won't notice.

Also, note that "the parent" is different things for ->files and ->mm. You can have a multithreaded process whose threads don't have the same ->files, or multiple process that share ->files without sharing ->mm,
...

Quoted 4 lines

When we are tearing down the
io_uring, then we wait for any async contexts (like the one above) to exit. Once they are exited, it's safe to proceed with the exit and teardown ->files[].

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Quoted 5 lines

That's the idea... Not trying to be clever, some of this dates back to the aio weirdness where it was impossible to have cross references like this, since it would lead to teardown deadlocks with how exit_aio() is used. I can probably grab a struct files reference above, but currently I don't see why it's needed.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#11 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 187 lines · Jann Horn
Quoted 182 lines · Jens Axboe
Quoted 172 lines · Jann Horn
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
fs/io_uring.c new file +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
Quoted 8 lines

fs/io_uring.c line 512

Quoted 5 lines
fs/io_uring.c new file +5 -0
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

Quoted 3 lines

'current' is the work queue item in this case, do we need to protect against anything else? I can add the locking around the assignments (both places).

Quoted 7 lines

Stuff like proc_fd_link() uses get_files_struct(), which grabs a reference to your current files_struct protected only by task_lock(); and it doesn't use anything like READ_ONCE(), so even if the object lifetime is not a problem, get_files_struct() could potentially crash due to a double-read (reading task->files twice and assuming that the result will be the same). As far as I can tell, this procfs code also works on kernel threads.

OK, that does make sense. I've added the locking.

Quoted 53 lines
Quoted 53 lines
Quoted 21 lines
Quoted 20 lines
fs/io_uring.c new file +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

Quoted 7 lines

fs/io_uring.c line 722

Quoted 5 lines
fs/io_uring.c new file +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

I think we're deadling with the same thing as the files below, I'll defer to that.

Quoted 20 lines

fs/io_uring.c line 732

Quoted 9 lines
fs/io_uring.c new file +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

Quoted 6 lines
Quoted 3 lines

The anonfd is CLOEXEC. The idea is exactly that it only runs when the last reference to the file has been dropped. Not sure why you think I need ->flush() here?

Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag? Or send the fd via SCM_RIGHTS?

That would obviously be a problem...

Quoted 11 lines
Quoted 8 lines
Quoted 4 lines

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Let me see if I can explain my intent better than that comment... We know the parent who set up the io_uring instance will be around for as long as io_uring instance persists.

That's the part that I think is wrong: As far as I can tell, the parent can go away and you won't notice.

If that's the case, then the mm/files referencing needs to be looked over for sure. It's currently relying on the fact that the parent stays alive. If it can go away without ->release() being called, then we have issues.

Quoted 4 lines

Also, note that "the parent" is different things for ->files and ->mm. You can have a multithreaded process whose threads don't have the same ->files, or multiple process that share ->files without sharing ->mm,
...

Of course, I do realize that.

Quoted 7 lines
Quoted 4 lines

When we are tearing down the
io_uring, then we wait for any async contexts (like the one above) to exit. Once they are exited, it's safe to proceed with the exit and teardown ->files[].

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#12 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 4 lines · Jens Axboe
Quoted 2 lines

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

OK, so I reworked the files struct to just grab it, then we ensure that doesn't go away. For mm, it's a bit more tricky. I think the best solution here is to add a fops->flush() and check for the process exiting its files. If it does, we quiesce the async contexts and prevent further use of that mm. We can't just keep holding a reference to the mm like we do with the files.

That should solve both cases.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#13 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 6 lines · Jens Axboe
Quoted 4 lines · Jens Axboe
Quoted 2 lines

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

Quoted 8 lines

OK, so I reworked the files struct to just grab it, then we ensure that doesn't go away. For mm, it's a bit more tricky. I think the best solution here is to add a fops->flush() and check for the process exiting its files. If it does, we quiesce the async contexts and prevent further use of that mm. We can't just keep holding a reference to the mm like we do with the files.

That should solve both cases.

You still have to hold a reference on the mm though, I think (for example, because two tasks might be sharing the fd table without sharing the mm).

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#14 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 14 lines · Jann Horn
Quoted 13 lines · Jens Axboe
Quoted 4 lines · Jens Axboe
Quoted 2 lines

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

OK, so I reworked the files struct to just grab it, then we ensure that doesn't go away. For mm, it's a bit more tricky. I think the best solution here is to add a fops->flush() and check for the process exiting its files. If it does, we quiesce the async contexts and prevent further use of that mm. We can't just keep holding a reference to the mm like we do with the files.

Quoted 5 lines
Quoted 1 line

That should solve both cases.

You still have to hold a reference on the mm though, I think (for example, because two tasks might be sharing the fd table without sharing the mm).

Yes good point, except we can't hold a reference to it. But I think we can get around this by using an mmu notifier instead. That eliminates the need for ->flush() as well.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#15 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 15 lines · Jens Axboe
Quoted 14 lines · Jann Horn
Quoted 13 lines · Jens Axboe
Quoted 4 lines · Jens Axboe
Quoted 2 lines

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

OK, so I reworked the files struct to just grab it, then we ensure that doesn't go away. For mm, it's a bit more tricky. I think the best solution here is to add a fops->flush() and check for the process exiting its files. If it does, we quiesce the async contexts and prevent further use of that mm. We can't just keep holding a reference to the mm like we do with the files.

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

That should solve both cases.

You still have to hold a reference on the mm though, I think (for example, because two tasks might be sharing the fd table without sharing the mm).

Yes good point, except we can't hold a reference to it.

Why not? kvm_create_vm() does it, too:

    mmgrab(current->mm);
    kvm->mm = current->mm;
Quoted 3 lines

But I think
we can get around this by using an mmu notifier instead. That eliminates the need for ->flush() as well.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#16 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 22 lines · Jann Horn
Quoted 21 lines · Jens Axboe
Quoted 19 lines · Jann Horn
Quoted 6 lines · Jens Axboe
Quoted 4 lines · Jens Axboe
Quoted 2 lines

But you only do that teardown on ->release, right? And ->release doesn't have much to do with the process lifetime.

Yes, only on ->relase().

Quoted 8 lines

OK, so I reworked the files struct to just grab it, then we ensure that doesn't go away. For mm, it's a bit more tricky. I think the best solution here is to add a fops->flush() and check for the process exiting its files. If it does, we quiesce the async contexts and prevent further use of that mm. We can't just keep holding a reference to the mm like we do with the files.

That should solve both cases.

You still have to hold a reference on the mm though, I think (for example, because two tasks might be sharing the fd table without sharing the mm).

Quoted 6 lines
Quoted 1 line

Yes good point, except we can't hold a reference to it.

Why not? kvm_create_vm() does it, too:

    mmgrab(current->mm);
    kvm->mm = current->mm;

I missed that helper, was only looking at mmget(). But yeah, that'll do it!

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#17 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jann Horn

T24gVHVlLCAyMDE5LTAxLTI5IGF0IDAwOjU5ICswMTAwLCBKYW5uIEhvcm4gd3JvdGU6DQo+IE9u IFR1ZSwgSmFuIDI5LCAyMDE5IGF0IDEyOjQ3IEFNIEplbnMgQXhib2UgPGF4Ym9lQGtlcm5lbC5k az4gd3JvdGU6DQo+ID4gT24gMS8yOC8xOSAzOjMyIFBNLCBKYW5uIEhvcm4gd3JvdGU6DQo+ID4g PiBPbiBNb24sIEphbiAyOCwgMjAxOSBhdCAxMDozNSBQTSBKZW5zIEF4Ym9lIDxheGJvZUBrZXJu ZWwuZGs+IHdyb3RlOg0KPiA+ID4gPiBUaGUgc3VibWlzc2lvbiBxdWV1ZSAoU1EpIGFuZCBjb21w bGV0aW9uIHF1ZXVlIChDUSkgcmluZ3MgYXJlIHNoYXJlZA0KPiA+ID4gPiBiZXR3ZWVuIHRoZSBh cHBsaWNhdGlvbiBhbmQgdGhlIGtlcm5lbC4gVGhpcyBlbGltaW5hdGVzIHRoZSBuZWVkIHRvDQo+ ID4gPiA+IGNvcHkgZGF0YSBiYWNrIGFuZCBmb3J0aCB0byBzdWJtaXQgYW5kIGNvbXBsZXRlIElP Lg0KPiA+ID4gPiANCj4gPiA+ID4gSU8gc3VibWlzc2lvbnMgdXNlIHRoZSBpb191cmluZ19zcWUg ZGF0YSBzdHJ1Y3R1cmUsIGFuZCBjb21wbGV0aW9ucw0KPiA+ID4gPiBhcmUgZ2VuZXJhdGVkIGlu IHRoZSBmb3JtIG9mIGlvX3VyaW5nX3NxZSBkYXRhIHN0cnVjdHVyZXMuIFRoZSBTUQ0KPiA+ID4g PiByaW5nIGlzIGFuIGluZGV4IGludG8gdGhlIGlvX3VyaW5nX3NxZSBhcnJheSwgd2hpY2ggbWFr ZXMgaXQgcG9zc2libGUNCj4gPiA+ID4gdG8gc3VibWl0IGEgYmF0Y2ggb2YgSU9zIHdpdGhvdXQg dGhlbSBiZWluZyBjb250aWd1b3VzIGluIHRoZSByaW5nLg0KPiA+ID4gPiBUaGUgQ1EgcmluZyBp cyBhbHdheXMgY29udGlndW91cywgYXMgY29tcGxldGlvbiBldmVudHMgYXJlIGluaGVyZW50bHkN Cj4gPiA+ID4gdW5vcmRlcmVkLCBhbmQgaGVuY2UgYW55IGlvX3VyaW5nX2NxZSBlbnRyeSBjYW4g cG9pbnQgYmFjayB0byBhbg0KPiA+ID4gPiBhcmJpdHJhcnkgc3VibWlzc2lvbi4NCj4gPiA+ID4g DQo+ID4gPiA+IFR3byBuZXcgc3lzdGVtIGNhbGxzIGFyZSBhZGRlZCBmb3IgdGhpczoNCj4gPiA+ ID4gDQo+ID4gPiA+IGlvX3VyaW5nX3NldHVwKGVudHJpZXMsIHBhcmFtcykNCj4gPiA+ID4gICAg ICAgICBTZXRzIHVwIGEgY29udGV4dCBmb3IgZG9pbmcgYXN5bmMgSU8uIE9uIHN1Y2Nlc3MsIHJl dHVybnMgYSBmaWxlDQo+ID4gPiA+ICAgICAgICAgZGVzY3JpcHRvciB0aGF0IHRoZSBhcHBsaWNh dGlvbiBjYW4gbW1hcCB0byBnYWluIGFjY2VzcyB0byB0aGUNCj4gPiA+ID4gICAgICAgICBTUSBy aW5nLCBDUSByaW5nLCBhbmQgaW9fdXJpbmdfc3Flcy4NCj4gPiA+ID4gDQo+ID4gPiA+IGlvX3Vy aW5nX2VudGVyKGZkLCB0b19zdWJtaXQsIG1pbl9jb21wbGV0ZSwgZmxhZ3MsIHNpZ3NldCwgc2ln c2V0c2l6ZSkNCj4gPiA+ID4gICAgICAgICBJbml0aWF0ZXMgSU8gYWdhaW5zdCB0aGUgcmluZ3Mg bWFwcGVkIHRvIHRoaXMgZmQsIG9yIHdhaXRzIGZvcg0KPiA+ID4gPiAgICAgICAgIHRoZW0gdG8g Y29tcGxldGUsIG9yIGJvdGguIFRoZSBiZWhhdmlvciBpcyBjb250cm9sbGVkIGJ5IHRoZQ0KPiA+ ID4gPiAgICAgICAgIHBhcmFtZXRlcnMgcGFzc2VkIGluLiBJZiAndG9fc3VibWl0JyBpcyBub24t emVybywgdGhlbiB3ZSdsbA0KPiA+ID4gPiAgICAgICAgIHRyeSBhbmQgc3VibWl0IG5ldyBJTy4g SWYgSU9SSU5HX0VOVEVSX0dFVEVWRU5UUyBpcyBzZXQsIHRoZQ0KPiA+ID4gPiAgICAgICAgIGtl cm5lbCB3aWxsIHdhaXQgZm9yICdtaW5fY29tcGxldGUnIGV2ZW50cywgaWYgdGhleSBhcmVuJ3QN Cj4gPiA+ID4gICAgICAgICBhbHJlYWR5IGF2YWlsYWJsZS4gSXQncyB2YWxpZCB0byBzZXQgSU9S SU5HX0VOVEVSX0dFVEVWRU5UUw0KPiA+ID4gPiAgICAgICAgIGFuZCAnbWluX2NvbXBsZXRlJyA9 PSAwIGF0IHRoZSBzYW1lIHRpbWUsIHRoaXMgYWxsb3dzIHRoZQ0KPiA+ID4gPiAgICAgICAgIGtl cm5lbCB0byByZXR1cm4gYWxyZWFkeSBjb21wbGV0ZWQgZXZlbnRzIHdpdGhvdXQgd2FpdGluZw0K PiA+ID4gPiAgICAgICAgIGZvciB0aGVtLiBUaGlzIGlzIHVzZWZ1bCBvbmx5IGZvciBwb2xsaW5n LCBhcyBmb3IgSVJRDQo+ID4gPiA+ICAgICAgICAgZHJpdmVuIElPLCB0aGUgYXBwbGljYXRpb24g Y2FuIGp1c3QgY2hlY2sgdGhlIENRIHJpbmcNCj4gPiA+ID4gICAgICAgICB3aXRob3V0IGVudGVy aW5nIHRoZSBrZXJuZWwuDQo+ID4gPiA+IA0KPiA+ID4gPiBXaXRoIHRoaXMgc2V0dXAsIGl0J3Mg cG9zc2libGUgdG8gZG8gYXN5bmMgSU8gd2l0aCBhIHNpbmdsZSBzeXN0ZW0NCj4gPiA+ID4gY2Fs bC4gRnV0dXJlIGRldmVsb3BtZW50cyB3aWxsIGVuYWJsZSBwb2xsZWQgSU8gd2l0aCB0aGlzIGlu dGVyZmFjZSwNCj4gPiA+ID4gYW5kIHBvbGxlZCBzdWJtaXNzaW9uIGFzIHdlbGwuIFRoZSBsYXR0 ZXIgd2lsbCBlbmFibGUgYW4gYXBwbGljYXRpb24NCj4gPiA+ID4gdG8gZG8gSU8gd2l0aG91dCBk b2luZyBBTlkgc3lzdGVtIGNhbGxzIGF0IGFsbC4NCj4gPiA+ID4gDQo+ID4gPiA+IEZvciBJUlEg ZHJpdmVuIElPLCBhbiBhcHBsaWNhdGlvbiBvbmx5IG5lZWRzIHRvIGVudGVyIHRoZSBrZXJuZWwg Zm9yDQo+ID4gPiA+IGNvbXBsZXRpb25zIGlmIGl0IHdhbnRzIHRvIHdhaXQgZm9yIHRoZW0gdG8g b2NjdXIuDQo+ID4gPiA+IA0KPiA+ID4gPiBFYWNoIGlvX3VyaW5nIGlzIGJhY2tlZCBieSBhIHdv cmtxdWV1ZSwgdG8gc3VwcG9ydCBidWZmZXJlZCBhc3luYyBJTw0KPiA+ID4gPiBhcyB3ZWxsLiBX ZSB3aWxsIG9ubHkgcHVudCB0byBhbiBhc3luYyBjb250ZXh0IGlmIHRoZSBjb21tYW5kIHdvdWxk DQo+ID4gPiA+IG5lZWQgdG8gd2FpdCBmb3IgSU8gb24gdGhlIGRldmljZSBzaWRlLiBBbnkgZGF0 YSB0aGF0IGNhbiBiZSBhY2Nlc3NlZA0KPiA+ID4gPiBkaXJlY3RseSBpbiB0aGUgcGFnZSBjYWNo ZSBpcyBkb25lIGlubGluZS4gVGhpcyBhdm9pZHMgdGhlIHNsb3duZXNzDQo+ID4gPiA+IGlzc3Vl IG9mIHVzdWFsIHRocmVhZHBvb2xzLCBzaW5jZSBjYWNoZWQgZGF0YSBpcyBhY2Nlc3NlZCBhcyBx dWlja2x5DQo+ID4gPiA+IGFzIGEgc3luYyBpbnRlcmZhY2UuDQo+ID4gPiA+IA0KPiA+ID4gPiBT YW1wbGUgYXBwbGljYXRpb246IGh0dHBzOi8vdXJsZGVmZW5zZS5wcm9vZnBvaW50LmNvbS92Mi91 cmw/dT1odHRwLTNBX19naXQua2VybmVsLmRrX2NnaXRfZmlvX3BsYWluX3RfaW8tNUZ1cmluZy5j JmQ9RHdJQmFRJmM9NVZEMFJUdE5sVGgzeWNkNDFiM01VdyZyPXBxTS1lTzRBMmhORmhJRmlYLTdl R2cmbT1NR3IxNHBPek5iQzdaLThfZFY0R01pSDNBYmtrSDBSU1FvUTg5NFR1MHljJnM9bWdiY3Vi ek9NaUNwRnBud1ctSEEzZXkwWURZUGtnTUlaN0JteTR3NkNoYyZlPQ0KPiA+ID4gDQo+ID4gPiBb Li4uXQ0KPiA+ID4gPiArc3RhdGljIGludCBpb19wcmVwX3J3KHN0cnVjdCBpb19raW9jYiAqcmVx LCBjb25zdCBzdHJ1Y3QgaW9fdXJpbmdfc3FlICpzcWUsDQo+ID4gPiA+ICsgICAgICAgICAgICAg ICAgICAgICBib29sIGZvcmNlX25vbmJsb2NrKQ0KPiA+ID4gPiArew0KPiA+ID4gPiArICAgICAg IHN0cnVjdCBraW9jYiAqa2lvY2IgPSAmcmVxLT5ydzsNCj4gPiA+ID4gKyAgICAgICBpbnQgcmV0 Ow0KPiA+ID4gPiArDQo+ID4gPiA+ICsgICAgICAga2lvY2ItPmtpX2ZpbHAgPSBmZ2V0KHNxZS0+ ZmQpOw0KPiA+ID4gPiArICAgICAgIGlmICh1bmxpa2VseSgha2lvY2ItPmtpX2ZpbHApKQ0KPiA+ ID4gPiArICAgICAgICAgICAgICAgcmV0dXJuIC1FQkFERjsNCj4gPiA+ID4gKyAgICAgICBraW9j Yi0+a2lfcG9zID0gc3FlLT5vZmY7DQo+ID4gPiA+ICsgICAgICAga2lvY2ItPmtpX2ZsYWdzID0g aW9jYl9mbGFncyhraW9jYi0+a2lfZmlscCk7DQo+ID4gPiA+ICsgICAgICAga2lvY2ItPmtpX2hp bnQgPSBraV9oaW50X3ZhbGlkYXRlKGZpbGVfd3JpdGVfaGludChraW9jYi0+a2lfZmlscCkpOw0K PiA+ID4gPiArICAgICAgIGlmIChzcWUtPmlvcHJpbykgew0KPiA+ID4gPiArICAgICAgICAgICAg ICAgcmV0ID0gaW9wcmlvX2NoZWNrX2NhcChzcWUtPmlvcHJpbyk7DQo+ID4gPiA+ICsgICAgICAg ICAgICAgICBpZiAocmV0KQ0KPiA+ID4gPiArICAgICAgICAgICAgICAgICAgICAgICBnb3RvIG91 dF9mcHV0Ow0KPiA+ID4gPiArDQo+ID4gPiA+ICsgICAgICAgICAgICAgICBraW9jYi0+a2lfaW9w cmlvID0gc3FlLT5pb3ByaW87DQo+ID4gPiA+ICsgICAgICAgfSBlbHNlDQo+ID4gPiA+ICsgICAg ICAgICAgICAgICBraW9jYi0+a2lfaW9wcmlvID0gZ2V0X2N1cnJlbnRfaW9wcmlvKCk7DQo+ID4g PiA+ICsNCj4gPiA+ID4gKyAgICAgICByZXQgPSBraW9jYl9zZXRfcndfZmxhZ3Moa2lvY2IsIHNx ZS0+cndfZmxhZ3MpOw0KPiA+ID4gPiArICAgICAgIGlmICh1bmxpa2VseShyZXQpKQ0KPiA+ID4g PiArICAgICAgICAgICAgICAgZ290byBvdXRfZnB1dDsNCj4gPiA+ID4gKyAgICAgICBpZiAoZm9y Y2Vfbm9uYmxvY2spIHsNCj4gPiA+ID4gKyAgICAgICAgICAgICAgIGtpb2NiLT5raV9mbGFncyB8 PSBJT0NCX05PV0FJVDsNCj4gPiA+ID4gKyAgICAgICAgICAgICAgIHJlcS0+ZmxhZ3MgfD0gUkVR X0ZfRk9SQ0VfTk9OQkxPQ0s7DQo+ID4gPiA+ICsgICAgICAgfQ0KPiA+ID4gPiArICAgICAgIGlm IChraW9jYi0+a2lfZmxhZ3MgJiBJT0NCX0hJUFJJKSB7DQo+ID4gPiA+ICsgICAgICAgICAgICAg ICByZXQgPSAtRUlOVkFMOw0KPiA+ID4gPiArICAgICAgICAgICAgICAgZ290byBvdXRfZnB1dDsN Cj4gPiA+ID4gKyAgICAgICB9DQo+ID4gPiA+ICsNCj4gPiA+ID4gKyAgICAgICBraW9jYi0+a2lf Y29tcGxldGUgPSBpb19jb21wbGV0ZV9ydzsNCj4gPiA+ID4gKyAgICAgICByZXR1cm4gMDsNCj4g PiA+ID4gK291dF9mcHV0Og0KPiA+ID4gPiArICAgICAgIGZwdXQoa2lvY2ItPmtpX2ZpbHApOw0K PiA+ID4gPiArICAgICAgIHJldHVybiByZXQ7DQo+ID4gPiA+ICt9DQo+ID4gPiANCj4gPiA+IFsu Li5dDQo+ID4gPiA+ICtzdGF0aWMgc3NpemVfdCBpb19yZWFkKHN0cnVjdCBpb19raW9jYiAqcmVx LCBjb25zdCBzdHJ1Y3QgaW9fdXJpbmdfc3FlICpzcWUsDQo+ID4gPiA+ICsgICAgICAgICAgICAg ICAgICAgICAgYm9vbCBmb3JjZV9ub25ibG9jaykNCj4gPiA+ID4gK3sNCj4gPiA+ID4gKyAgICAg ICBzdHJ1Y3QgaW92ZWMgaW5saW5lX3ZlY3NbVUlPX0ZBU1RJT1ZdLCAqaW92ZWMgPSBpbmxpbmVf dmVjczsNCj4gPiA+ID4gKyAgICAgICBzdHJ1Y3Qga2lvY2IgKmtpb2NiID0gJnJlcS0+cnc7DQo+ ID4gPiA+ICsgICAgICAgc3RydWN0IGlvdl9pdGVyIGl0ZXI7DQo+ID4gPiA+ICsgICAgICAgc3Ry dWN0IGZpbGUgKmZpbGU7DQo+ID4gPiA+ICsgICAgICAgc3NpemVfdCByZXQ7DQo+ID4gPiA+ICsN Cj4gPiA+ID4gKyAgICAgICByZXQgPSBpb19wcmVwX3J3KHJlcSwgc3FlLCBmb3JjZV9ub25ibG9j ayk7DQo+ID4gPiA+ICsgICAgICAgaWYgKHJldCkNCj4gPiA+ID4gKyAgICAgICAgICAgICAgIHJl dHVybiByZXQ7DQo+ID4gPiA+ICsgICAgICAgZmlsZSA9IGtpb2NiLT5raV9maWxwOw0KPiA+ID4g PiArDQo+ID4gPiA+ICsgICAgICAgcmV0ID0gLUVCQURGOw0KPiA+ID4gPiArICAgICAgIGlmICh1 bmxpa2VseSghKGZpbGUtPmZfbW9kZSAmIEZNT0RFX1JFQUQpKSkNCj4gPiA+ID4gKyAgICAgICAg ICAgICAgIGdvdG8gb3V0X2ZwdXQ7DQo+ID4gPiA+ICsgICAgICAgcmV0ID0gLUVJTlZBTDsNCj4g PiA+ID4gKyAgICAgICBpZiAodW5saWtlbHkoIWZpbGUtPmZfb3AtPnJlYWRfaXRlcikpDQo+ID4g PiA+ICsgICAgICAgICAgICAgICBnb3RvIG91dF9mcHV0Ow0KPiA+ID4gPiArDQo+ID4gPiA+ICsg ICAgICAgcmV0ID0gaW9faW1wb3J0X2lvdmVjKHJlcS0+Y3R4LCBSRUFELCBzcWUsICZpb3ZlYywg Jml0ZXIpOw0KPiA+ID4gPiArICAgICAgIGlmIChyZXQpDQo+ID4gPiA+ICsgICAgICAgICAgICAg ICBnb3RvIG91dF9mcHV0Ow0KPiA+ID4gPiArDQo+ID4gPiA+ICsgICAgICAgcmV0ID0gcndfdmVy aWZ5X2FyZWEoUkVBRCwgZmlsZSwgJmtpb2NiLT5raV9wb3MsIGlvdl9pdGVyX2NvdW50KCZpdGVy KSk7DQo+ID4gPiA+ICsgICAgICAgaWYgKCFyZXQpIHsNCj4gPiA+ID4gKyAgICAgICAgICAgICAg IHNzaXplX3QgcmV0MjsNCj4gPiA+ID4gKw0KPiA+ID4gPiArICAgICAgICAgICAgICAgLyogQ2F0 Y2ggLUVBR0FJTiByZXR1cm4gZm9yIGZvcmNlZCBub24tYmxvY2tpbmcgc3VibWlzc2lvbiAqLw0K PiA+ID4gPiArICAgICAgICAgICAgICAgcmV0MiA9IGNhbGxfcmVhZF9pdGVyKGZpbGUsIGtpb2Ni LCAmaXRlcik7DQo+ID4gPiA+ICsgICAgICAgICAgICAgICBpZiAoIWZvcmNlX25vbmJsb2NrIHx8 IHJldDIgIT0gLUVBR0FJTikNCj4gPiA+ID4gKyAgICAgICAgICAgICAgICAgICAgICAgaW9fcndf ZG9uZShraW9jYiwgcmV0Mik7DQo+ID4gPiA+ICsgICAgICAgICAgICAgICBlbHNlDQo+ID4gPiA+ ICsgICAgICAgICAgICAgICAgICAgICAgIHJldCA9IC1FQUdBSU47DQo+ID4gPiA+ICsgICAgICAg fQ0KPiA+ID4gPiArICAgICAgIGtmcmVlKGlvdmVjKTsNCj4gPiA+ID4gK291dF9mcHV0Og0KPiA+ ID4gPiArICAgICAgIGlmICh1bmxpa2VseShyZXQpKQ0KPiA+ID4gPiArICAgICAgICAgICAgICAg ZnB1dChmaWxlKTsNCj4gPiA+ID4gKyAgICAgICByZXR1cm4gcmV0Ow0KPiA+ID4gPiArfQ0KPiA+ ID4gDQo+ID4gPiBbLi4uXQ0KPiA+ID4gPiArc3RhdGljIGludCBfX2lvX3N1Ym1pdF9zcWUoc3Ry dWN0IGlvX3JpbmdfY3R4ICpjdHgsIHN0cnVjdCBpb19raW9jYiAqcmVxLA0KPiA+ID4gPiArICAg ICAgICAgICAgICAgICAgICAgICAgICBzdHJ1Y3Qgc3FlX3N1Ym1pdCAqcywgYm9vbCBmb3JjZV9u b25ibG9jaykNCj4gPiA+ID4gK3sNCj4gPiA+ID4gKyAgICAgICBjb25zdCBzdHJ1Y3QgaW9fdXJp bmdfc3FlICpzcWUgPSBzLT5zcWU7DQo+ID4gPiA+ICsgICAgICAgc3NpemVfdCByZXQ7DQo+ID4g PiA+ICsNCj4gPiA+ID4gKyAgICAgICBpZiAodW5saWtlbHkocy0+aW5kZXggPj0gY3R4LT5zcV9l bnRyaWVzKSkNCj4gPiA+ID4gKyAgICAgICAgICAgICAgIHJldHVybiAtRUlOVkFMOw0KPiA+ID4g PiArICAgICAgIHJlcS0+dXNlcl9kYXRhID0gc3FlLT51c2VyX2RhdGE7DQo+ID4gPiA+ICsNCj4g PiA+ID4gKyAgICAgICByZXQgPSAtRUlOVkFMOw0KPiA+ID4gPiArICAgICAgIHN3aXRjaCAoc3Fl LT5vcGNvZGUpIHsNCj4gPiA+ID4gKyAgICAgICBjYXNlIElPUklOR19PUF9OT1A6DQo+ID4gPiA+ ICsgICAgICAgICAgICAgICByZXQgPSBpb19ub3AocmVxLCBzcWUpOw0KPiA+ID4gPiArICAgICAg ICAgICAgICAgYnJlYWs7DQo+ID4gPiA+ICsgICAgICAgY2FzZSBJT1JJTkdfT1BfUkVBRFY6DQo+ ID4gPiA+ICsgICAgICAgICAgICAgICByZXQgPSBpb19yZWFkKHJlcSwgc3FlLCBmb3JjZV9ub25i bG9jayk7DQo+ID4gPiA+ICsgICAgICAgICAgICAgICBicmVhazsNCj4gPiA+ID4gKyAgICAgICBj YXNlIElPUklOR19PUF9XUklURVY6DQo+ID4gPiA+ICsgICAgICAgICAgICAgICByZXQgPSBpb193 cml0ZShyZXEsIHNxZSwgZm9yY2Vfbm9uYmxvY2spOw0KPiA+ID4gPiArICAgICAgICAgICAgICAg YnJlYWs7DQo+ID4gPiA+ICsgICAgICAgZGVmYXVsdDoNCj4gPiA+ID4gKyAgICAgICAgICAgICAg IHJldCA9IC1FSU5WQUw7DQo+ID4gPiA+ICsgICAgICAgICAgICAgICBicmVhazsNCj4gPiA+ID4g KyAgICAgICB9DQo+ID4gPiA+ICsNCj4gPiA+ID4gKyAgICAgICByZXR1cm4gcmV0Ow0KPiA+ID4g PiArfQ0KPiA+ID4gPiArDQo+ID4gPiA+ICtzdGF0aWMgdm9pZCBpb19zcV93cV9zdWJtaXRfd29y ayhzdHJ1Y3Qgd29ya19zdHJ1Y3QgKndvcmspDQo+ID4gPiA+ICt7DQo+ID4gPiA+ICsgICAgICAg c3RydWN0IGlvX2tpb2NiICpyZXEgPSBjb250YWluZXJfb2Yod29yaywgc3RydWN0IGlvX2tpb2Ni LCB3b3JrKTsNCj4gPiA+ID4gKyAgICAgICBzdHJ1Y3Qgc3FlX3N1Ym1pdCAqcyA9ICZyZXEtPnN1 Ym1pdDsNCj4gPiA+ID4gKyAgICAgICB1NjQgdXNlcl9kYXRhID0gcy0+c3FlLT51c2VyX2RhdGE7 DQo+ID4gPiA+ICsgICAgICAgc3RydWN0IGlvX3JpbmdfY3R4ICpjdHggPSByZXEtPmN0eDsNCj4g PiA+ID4gKyAgICAgICBtbV9zZWdtZW50X3Qgb2xkX2ZzID0gZ2V0X2ZzKCk7DQo+ID4gPiA+ICsg ICAgICAgc3RydWN0IGZpbGVzX3N0cnVjdCAqb2xkX2ZpbGVzOw0KPiA+ID4gPiArICAgICAgIGlu dCByZXQ7DQo+ID4gPiA+ICsNCj4gPiA+ID4gKyAgICAgICAgLyogRW5zdXJlIHdlIGNsZWFyIHBy ZXZpb3VzbHkgc2V0IGZvcmNlZCBub24tYmxvY2sgZmxhZyAqLw0KPiA+ID4gPiArICAgICAgIHJl cS0+ZmxhZ3MgJj0gflJFUV9GX0ZPUkNFX05PTkJMT0NLOw0KPiA+ID4gPiArDQo+ID4gPiA+ICsg ICAgICAgb2xkX2ZpbGVzID0gY3VycmVudC0+ZmlsZXM7DQo+ID4gPiA+ICsgICAgICAgY3VycmVu dC0+ZmlsZXMgPSBjdHgtPnNxb19maWxlczsNCj4gPiA+IA0KPiA+ID4gSSB0aGluayB5b3UncmUg bm90IHN1cHBvc2VkIHRvIHR3aWRkbGUgd2l0aCBjdXJyZW50LT5maWxlcyB3aXRob3V0DQo+ID4g PiBob2xkaW5nIHRhc2tfbG9jayhjdXJyZW50KS4NCj4gPiANCj4gPiAnY3VycmVudCcgaXMgdGhl IHdvcmsgcXVldWUgaXRlbSBpbiB0aGlzIGNhc2UsIGRvIHdlIG5lZWQgdG8gcHJvdGVjdA0KPiA+ IGFnYWluc3QgYW55dGhpbmcgZWxzZT8gSSBjYW4gYWRkIHRoZSBsb2NraW5nIGFyb3VuZCB0aGUg YXNzaWdubWVudHMNCj4gPiAoYm90aCBwbGFjZXMpLg0KPiANCj4gU3R1ZmYgbGlrZSBwcm9jX2Zk X2xpbmsoKSB1c2VzIGdldF9maWxlc19zdHJ1Y3QoKSwgd2hpY2ggZ3JhYnMgYQ0KPiByZWZlcmVu Y2UgdG8geW91ciBjdXJyZW50IGZpbGVzX3N0cnVjdCBwcm90ZWN0ZWQgb25seSBieSB0YXNrX2xv Y2soKTsNCj4gYW5kIGl0IGRvZXNuJ3QgdXNlIGFueXRoaW5nIGxpa2UgUkVBRF9PTkNFKCksIHNv IGV2ZW4gaWYgdGhlIG9iamVjdA0KPiBsaWZldGltZSBpcyBub3QgYSBwcm9ibGVtLCBnZXRfZmls ZXNfc3RydWN0KCkgY291bGQgcG90ZW50aWFsbHkgY3Jhc2gNCj4gZHVlIHRvIGEgZG91YmxlLXJl YWQgKHJlYWRpbmcgdGFzay0+ZmlsZXMgdHdpY2UgYW5kIGFzc3VtaW5nIHRoYXQgdGhlDQo+IHJl c3VsdCB3aWxsIGJlIHRoZSBzYW1lKS4gQXMgZmFyIGFzIEkgY2FuIHRlbGwsIHRoaXMgcHJvY2Zz IGNvZGUgYWxzbw0KPiB3b3JrcyBvbiBrZXJuZWwgdGhyZWFkcy4NCj4gDQo+ID4gPiA+ICsgICAg ICAgaWYgKCFtbWdldF9ub3RfemVybyhjdHgtPnNxb19tbSkpIHsNCj4gPiA+ID4gKyAgICAgICAg ICAgICAgIHJldCA9IC1FRkFVTFQ7DQo+ID4gPiA+ICsgICAgICAgICAgICAgICBnb3RvIGVycjsN Cj4gPiA+ID4gKyAgICAgICB9DQo+ID4gPiA+ICsNCj4gPiA+ID4gKyAgICAgICB1c2VfbW0oY3R4 LT5zcW9fbW0pOw0KPiA+ID4gPiArICAgICAgIHNldF9mcyhVU0VSX0RTKTsNCj4gPiA+ID4gKw0K PiA+ID4gPiArICAgICAgIHJldCA9IF9faW9fc3VibWl0X3NxZShjdHgsIHJlcSwgcywgZmFsc2Up Ow0KPiA+ID4gPiArDQo+ID4gPiA+ICsgICAgICAgc2V0X2ZzKG9sZF9mcyk7DQo+ID4gPiA+ICsg ICAgICAgdW51c2VfbW0oY3R4LT5zcW9fbW0pOw0KPiA+ID4gPiArICAgICAgIG1tcHV0KGN0eC0+ c3FvX21tKTsNCj4gPiA+ID4gK2VycjoNCj4gPiA+ID4gKyAgICAgICBpZiAocmV0KSB7DQo+ID4g PiA+ICsgICAgICAgICAgICAgICBpb19jcXJpbmdfYWRkX2V2ZW50KGN0eCwgdXNlcl9kYXRhLCBy ZXQsIDApOw0KPiA+ID4gPiArICAgICAgICAgICAgICAgaW9fZnJlZV9yZXEocmVxKTsNCj4gPiA+ ID4gKyAgICAgICB9DQo+ID4gPiA+ICsgICAgICAgY3VycmVudC0+ZmlsZXMgPSBvbGRfZmlsZXM7 DQo+ID4gPiA+ICt9DQo+ID4gPiANCj4gPiA+IFsuLi5dDQo+ID4gPiA+ICtzdGF0aWMgaW50IGlv X3NxX29mZmxvYWRfc3RhcnQoc3RydWN0IGlvX3JpbmdfY3R4ICpjdHgpDQo+ID4gPiA+ICt7DQo+ ID4gPiA+ICsgICAgICAgaW50IHJldDsNCj4gPiA+ID4gKw0KPiA+ID4gPiArICAgICAgIGN0eC0+ c3FvX21tID0gY3VycmVudC0+bW07DQo+ID4gPiANCj4gPiA+IFdoYXQga2VlcHMgdGhpcyB0aGlu ZyBhbGl2ZT8NCj4gPiANCj4gPiBJIHRoaW5rIHdlJ3JlIGRlYWRsaW5nIHdpdGggdGhlIHNhbWUg dGhpbmcgYXMgdGhlIGZpbGVzIGJlbG93LCBJJ2xsDQo+ID4gZGVmZXIgdG8gdGhhdC4NCj4gPiAN Cj4gPiA+ID4gKyAgICAgICAvKg0KPiA+ID4gPiArICAgICAgICAqIFRoaXMgaXMgc2FmZSBzaW5j ZSAnY3VycmVudCcgaGFzIHRoZSBmZCBpbnN0YWxsZWQsIGFuZCBpZiB0aGF0IGdldHMNCj4gPiA+ ID4gKyAgICAgICAgKiBjbG9zZWQgb24gZXhpdCwgdGhlbiBmb3BzLT5yZWxlYXNlKCkgaXMgaW52 b2tlZCB3aGljaCB3YWl0cyBmb3IgdGhlDQo+ID4gPiA+ICsgICAgICAgICogYXN5bmMgY29udGV4 dHMgdG8gZmx1c2ggYW5kIGV4aXQgYmVmb3JlIGV4aXRpbmcuDQo+ID4gPiA+ICsgICAgICAgICov DQo+ID4gPiA+ICsgICAgICAgcmV0ID0gLUVCQURGOw0KPiA+ID4gPiArICAgICAgIGN0eC0+c3Fv X2ZpbGVzID0gY3VycmVudC0+ZmlsZXM7DQo+ID4gPiA+ICsgICAgICAgaWYgKCFjdHgtPnNxb19m aWxlcykNCj4gPiA+ID4gKyAgICAgICAgICAgICAgIGdvdG8gZXJyOw0KPiA+ID4gDQo+ID4gPiBU aGF0J3MgZ25hcmx5LiBBZGRpbmcgQWwgVmlybyB0byB0aGUgdGhyZWFkLg0KPiA+ID4gDQo+ID4g PiBJIHRoaW5rIHlvdSBtaXN1bmRlcnN0YW5kIHRoZSBzZW1hbnRpY3Mgb2YgZl9vcC0+cmVsZWFz ZS4gVGhlIC0+Zmx1c2gNCj4gPiA+IGhhbmRsZXIgaXMgaW52b2tlZCB3aGVuZXZlciBhIGZpbGUg ZGVzY3JpcHRvciBpcyBjbG9zZWQgdGhyb3VnaA0KPiA+ID4gZmlscF9jbG9zZSgpICh2aWEgZGVs ZXRpb24gb2YgdGhlIGZpbGVzX3N0cnVjdCwgc3lzX2Nsb3NlKCksDQo+ID4gPiBzeXNfZHVwMigp LCAuLi4pLCBzbyBpZiB5b3UgaGFkIHVzZWQgdGhhdCBvbmUsIF9tYXliZV8gdGhpcyB3b3VsZA0K PiA+ID4gd29yay4gQnV0IHRoZSAtPnJlbGVhc2UgaGFuZGxlciBvbmx5IHJ1bnMgd2hlbiB0aGUg X2xhc3RfIHJlZmVyZW5jZSB0bw0KPiA+ID4gYSBzdHJ1Y3QgZmlsZSBoYXMgYmVlbiBkcm9wcGVk IC0gc28geW91IGNhbiwgZm9yIGV4YW1wbGUsIGZvcmsoKSBhDQo+ID4gPiBjaGlsZCwgdGhlbiBl eGl0KCkgaW4gdGhlIHBhcmVudCwgYW5kIHRoZSAtPnJlbGVhc2UgaGFuZGxlciBpc24ndA0KPiA+ ID4gaW52b2tlZC4gU28gSSBkb24ndCBzZWUgaG93IHRoaXMgY2FuIHdvcmsuDQo+ID4gDQo+ID4g VGhlIGFub25mZCBpcyBDTE9FWEVDLiBUaGUgaWRlYSBpcyBleGFjdGx5IHRoYXQgaXQgb25seSBy dW5zIHdoZW4gdGhlDQo+ID4gbGFzdCByZWZlcmVuY2UgdG8gdGhlIGZpbGUgaGFzIGJlZW4gZHJv cHBlZC4gTm90IHN1cmUgd2h5IHlvdSB0aGluayBJDQo+ID4gbmVlZCAtPmZsdXNoKCkgaGVyZT8N Cj4gDQo+IENhbid0IEkganVzdCB1c2UgZmNudGwoZmQsIEZfU0VURkQsIGZkLCAwKSB0byBjbGVh ciB0aGUgQ0xPRVhFQyBmbGFnPw0KPiBPciBzZW5kIHRoZSBmZCB2aWEgU0NNX1JJR0hUUz8NCj4g DQo+ID4gPiBCdXQgZXZlbiBpZiB5b3UgaGFkIGFidXNlZCAtPmZsdXNoIGZvciB0aGlzIGluc3Rl YWQ6IGNsb3NlX2ZpbGVzKCkNCj4gPiA+IGN1cnJlbnRseSBoYXMgYSBjb21tZW50IGluIGl0IHRo YXQgY2xhaW1zIHRoYXQgInRoaXMgaXMgdGhlIGxhc3QNCj4gPiA+IHJlZmVyZW5jZSB0byB0aGUg ZmlsZXMgc3RydWN0dXJlIjsgdGhpcyBjaGFuZ2Ugd291bGQgbWFrZSB0aGF0IGNsYWltDQo+ID4g PiB1bnRydWUuDQo+ID4gDQo+ID4gTGV0IG1lIHNlZSBpZiBJIGNhbiBleHBsYWluIG15IGludGVu dCBiZXR0ZXIgdGhhbiB0aGF0IGNvbW1lbnQuLi4gV2UNCj4gPiBrbm93IHRoZSBwYXJlbnQgd2hv IHNldCB1cCB0aGUgaW9fdXJpbmcgaW5zdGFuY2Ugd2lsbCBiZSBhcm91bmQgZm9yIGFzDQo+ID4g bG9uZyBhcyBpb191cmluZyBpbnN0YW5jZSBwZXJzaXN0cy4NCj4gDQo+IFRoYXQncyB0aGUgcGFy dCB0aGF0IEkgdGhpbmsgaXMgd3Jvbmc6IEFzIGZhciBhcyBJIGNhbiB0ZWxsLCB0aGUNCj4gcGFy ZW50IGNhbiBnbyBhd2F5IGFuZCB5b3Ugd29uJ3Qgbm90aWNlLg0KPiANCj4gQWxzbywgbm90ZSB0 aGF0ICJ0aGUgcGFyZW50IiBpcyBkaWZmZXJlbnQgdGhpbmdzIGZvciAtPmZpbGVzIGFuZCAtPm1t Lg0KPiBZb3UgY2FuIGhhdmUgYSBtdWx0aXRocmVhZGVkIHByb2Nlc3Mgd2hvc2UgdGhyZWFkcyBk b24ndCBoYXZlIHRoZSBzYW1lDQo+IC0+ZmlsZXMsIG9yIG11bHRpcGxlIHByb2Nlc3MgdGhhdCBz aGFyZSAtPmZpbGVzIHdpdGhvdXQgc2hhcmluZyAtPm1tLA0KPiAuLi4NCg0KVGhpcyBoYWQgYWN0 dWFsbHkgYmVlbiBnZXRfZmlsZXNfc3RydWN0KCkgaW4gZWFybHkgdmVyc2lvbnMsIGFuZCBJIGhh ZA0KcmVwb3J0ZWQgdG8gSmVucyB0aGF0IGl0IGFsbG93cyBzb21ldGhpbmcgbGlrZQ0KDQppbnQg bWFpbigpIHsNCiAgc3RydWN0IGlvX3VyaW5nX3BhcmFtcyB1cmluZ19wYXJhbXMgPSB7DQogIAku ZmxhZ3MgPSBJT1JJTkdfU0VUVVBfU1FQT0xMLA0KICB9Ow0KICBpbnQgdXJpbmdfZmQgPSBzeXNj YWxsKDQyNSAvKiBpb191cmluZ19zZXR1cCAqLywgMTYsICZ1cmluZ19wYXJhbXMpOw0KfQ0KDQp0 byBsZWFrIGJvdGggdGhlIGZpbGVzX3N0cnVjdCBhbmQgdGhlIGt0aHJlYWQsIGFzIHRoZSBmaWxl c19zdHJ1Y3QgYW5kDQp0aGUgdXJpbmcgY29udGV4dCBmb3JtIGEgY2lyY3VsYXIgcmVmZXJlbmNl LiAgSSBoYXZlbid0IHJlYWxseSBjb21lIHVwDQp3aXRoIGEgZ29vZCB3YXkgdG8gcmVjb25jaWxl IHRoZSByZXF1aXJlbWVudHMgaGVyZTsgcGVyaGFwcyB3ZSBuZWVkIGFuDQpleGl0X3VyaW5nKCkg YWtpbiB0byBleGl0X2FpbygpPw0KDQo+ID4gV2hlbiB3ZSBhcmUgdGVhcmluZyBkb3duIHRoZQ0K PiA+IGlvX3VyaW5nLCB0aGVuIHdlIHdhaXQgZm9yIGFueSBhc3luYyBjb250ZXh0cyAobGlrZSB0 aGUgb25lIGFib3ZlKSB0bw0KPiA+IGV4aXQuIE9uY2UgdGhleSBhcmUgZXhpdGVkLCBpdCdzIHNh ZmUgdG8gcHJvY2VlZCB3aXRoIHRoZSBleGl0IGFuZA0KPiA+IHRlYXJkb3duIC0+ZmlsZXNbXS4N Cj4gDQo+IEJ1dCB5b3Ugb25seSBkbyB0aGF0IHRlYXJkb3duIG9uIC0+cmVsZWFzZSwgcmlnaHQ/ IEFuZCAtPnJlbGVhc2UNCj4gZG9lc24ndCBoYXZlIG11Y2ggdG8gZG8gd2l0aCB0aGUgcHJvY2Vz cyBsaWZldGltZS4NCj4gDQo+ID4gVGhhdCdzIHRoZSBpZGVhLi4uIE5vdCB0cnlpbmcgdG8gYmUg Y2xldmVyLCBzb21lIG9mIHRoaXMgZGF0ZXMgYmFjayB0bw0KPiA+IHRoZSBhaW8gd2VpcmRuZXNz IHdoZXJlIGl0IHdhcyBpbXBvc3NpYmxlIHRvIGhhdmUgY3Jvc3MgcmVmZXJlbmNlcyBsaWtlDQo+ ID4gdGhpcywgc2luY2UgaXQgd291bGQgbGVhZCB0byB0ZWFyZG93biBkZWFkbG9ja3Mgd2l0aCBo b3cgZXhpdF9haW8oKSBpcw0KPiA+IHVzZWQuIEkgY2FuIHByb2JhYmx5IGdyYWIgYSBzdHJ1Y3Qg ZmlsZXMgcmVmZXJlbmNlIGFib3ZlLCBidXQgY3VycmVudGx5DQo+ID4gSSBkb24ndCBzZWUgd2h5 IGl0J3MgbmVlZGVkLg0K

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#18 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 287 lines · Matt Mullins
Quoted 268 lines · Jann Horn
Quoted 185 lines · Jens Axboe
Quoted 175 lines · Jann Horn
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: https://urldefense.proofpoint.com/v2/url?u=http-3A__git.kernel.dk_cgit_fio_plain_t_io-5Furing.c&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=pqM-eO4A2hNFhIFiX-7eGg&m=MGr14pOzNbC7Z-8_dV4GMiH3AbkkH0RSQoQ894Tu0yc&s=mgbcubzOMiCpFpnwW-HA3ey0YDYPkgMIZ7Bmy4w6Chc&e=

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
fs/io_uring.c new file +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 39 lines
fs/io_uring.c new file +39 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
Quoted 8 lines

fs/io_uring.c line 512

Quoted 5 lines
fs/io_uring.c new file +5 -0
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

Quoted 3 lines

'current' is the work queue item in this case, do we need to protect against anything else? I can add the locking around the assignments (both places).

Stuff like proc_fd_link() uses get_files_struct(), which grabs a reference to your current files_struct protected only by task_lock(); and it doesn't use anything like READ_ONCE(), so even if the object lifetime is not a problem, get_files_struct() could potentially crash due to a double-read (reading task->files twice and assuming that the result will be the same). As far as I can tell, this procfs code also works on kernel threads.

Quoted 54 lines
Quoted 22 lines
Quoted 20 lines
fs/io_uring.c new file +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

Quoted 7 lines

fs/io_uring.c line 722

Quoted 5 lines
fs/io_uring.c new file +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

I think we're deadling with the same thing as the files below, I'll defer to that.

Quoted 12 lines

fs/io_uring.c line 732

Quoted 9 lines
fs/io_uring.c new file +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

Quoted 8 lines

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

Quoted 3 lines

The anonfd is CLOEXEC. The idea is exactly that it only runs when the last reference to the file has been dropped. Not sure why you think I need ->flush() here?

Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag? Or send the fd via SCM_RIGHTS?

Quoted 8 lines
Quoted 4 lines

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Let me see if I can explain my intent better than that comment... We know the parent who set up the io_uring instance will be around for as long as io_uring instance persists.

Quoted 7 lines

That's the part that I think is wrong: As far as I can tell, the parent can go away and you won't notice.

Also, note that "the parent" is different things for ->files and ->mm. You can have a multithreaded process whose threads don't have the same ->files, or multiple process that share ->files without sharing ->mm,
...

This had actually been get_files_struct() in early versions, and I had reported to Jens that it allows something like

int main() {
struct io_uring_params uring_params = {
.flags = IORING_SETUP_SQPOLL,
};
int uring_fd = syscall(425 /* io_uring_setup */, 16, &uring_params);
}

Quoted 4 lines

to leak both the files_struct and the kthread, as the files_struct and the uring context form a circular reference. I haven't really come up with a good way to reconcile the requirements here; perhaps we need an exit_uring() akin to exit_aio()?

Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted, like "struct mm_struct"? One reference type to keep the contents intact (the reference type you normally use, and the type used by uring when the thread is running), and one reference type to just keep the struct itself existing, but without preserving its contents (reference held consistently by the uring thread)?

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#19 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 294 lines · Jann Horn
Quoted 287 lines · Matt Mullins
Quoted 268 lines · Jann Horn
Quoted 185 lines · Jens Axboe
Quoted 175 lines · Jann Horn
Quoted 48 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: https://urldefense.proofpoint.com/v2/url?u=http-3A__git.kernel.dk_cgit_fio_plain_t_io-5Furing.c&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=pqM-eO4A2hNFhIFiX-7eGg&m=MGr14pOzNbC7Z-8_dV4GMiH3AbkkH0RSQoQ894Tu0yc&s=mgbcubzOMiCpFpnwW-HA3ey0YDYPkgMIZ7Bmy4w6Chc&e=

[...]

Quoted 39 lines
(quoted diff) +39 -0
+static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                     bool force_nonblock)
+{
+       struct kiocb *kiocb = &req->rw;
+       int ret;
+
+       kiocb->ki_filp = fget(sqe->fd);
+       if (unlikely(!kiocb->ki_filp))
+               return -EBADF;
+       kiocb->ki_pos = sqe->off;
+       kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
+       kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
+       if (sqe->ioprio) {
+               ret = ioprio_check_cap(sqe->ioprio);
+               if (ret)
+                       goto out_fput;
+
+               kiocb->ki_ioprio = sqe->ioprio;
+       } else
+               kiocb->ki_ioprio = get_current_ioprio();
+
+       ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+       if (unlikely(ret))
+               goto out_fput;
+       if (force_nonblock) {
+               kiocb->ki_flags |= IOCB_NOWAIT;
+               req->flags |= REQ_F_FORCE_NONBLOCK;
+       }
+       if (kiocb->ki_flags & IOCB_HIPRI) {
+               ret = -EINVAL;
+               goto out_fput;
+       }
+
+       kiocb->ki_complete = io_complete_rw;
+       return 0;
+out_fput:
+       fput(kiocb->ki_filp);
+       return ret;
+}

[...]

Quoted 42 lines
(quoted diff) +42 -0
+static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+                      bool force_nonblock)
+{
+       struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+       struct kiocb *kiocb = &req->rw;
+       struct iov_iter iter;
+       struct file *file;
+       ssize_t ret;
+
+       ret = io_prep_rw(req, sqe, force_nonblock);
+       if (ret)
+               return ret;
+       file = kiocb->ki_filp;
+
+       ret = -EBADF;
+       if (unlikely(!(file->f_mode & FMODE_READ)))
+               goto out_fput;
+       ret = -EINVAL;
+       if (unlikely(!file->f_op->read_iter))
+               goto out_fput;
+
+       ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
+       if (ret)
+               goto out_fput;
+
+       ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+       if (!ret) {
+               ssize_t ret2;
+
+               /* Catch -EAGAIN return for forced non-blocking submission */
+               ret2 = call_read_iter(file, kiocb, &iter);
+               if (!force_nonblock || ret2 != -EAGAIN)
+                       io_rw_done(kiocb, ret2);
+               else
+                       ret = -EAGAIN;
+       }
+       kfree(iovec);
+out_fput:
+       if (unlikely(ret))
+               fput(file);
+       return ret;
+}

[...]

Quoted 39 lines
(quoted diff) +39 -0
+static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
+                          struct sqe_submit *s, bool force_nonblock)
+{
+       const struct io_uring_sqe *sqe = s->sqe;
+       ssize_t ret;
+
+       if (unlikely(s->index >= ctx->sq_entries))
+               return -EINVAL;
+       req->user_data = sqe->user_data;
+
+       ret = -EINVAL;
+       switch (sqe->opcode) {
+       case IORING_OP_NOP:
+               ret = io_nop(req, sqe);
+               break;
+       case IORING_OP_READV:
+               ret = io_read(req, sqe, force_nonblock);
+               break;
+       case IORING_OP_WRITEV:
+               ret = io_write(req, sqe, force_nonblock);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       return ret;
+}
+
+static void io_sq_wq_submit_work(struct work_struct *work)
+{
+       struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+       struct sqe_submit *s = &req->submit;
+       u64 user_data = s->sqe->user_data;
+       struct io_ring_ctx *ctx = req->ctx;
+       mm_segment_t old_fs = get_fs();
+       struct files_struct *old_files;
+       int ret;
+
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+        /* Ensure we clear previously set forced non-block flag */
+       req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+       old_files = current->files;
+       current->files = ctx->sqo_files;

I think you're not supposed to twiddle with current->files without holding task_lock(current).

Quoted 3 lines

'current' is the work queue item in this case, do we need to protect against anything else? I can add the locking around the assignments (both places).

Stuff like proc_fd_link() uses get_files_struct(), which grabs a reference to your current files_struct protected only by task_lock(); and it doesn't use anything like READ_ONCE(), so even if the object lifetime is not a problem, get_files_struct() could potentially crash due to a double-read (reading task->files twice and assuming that the result will be the same). As far as I can tell, this procfs code also works on kernel threads.

Quoted 54 lines
Quoted 22 lines
Quoted 20 lines
(quoted diff) +20 -0
+       if (!mmget_not_zero(ctx->sqo_mm)) {
+               ret = -EFAULT;
+               goto err;
+       }
+
+       use_mm(ctx->sqo_mm);
+       set_fs(USER_DS);
+
+       ret = __io_submit_sqe(ctx, req, s, false);
+
+       set_fs(old_fs);
+       unuse_mm(ctx->sqo_mm);
+       mmput(ctx->sqo_mm);
+err:
+       if (ret) {
+               io_cqring_add_event(ctx, user_data, ret, 0);
+               io_free_req(req);
+       }
+       current->files = old_files;
+}

[...]

Quoted 7 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+static int io_sq_offload_start(struct io_ring_ctx *ctx)
+{
+       int ret;
+
+       ctx->sqo_mm = current->mm;

What keeps this thing alive?

I think we're deadling with the same thing as the files below, I'll defer to that.

Quoted 12 lines

Review

Quoted 9 lines
(quoted diff) +9 -0
+       /*
+        * This is safe since 'current' has the fd installed, and if that gets
+        * closed on exit, then fops->release() is invoked which waits for the
+        * async contexts to flush and exit before exiting.
+        */
+       ret = -EBADF;
+       ctx->sqo_files = current->files;
+       if (!ctx->sqo_files)
+               goto err;

That's gnarly. Adding Al Viro to the thread.

Quoted 8 lines

I think you misunderstand the semantics of f_op->release. The ->flush handler is invoked whenever a file descriptor is closed through filp_close() (via deletion of the files_struct, sys_close(), sys_dup2(), ...), so if you had used that one, _maybe_ this would work. But the ->release handler only runs when the _last_ reference to a struct file has been dropped - so you can, for example, fork() a child, then exit() in the parent, and the ->release handler isn't invoked. So I don't see how this can work.

Quoted 3 lines

The anonfd is CLOEXEC. The idea is exactly that it only runs when the last reference to the file has been dropped. Not sure why you think I need ->flush() here?

Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag? Or send the fd via SCM_RIGHTS?

Quoted 8 lines
Quoted 4 lines

But even if you had abused ->flush for this instead: close_files() currently has a comment in it that claims that "this is the last reference to the files structure"; this change would make that claim untrue.

Let me see if I can explain my intent better than that comment... We know the parent who set up the io_uring instance will be around for as long as io_uring instance persists.

Quoted 7 lines

That's the part that I think is wrong: As far as I can tell, the parent can go away and you won't notice.

Also, note that "the parent" is different things for ->files and ->mm. You can have a multithreaded process whose threads don't have the same ->files, or multiple process that share ->files without sharing ->mm,
...

This had actually been get_files_struct() in early versions, and I had reported to Jens that it allows something like

int main() {
struct io_uring_params uring_params = {
.flags = IORING_SETUP_SQPOLL,
};
int uring_fd = syscall(425 /* io_uring_setup */, 16, &uring_params);
}

Quoted 4 lines

to leak both the files_struct and the kthread, as the files_struct and the uring context form a circular reference. I haven't really come up with a good way to reconcile the requirements here; perhaps we need an exit_uring() akin to exit_aio()?

Quoted 6 lines

Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted, like "struct mm_struct"? One reference type to keep the contents intact (the reference type you normally use, and the type used by uring when the thread is running), and one reference type to just keep the struct itself existing, but without preserving its contents (reference held consistently by the uring thread)?

Something like this (completely untested); and then instead of the current get_files_struct(), you'd do get_files_struct_weak(), and while the thread is running, it protects the files_struct from dying with tryget_weak_files_struct() / put_files_struct().

Al, do you have opinions on this?

===============

fs/file.c +0 -0
index 3209ee271c41..fbf02ef2753d 100644
@@ -281,6 +281,7 @@ struct files_struct *dup_fd(struct files_struct

*oldf, int *errorp)

        if (!newf)
                goto out;
+       kref_init(&newf->weak_refs);
        atomic_set(&newf->count, 1);

        spin_lock_init(&newf->file_lock);
@@ -410,6 +411,26 @@ struct files_struct *get_files_struct(struct
task_struct *task)
        return files;

}

+static void free_files_struct(struct kref *ref) {
+ struct files_struct *files =
+ container_of(ref, struct files_struct, weak_refs);
+ kmem_cache_free(files_cachep, files);
+}
+
+void put_files_struct_weak(struct files_struct *files) {
+ kref_put(&files->weak_refs, free_files_struct);
+}
+
+struct files_struct *get_files_struct_weak(struct task_struct *task) +{
+ struct files_struct *files = get_files_struct(task);
+ if (files) {
+ kref_get(&files->weak_refs);
+ put_files_struct(files);
+ }
+ return files;
+}
+
void put_files_struct(struct files_struct *files)

 {
        if (atomic_dec_and_test(&files->count)) {
@@ -418,10 +439,17 @@ void put_files_struct(struct files_struct *files)
                /* free the arrays if they are not embedded */
                if (fdt != &files->fdtab)
                        __free_fdtable(fdt);
-               kmem_cache_free(files_cachep, files);
+               put_files_struct_weak(files);
        }

}

+struct files_struct *tryget_weak_files_struct(struct files_struct *fs) {
+ if (atomic_inc_not_zero(&fs->count)) {
+ return fs;
+ }
+ return NULL;
+}
+
void reset_files_struct(struct files_struct *files)

 {
        struct task_struct *tsk = current;

@@ -448,6 +476,7 @@ void exit_files(struct task_struct *tsk)

struct files_struct init_files = {

        .count          = ATOMIC_INIT(1),
+       .weak_refs      = KREF_INIT(1),
        .fdt            = &init_files.fdtab,
        .fdtab          = {

.max_fds = NR_OPEN_DEFAULT,

include/linux/fdtable.h +5 -0
index f07c55ea0c22..6ad95a95cc0b 100644
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/init.h>
 #include <linux/fs.h>
+#include <linux/kref.h>
 
 #include <linux/atomic.h>
 
@@ -50,6 +51,7 @@ struct files_struct {
    * read mostly part
    */
        atomic_t count;
+       struct kref weak_refs;
        bool resize_in_progress;
        wait_queue_head_t resize_wait;
 
@@ -107,6 +109,9 @@ struct task_struct;
 
 struct files_struct *get_files_struct(struct task_struct *);
 void put_files_struct(struct files_struct *fs);
+void put_files_struct_weak(struct files_struct *files);
+struct files_struct *get_files_struct_weak(struct task_struct *);
+struct files_struct *tryget_weak_files_struct(struct files_struct *);
 void reset_files_struct(struct files_struct *);
 int unshare_files(struct files_struct **);
 struct files_struct *dup_fd(struct files_struct *, int *) __latent_entropy;

===============

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

maintainer
#20 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 7 lines · Jann Horn
Quoted 6 lines

Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted, like "struct mm_struct"? One reference type to keep the contents intact (the reference type you normally use, and the type used by uring when the thread is running), and one reference type to just keep the struct itself existing, but without preserving its contents (reference held consistently by the uring thread)?

Quoted 6 lines

Something like this (completely untested); and then instead of the current get_files_struct(), you'd do get_files_struct_weak(), and while the thread is running, it protects the files_struct from dying with tryget_weak_files_struct() / put_files_struct().

Al, do you have opinions on this?

Yes, but they are not fit for polite company. IMO the entire approach is FUBAR; I'll post more detailed review, but what I'd seen so far is a veto fodder.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#21 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 12 lines
fs/io_uring.c new file +12 -0
+static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
+{
+       struct io_sq_ring *ring = ctx->sq_ring;
+       unsigned head;
+
+       /*
+        * The cached sq head (or cq tail) serves two purposes:
+        *
+        * 1) allows us to batch the cost of updating the user visible
+        *    head updates.
+        * 2) allows the kernel side to track the head on its own, even
+        *    though the application is the one updating it.

fs/io_uring.c line 601

Quoted 10 lines
fs/io_uring.c new file +10 -0
+        */
+       head = ctx->cached_sq_head;
+       smp_rmb();
+       if (head == READ_ONCE(ring->r.tail))
+               return false;
+
+       head = ring->array[head & ctx->sq_mask];
+       if (head < ctx->sq_entries) {
+               s->index = head;
+               s->sqe = &ctx->sq_sqes[head];

ring->array can be mapped writable into userspace, right? If so: This looks like a double-read issue; the compiler might assume that ring->array is not modified concurrently and perform separate memory accesses for the "if (head < ctx->sq_entries)" check and the "&ctx->sq_sqes[head]" computation. Please use READ_ONCE()/WRITE_ONCE() for all accesses to memory that userspace could concurrently modify in a malicious way.

There have been some pretty severe security bugs caused by missing
READ_ONCE() annotations around accesses to shared memory; see, for
example, https://www.blackhat.com/docs/us-16/materials/us-16-Wilhelm-Xenpwn-Breaking-Paravirtualized-Devices.pdf
. Slides 35-48 show how the code "switch (op->cmd)", where "op" is a
pointer to shared memory, allowed an attacker to break out of a Xen
virtual machine because the compiler generated multiple memory
accesses.

Quoted 10 lines
fs/io_uring.c new file +10 -0
+               ctx->cached_sq_head++;
+               return true;
+       }
+
+       /* drop invalid entries */
+       ctx->cached_sq_head++;
+       ring->dropped++;
+       smp_wmb();
+       return false;
+}

[...]

Quoted 5 lines
fs/io_uring.c new file +5 -0
+SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
+               u32, min_complete, u32, flags, const sigset_t __user *, sig,
+               size_t, sigsz)
+{
+       struct io_ring_ctx *ctx;

fs/io_uring.c line 907

Quoted 10 lines
fs/io_uring.c new file +10 -0
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
+               goto out_fput;

Oh, by the way: If you feel like it, maybe you could add a helper fdget_typed(int fd, struct file_operations *f_op), or something like that, so that there is less boilerplate code for first doing fdget(), then checking ->f_op, and then coding an extra bailout path for that? But that doesn't really have much to do with your patchset, feel free to ignore this comment.

[...]

fs/io_uring.c line 925

Quoted 4 lines
fs/io_uring.c new file +4 -0
+out_fput:
+       fdput(f);
+       return ret;
+}

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#22 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 36 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 12 lines
fs/io_uring.c new file +12 -0
+static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
+{
+       struct io_sq_ring *ring = ctx->sq_ring;
+       unsigned head;
+
+       /*
+        * The cached sq head (or cq tail) serves two purposes:
+        *
+        * 1) allows us to batch the cost of updating the user visible
+        *    head updates.
+        * 2) allows the kernel side to track the head on its own, even
+        *    though the application is the one updating it.

fs/io_uring.c line 601

Quoted 10 lines
fs/io_uring.c new file +10 -0
+        */
+       head = ctx->cached_sq_head;
+       smp_rmb();
+       if (head == READ_ONCE(ring->r.tail))
+               return false;
+
+       head = ring->array[head & ctx->sq_mask];
+       if (head < ctx->sq_entries) {
+               s->index = head;
+               s->sqe = &ctx->sq_sqes[head];

ring->array can be mapped writable into userspace, right? If so: This looks like a double-read issue; the compiler might assume that ring->array is not modified concurrently and perform separate memory accesses for the "if (head < ctx->sq_entries)" check and the "&ctx->sq_sqes[head]" computation. Please use READ_ONCE()/WRITE_ONCE() for all accesses to memory that userspace could concurrently modify in a malicious way.

Quoted 7 lines

There have been some pretty severe security bugs caused by missing
READ_ONCE() annotations around accesses to shared memory; see, for
example, https://www.blackhat.com/docs/us-16/materials/us-16-Wilhelm-Xenpwn-Breaking-Paravirtualized-Devices.pdf
. Slides 35-48 show how the code "switch (op->cmd)", where "op" is a
pointer to shared memory, allowed an attacker to break out of a Xen
virtual machine because the compiler generated multiple memory
accesses.

Oh, actually, it's even worse (comments with "//" added by me):

io_sq_thread() does this:

        do {
                // sqes[i].sqe is pointer to shared memory, result of
                // io_sqe_needs_user() is unreliable
                if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
                        all_fixed = false;

                i++;
                if (i == ARRAY_SIZE(sqes))
                        break;
        } while (io_get_sqring(ctx, &sqes[i]));
        // sqes[...].sqe are pointers to shared memory

        io_commit_sqring(ctx);

        /* Unless all new commands are FIXED regions, grab mm */
        if (!all_fixed && !cur_mm) {
                mm_fault = !mmget_not_zero(ctx->sqo_mm);
                if (!mm_fault) {
                        use_mm(ctx->sqo_mm);
                        cur_mm = ctx->sqo_mm;
                }
        }

        inflight += io_submit_sqes(ctx, sqes, i, mm_fault);

Then the shared memory pointers go into io_submit_sqes():

static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
unsigned int nr, bool mm_fault)

{
        struct io_submit_state state, *statep = NULL;
        int ret, i, submitted = 0;
        // sqes[...].sqe are pointers to shared memory
        [...]
        for (i = 0; i < nr; i++) {
                if (unlikely(mm_fault))
                        ret = -EFAULT;
                else
                        ret = io_submit_sqe(ctx, &sqes[i], statep);
                [...]
        }
        [...]
}

And on into io_submit_sqe():

static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
struct io_submit_state *state)

{
        [...]
        ret = __io_submit_sqe(ctx, req, s, true, state);
        [...]
}

And there it gets interesting:

static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
struct sqe_submit *s, bool force_nonblock, struct io_submit_state *state)

{
        // s->sqe is a pointer to shared memory
        const struct io_uring_sqe *sqe = s->sqe;
        // sqe is a pointer to shared memory
        ssize_t ret;

        if (unlikely(s->index >= ctx->sq_entries))
                return -EINVAL;
        req->user_data = sqe->user_data;

        ret = -EINVAL;
        // switch() on read from shared memory, potential instruction pointer
        // control
        switch (sqe->opcode) {
        [...]
        case IORING_OP_READV:
                if (unlikely(sqe->buf_index))
                        return -EINVAL;
                ret = io_read(req, sqe, force_nonblock, state);
                break;
        [...]
        case IORING_OP_READ_FIXED:
                ret = io_read(req, sqe, force_nonblock, state);
                break;
        [...]
        }
        [...]
}

On into io_read():

static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
bool force_nonblock, struct io_submit_state *state)
{
[...]

        // sqe is a pointer to shared memory
        ret = io_prep_rw(req, sqe, force_nonblock, state);
        [...]
}

And then io_prep_rw() does multiple reads even in the source code:

static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
bool force_nonblock, struct io_submit_state *state)

{
        struct io_ring_ctx *ctx = req->ctx;
        struct kiocb *kiocb = &req->rw;
        int ret;

        // sqe is a pointer to shared memory

        // double-read of sqe->flags, see end of function
        if (sqe->flags & IOSQE_FIXED_FILE) {
                // double-read of sqe->fd for the bounds check and the
array access, potential OOB pointer read
                if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
                        return -EBADF;
                kiocb->ki_filp = ctx->user_files[sqe->fd];
                req->flags |= REQ_F_FIXED_FILE;
        } else {
                kiocb->ki_filp = io_file_get(state, sqe->fd);
        }
        if (unlikely(!kiocb->ki_filp))
                return -EBADF;
        kiocb->ki_pos = sqe->off;
        kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
        kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
        // three reads of sqe->ioprio, bypassable capability check
        if (sqe->ioprio) {
                ret = ioprio_check_cap(sqe->ioprio);
                if (ret)
                        goto out_fput;

                kiocb->ki_ioprio = sqe->ioprio;
        } else
                kiocb->ki_ioprio = get_current_ioprio();
        [...]
        return 0;
out_fput:
        // double-read of sqe->flags, changed value can lead to
unbalanced refcount
        if (!(sqe->flags & IOSQE_FIXED_FILE))
                io_file_put(state, kiocb->ki_filp);
        return ret;
}

Please create a local copy of the request before parsing it to keep the data from changing under you. Additionally, it might make sense to annotate every pointer to shared memory with a comment, or something like that, to ensure that anyone looking at the code can immediately see for which pointers special caution is required on access.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#23 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 194 lines · Jann Horn
Quoted 36 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 12 lines
fs/io_uring.c new file +12 -0
+static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
+{
+       struct io_sq_ring *ring = ctx->sq_ring;
+       unsigned head;
+
+       /*
+        * The cached sq head (or cq tail) serves two purposes:
+        *
+        * 1) allows us to batch the cost of updating the user visible
+        *    head updates.
+        * 2) allows the kernel side to track the head on its own, even
+        *    though the application is the one updating it.

fs/io_uring.c line 601

Quoted 10 lines
fs/io_uring.c new file +10 -0
+        */
+       head = ctx->cached_sq_head;
+       smp_rmb();
+       if (head == READ_ONCE(ring->r.tail))
+               return false;
+
+       head = ring->array[head & ctx->sq_mask];
+       if (head < ctx->sq_entries) {
+               s->index = head;
+               s->sqe = &ctx->sq_sqes[head];

ring->array can be mapped writable into userspace, right? If so: This looks like a double-read issue; the compiler might assume that ring->array is not modified concurrently and perform separate memory accesses for the "if (head < ctx->sq_entries)" check and the "&ctx->sq_sqes[head]" computation. Please use READ_ONCE()/WRITE_ONCE() for all accesses to memory that userspace could concurrently modify in a malicious way.

Quoted 7 lines

There have been some pretty severe security bugs caused by missing
READ_ONCE() annotations around accesses to shared memory; see, for
example, https://www.blackhat.com/docs/us-16/materials/us-16-Wilhelm-Xenpwn-Breaking-Paravirtualized-Devices.pdf
. Slides 35-48 show how the code "switch (op->cmd)", where "op" is a
pointer to shared memory, allowed an attacker to break out of a Xen
virtual machine because the compiler generated multiple memory
accesses.

Oh, actually, it's even worse (comments with "//" added by me):

io_sq_thread() does this:

        do {
                // sqes[i].sqe is pointer to shared memory, result of
                // io_sqe_needs_user() is unreliable
                if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
                        all_fixed = false;

                i++;
                if (i == ARRAY_SIZE(sqes))
                        break;
        } while (io_get_sqring(ctx, &sqes[i]));
        // sqes[...].sqe are pointers to shared memory

        io_commit_sqring(ctx);

        /* Unless all new commands are FIXED regions, grab mm */
        if (!all_fixed && !cur_mm) {
                mm_fault = !mmget_not_zero(ctx->sqo_mm);
                if (!mm_fault) {
                        use_mm(ctx->sqo_mm);
                        cur_mm = ctx->sqo_mm;
                }
        }

        inflight += io_submit_sqes(ctx, sqes, i, mm_fault);

Then the shared memory pointers go into io_submit_sqes():

static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
unsigned int nr, bool mm_fault)

{
        struct io_submit_state state, *statep = NULL;
        int ret, i, submitted = 0;
        // sqes[...].sqe are pointers to shared memory
        [...]
        for (i = 0; i < nr; i++) {
                if (unlikely(mm_fault))
                        ret = -EFAULT;
                else
                        ret = io_submit_sqe(ctx, &sqes[i], statep);
                [...]
        }
        [...]
}

And on into io_submit_sqe():

static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
struct io_submit_state *state)

{
        [...]
        ret = __io_submit_sqe(ctx, req, s, true, state);
        [...]
}

And there it gets interesting:

static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
struct sqe_submit *s, bool force_nonblock, struct io_submit_state *state)

{
        // s->sqe is a pointer to shared memory
        const struct io_uring_sqe *sqe = s->sqe;
        // sqe is a pointer to shared memory
        ssize_t ret;

        if (unlikely(s->index >= ctx->sq_entries))
                return -EINVAL;
        req->user_data = sqe->user_data;

        ret = -EINVAL;
        // switch() on read from shared memory, potential instruction pointer
        // control
        switch (sqe->opcode) {
        [...]
        case IORING_OP_READV:
                if (unlikely(sqe->buf_index))
                        return -EINVAL;
                ret = io_read(req, sqe, force_nonblock, state);
                break;
        [...]
        case IORING_OP_READ_FIXED:
                ret = io_read(req, sqe, force_nonblock, state);
                break;
        [...]
        }
        [...]
}

On into io_read():

static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
bool force_nonblock, struct io_submit_state *state)
{
[...]

        // sqe is a pointer to shared memory
        ret = io_prep_rw(req, sqe, force_nonblock, state);
        [...]
}

And then io_prep_rw() does multiple reads even in the source code:

static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
bool force_nonblock, struct io_submit_state *state)

{
        struct io_ring_ctx *ctx = req->ctx;
        struct kiocb *kiocb = &req->rw;
        int ret;

        // sqe is a pointer to shared memory

        // double-read of sqe->flags, see end of function
        if (sqe->flags & IOSQE_FIXED_FILE) {
                // double-read of sqe->fd for the bounds check and the
array access, potential OOB pointer read
                if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
                        return -EBADF;
                kiocb->ki_filp = ctx->user_files[sqe->fd];
                req->flags |= REQ_F_FIXED_FILE;
        } else {
                kiocb->ki_filp = io_file_get(state, sqe->fd);
        }
        if (unlikely(!kiocb->ki_filp))
                return -EBADF;
        kiocb->ki_pos = sqe->off;
        kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
        kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
        // three reads of sqe->ioprio, bypassable capability check
        if (sqe->ioprio) {
                ret = ioprio_check_cap(sqe->ioprio);
                if (ret)
                        goto out_fput;

                kiocb->ki_ioprio = sqe->ioprio;
        } else
                kiocb->ki_ioprio = get_current_ioprio();
        [...]
        return 0;
out_fput:
        // double-read of sqe->flags, changed value can lead to
unbalanced refcount
        if (!(sqe->flags & IOSQE_FIXED_FILE))
                io_file_put(state, kiocb->ki_filp);
        return ret;
}
Quoted 5 lines

Please create a local copy of the request before parsing it to keep the data from changing under you. Additionally, it might make sense to annotate every pointer to shared memory with a comment, or something like that, to ensure that anyone looking at the code can immediately see for which pointers special caution is required on access.

Ugh, that's pretty dire. But good catch, I'll fix that up so the application changing sqe malicously won't affect the kernel. I hope we can get away with NOT copying the whole sqe, but we'll do that if we have to.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#24 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jann Horn
Quoted 5 lines · Jann Horn

Please create a local copy of the request before parsing it to keep the data from changing under you. Additionally, it might make sense to annotate every pointer to shared memory with a comment, or something like that, to ensure that anyone looking at the code can immediately see for which pointers special caution is required on access.

I took a look at the viability of NOT having to local copy the data, and I don't think it's too bad. Local copy has a noticeable impact on the performance, hence I'd really (REALLY) like to avoid it.

Here's something on top of the current git branch. I think I even went a bit too far in some areas, but it should hopefully catch the cases where we might end up double evaluating the parts of the sqe that we depend on. For most of the sqe reading we don't really care too much. For instance, the sqe->user_data. If the app changes this field, then it just gets whatever passed back in cqe->user_data. That's not a kernel issue.

For cases like addr/len etc validation, it should be sound. I'll double check this in the morning as well, and obviously would need to be folded in along the way.

I'd appreciate your opinion on this part, if you see any major issues with it, or if I missed something.

fs/io_uring.c +68 -45
index e8760ad02e82..64d090300990 100644
@@ -668,31 +668,35 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	struct kiocb *kiocb = &req->rw;
-	int ret;
+	unsigned flags, ioprio;
+	int fd, ret;
 
-	if (sqe->flags & IOSQE_FIXED_FILE) {
-		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+	flags = READ_ONCE(sqe->flags);
+	fd = READ_ONCE(sqe->fd);
+	if (flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || fd >= ctx->nr_user_files))
 			return -EBADF;
-		kiocb->ki_filp = ctx->user_files[sqe->fd];
+		kiocb->ki_filp = ctx->user_files[fd];
 		req->flags |= REQ_F_FIXED_FILE;
 	} else {
-		kiocb->ki_filp = io_file_get(state, sqe->fd);
+		kiocb->ki_filp = io_file_get(state, fd);
 	}
 	if (unlikely(!kiocb->ki_filp))
 		return -EBADF;
-	kiocb->ki_pos = sqe->off;
+	kiocb->ki_pos = READ_ONCE(sqe->off);
 	kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
 	kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
-	if (sqe->ioprio) {
-		ret = ioprio_check_cap(sqe->ioprio);
+	ioprio = READ_ONCE(sqe->ioprio);
+	if (ioprio) {
+		ret = ioprio_check_cap(ioprio);
 		if (ret)
 			goto out_fput;
 
-		kiocb->ki_ioprio = sqe->ioprio;
+		kiocb->ki_ioprio = ioprio;
 	} else
 		kiocb->ki_ioprio = get_current_ioprio();
 
-	ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
+	ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
 	if (unlikely(ret))
 		goto out_fput;
 	if (force_nonblock) {
@@ -716,7 +720,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	}
 	return 0;
 out_fput:
-	if (!(sqe->flags & IOSQE_FIXED_FILE))
+	if (!(flags & IOSQE_FIXED_FILE))
 		io_file_put(state, kiocb->ki_filp);
 	return ret;
 }
@@ -746,28 +750,31 @@ static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
 			   const struct io_uring_sqe *sqe,
 			   struct iov_iter *iter)
 {
+	size_t len = READ_ONCE(sqe->len);
 	struct io_mapped_ubuf *imu;
-	size_t len = sqe->len;
+	int buf_index, index;
 	size_t offset;
-	int index;
+	u64 buf_addr;
 
 	/* attempt to use fixed buffers without having provided iovecs */
 	if (unlikely(!ctx->user_bufs))
 		return -EFAULT;
-	if (unlikely(sqe->buf_index >= ctx->nr_user_bufs))
+
+	buf_index = READ_ONCE(sqe->buf_index);
+	if (unlikely(buf_index >= ctx->nr_user_bufs))
 		return -EFAULT;
 
-	index = array_index_nospec(sqe->buf_index, ctx->sq_entries);
+	index = array_index_nospec(buf_index, ctx->sq_entries);
 	imu = &ctx->user_bufs[index];
-	if ((unsigned long) sqe->addr < imu->ubuf ||
-	    (unsigned long) sqe->addr + len > imu->ubuf + imu->len)
+	buf_addr = READ_ONCE(sqe->addr);
+	if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
 		return -EFAULT;
 
 	/*
 	 * May not be a start of buffer, set size appropriately
 	 * and advance us to the beginning.
 	 */
-	offset = (unsigned long) sqe->addr - imu->ubuf;
+	offset = buf_addr - imu->ubuf;
 	iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
 	if (offset)
 		iov_iter_advance(iter, offset);
@@ -778,10 +785,12 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
 			   const struct io_uring_sqe *sqe,
 			   struct iovec **iovec, struct iov_iter *iter)
 {
-	void __user *buf = u64_to_user_ptr(sqe->addr);
+	void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
+	int opcode;
 
-	if (sqe->opcode == IORING_OP_READ_FIXED ||
-	    sqe->opcode == IORING_OP_WRITE_FIXED) {
+	opcode = READ_ONCE(sqe->opcode);
+	if (opcode == IORING_OP_READ_FIXED ||
+	    opcode == IORING_OP_WRITE_FIXED) {
 		ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
 		*iovec = NULL;
 		return ret;
@@ -789,11 +798,12 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
 
 #ifdef CONFIG_COMPAT
 	if (in_compat_syscall())
-		return compat_import_iovec(rw, buf, sqe->len, UIO_FASTIOV,
-						iovec, iter);
+		return compat_import_iovec(rw, buf, READ_ONCE(sqe->len),
+						UIO_FASTIOV, iovec, iter);
 #endif
 
-	return import_iovec(rw, buf, sqe->len, UIO_FASTIOV, iovec, iter);
+	return import_iovec(rw, buf, READ_ONCE(sqe->len), UIO_FASTIOV, iovec,
+				iter);
 }
 
 static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
@@ -939,14 +949,14 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 /*
  * IORING_OP_NOP just posts a completion event, nothing else.
  */
-static int io_nop(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+static int io_nop(struct io_kiocb *req, u64 user_data)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 
 	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
 		return -EINVAL;
 
-	io_cqring_add_event(ctx, sqe->user_data, 0, 0);
+	io_cqring_add_event(ctx, user_data, 0, 0);
 	io_free_req(req);
 	return 0;
 }
@@ -955,9 +965,12 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		    bool force_nonblock)
 {
 	struct io_ring_ctx *ctx = req->ctx;
-	loff_t end = sqe->off + sqe->len;
+	loff_t sqe_off = READ_ONCE(sqe->off);
+	loff_t sqe_len = READ_ONCE(sqe->len);
+	loff_t end = sqe_off + sqe_len;
 	struct file *file;
-	int ret;
+	unsigned flags;
+	int ret, fd;
 
 	/* fsync always requires a blocking context */
 	if (force_nonblock)
@@ -970,21 +983,23 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	if (unlikely(sqe->fsync_flags & ~IORING_FSYNC_DATASYNC))
 		return -EINVAL;
 
-	if (sqe->flags & IOSQE_FIXED_FILE) {
-		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+	fd = READ_ONCE(sqe->fd);
+	flags = READ_ONCE(sqe->flags);
+	if (flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || fd >= ctx->nr_user_files))
 			return -EBADF;
-		file = ctx->user_files[sqe->fd];
+		file = ctx->user_files[fd];
 	} else {
-		file = fget(sqe->fd);
+		file = fget(fd);
 	}
 
 	if (unlikely(!file))
 		return -EBADF;
 
-	ret = vfs_fsync_range(file, sqe->off, end > 0 ? end : LLONG_MAX,
+	ret = vfs_fsync_range(file, sqe_off, end > 0 ? end : LLONG_MAX,
 			sqe->fsync_flags & IORING_FSYNC_DATASYNC);
 
-	if (!(sqe->flags & IOSQE_FIXED_FILE))
+	if (!(flags & IOSQE_FIXED_FILE))
 		fput(file);
 
 	io_cqring_add_event(ctx, sqe->user_data, ret, 0);
@@ -1037,7 +1052,7 @@ static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 
 	spin_lock_irq(&ctx->completion_lock);
 	list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
-		if (sqe->addr == poll_req->user_data) {
+		if (READ_ONCE(sqe->addr) == poll_req->user_data) {
 			io_poll_remove_one(poll_req);
 			ret = 0;
 			break;
@@ -1145,7 +1160,10 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	struct io_poll_iocb *poll = &req->poll;
 	struct io_ring_ctx *ctx = req->ctx;
 	struct io_poll_table ipt;
+	unsigned flags;
 	__poll_t mask;
+	u16 events;
+	int fd;
 
 	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
 		return -EINVAL;
@@ -1153,15 +1171,18 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 		return -EINVAL;
 
 	INIT_WORK(&req->work, io_poll_complete_work);
-	poll->events = demangle_poll(sqe->poll_events) | EPOLLERR | EPOLLHUP;
+	events = READ_ONCE(sqe->poll_events);
+	poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
 
-	if (sqe->flags & IOSQE_FIXED_FILE) {
-		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+	flags = READ_ONCE(sqe->flags);
+	fd = READ_ONCE(sqe->fd);
+	if (flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || fd >= ctx->nr_user_files))
 			return -EBADF;
-		poll->file = ctx->user_files[sqe->fd];
+		poll->file = ctx->user_files[fd];
 		req->flags |= REQ_F_FIXED_FILE;
 	} else {
-		poll->file = fget(sqe->fd);
+		poll->file = fget(fd);
 	}
 	if (unlikely(!poll->file))
 		return -EBADF;
@@ -1207,7 +1228,7 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 
 out:
 	if (unlikely(ipt.error)) {
-		if (!(sqe->flags & IOSQE_FIXED_FILE))
+		if (!(flags & IOSQE_FIXED_FILE))
 			fput(poll->file);
 		return ipt.error;
 	}
@@ -1224,15 +1245,17 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 {
 	const struct io_uring_sqe *sqe = s->sqe;
 	ssize_t ret;
+	int opcode;
 
 	if (unlikely(s->index >= ctx->sq_entries))
 		return -EINVAL;
-	req->user_data = sqe->user_data;
+	req->user_data = READ_ONCE(sqe->user_data);
 
 	ret = -EINVAL;
-	switch (sqe->opcode) {
+	opcode = READ_ONCE(sqe->opcode);
+	switch (opcode) {
 	case IORING_OP_NOP:
-		ret = io_nop(req, sqe);
+		ret = io_nop(req, req->user_data);
 		break;
 	case IORING_OP_READV:
 		if (unlikely(sqe->buf_index))
@@ -1317,7 +1340,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)
 restart:
 	do {
 		struct sqe_submit *s = &req->submit;
-		u64 user_data = s->sqe->user_data;
+		u64 user_data = READ_ONCE(s->sqe->user_data);
 
 		/* Ensure we clear previously set forced non-block flag */
 		req->flags &= ~REQ_F_FORCE_NONBLOCK;
Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#25 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 19 lines · Jens Axboe
Quoted 5 lines · Jann Horn

Please create a local copy of the request before parsing it to keep the data from changing under you. Additionally, it might make sense to annotate every pointer to shared memory with a comment, or something like that, to ensure that anyone looking at the code can immediately see for which pointers special caution is required on access.

I took a look at the viability of NOT having to local copy the data, and I don't think it's too bad. Local copy has a noticeable impact on the performance, hence I'd really (REALLY) like to avoid it.

Here's something on top of the current git branch. I think I even went a bit too far in some areas, but it should hopefully catch the cases where we might end up double evaluating the parts of the sqe that we depend on. For most of the sqe reading we don't really care too much. For instance, the sqe->user_data. If the app changes this field, then it just gets whatever passed back in cqe->user_data. That's not a kernel issue.

Quoted 6 lines

For cases like addr/len etc validation, it should be sound. I'll double check this in the morning as well, and obviously would need to be folded in along the way.

I'd appreciate your opinion on this part, if you see any major issues with it, or if I missed something.

The io_sqe_needs_user() checks still look racy. If that helper sees a IORING_OP_READ_FIXED, but then __io_submit_sqe() sees a IORING_OP_READV - especially if this happens in io_sq_wq_submit_work() -, I think you could potentially end up in places like io_import_iovec() without having done the set_fs(USER_DS) and use_mm(), causing the access to potentially occur with KERNEL_DS and a lazy mm.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#26 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 27 lines · Jann Horn
Quoted 19 lines · Jens Axboe
Quoted 5 lines · Jann Horn

Please create a local copy of the request before parsing it to keep the data from changing under you. Additionally, it might make sense to annotate every pointer to shared memory with a comment, or something like that, to ensure that anyone looking at the code can immediately see for which pointers special caution is required on access.

I took a look at the viability of NOT having to local copy the data, and I don't think it's too bad. Local copy has a noticeable impact on the performance, hence I'd really (REALLY) like to avoid it.

Here's something on top of the current git branch. I think I even went a bit too far in some areas, but it should hopefully catch the cases where we might end up double evaluating the parts of the sqe that we depend on. For most of the sqe reading we don't really care too much. For instance, the sqe->user_data. If the app changes this field, then it just gets whatever passed back in cqe->user_data. That's not a kernel issue.

Quoted 6 lines

For cases like addr/len etc validation, it should be sound. I'll double check this in the morning as well, and obviously would need to be folded in along the way.

I'd appreciate your opinion on this part, if you see any major issues with it, or if I missed something.

Quoted 7 lines

The io_sqe_needs_user() checks still look racy. If that helper sees a IORING_OP_READ_FIXED, but then __io_submit_sqe() sees a IORING_OP_READV - especially if this happens in io_sq_wq_submit_work() -, I think you could potentially end up in places like io_import_iovec() without having done the set_fs(USER_DS) and use_mm(), causing the access to potentially occur with KERNEL_DS and a lazy mm.

Indeed, for that case I think we should just copy the sqe. It's in the async offload context anyway, so a copy won't really change anything in terms of performance. And since the gap is so large between the two problematic spots, it'd be trickier to fix.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#27 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jann Horn
Quoted 36 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 12 lines
fs/io_uring.c new file +12 -0
+static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
+{
+       struct io_sq_ring *ring = ctx->sq_ring;
+       unsigned head;
+
+       /*
+        * The cached sq head (or cq tail) serves two purposes:
+        *
+        * 1) allows us to batch the cost of updating the user visible
+        *    head updates.
+        * 2) allows the kernel side to track the head on its own, even
+        *    though the application is the one updating it.

fs/io_uring.c line 601

Quoted 10 lines
fs/io_uring.c new file +10 -0
+        */
+       head = ctx->cached_sq_head;
+       smp_rmb();
+       if (head == READ_ONCE(ring->r.tail))
+               return false;
+
+       head = ring->array[head & ctx->sq_mask];
+       if (head < ctx->sq_entries) {
+               s->index = head;
+               s->sqe = &ctx->sq_sqes[head];

ring->array can be mapped writable into userspace, right? If so: This looks like a double-read issue; the compiler might assume that ring->array is not modified concurrently and perform separate memory accesses for the "if (head < ctx->sq_entries)" check and the "&ctx->sq_sqes[head]" computation. Please use READ_ONCE()/WRITE_ONCE() for all accesses to memory that userspace could concurrently modify in a malicious way.

Quoted 7 lines

There have been some pretty severe security bugs caused by missing
READ_ONCE() annotations around accesses to shared memory; see, for
example, https://www.blackhat.com/docs/us-16/materials/us-16-Wilhelm-Xenpwn-Breaking-Paravirtualized-Devices.pdf
. Slides 35-48 show how the code "switch (op->cmd)", where "op" is a
pointer to shared memory, allowed an attacker to break out of a Xen
virtual machine because the compiler generated multiple memory
accesses.

Thanks, I'll update these to use READ/WRITE_ONCE.

Quoted 25 lines
Quoted 10 lines
fs/io_uring.c new file +10 -0
+               ctx->cached_sq_head++;
+               return true;
+       }
+
+       /* drop invalid entries */
+       ctx->cached_sq_head++;
+       ring->dropped++;
+       smp_wmb();
+       return false;
+}

[...]

Quoted 14 lines
fs/io_uring.c new file +14 -0
+SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
+               u32, min_complete, u32, flags, const sigset_t __user *, sig,
+               size_t, sigsz)
+{
+       struct io_ring_ctx *ctx;
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
Quoted 8 lines
Quoted 1 line

+ goto out_fput;

Oh, by the way: If you feel like it, maybe you could add a helper fdget_typed(int fd, struct file_operations *f_op), or something like that, so that there is less boilerplate code for first doing fdget(), then checking ->f_op, and then coding an extra bailout path for that? But that doesn't really have much to do with your patchset, feel free to ignore this comment.

That's not a bad idea, I think this is a fairly common code pattern. I'll look into it.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#28 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

fs/io_uring.c line 238

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+{
+       struct io_kiocb *req;
+
+       /* safe to use the non tryget, as we're inside ring ref already */
+       percpu_ref_get(&ctx->refs);

Is that true? In the path io_sq_thread() -> io_submit_sqes() -> io_submit_sqe() -> io_get_req(), I don't see anything that's already holding a reference for you. Is the worker thread holding a reference somewhere that I'm missing?

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#29 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 5 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 11 lines

fs/io_uring.c line 238

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+{
+       struct io_kiocb *req;
+
+       /* safe to use the non tryget, as we're inside ring ref already */
+       percpu_ref_get(&ctx->refs);

Is that true? In the path io_sq_thread() -> io_submit_sqes() -> io_submit_sqe() -> io_get_req(), I don't see anything that's already holding a reference for you. Is the worker thread holding a reference somewhere that I'm missing?

If the thread is alive, then the ctx is alive. Before we drop the last ref to the ctx (and kill it), we wait for the thread to exit.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#30 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 18 lines · Jens Axboe
Quoted 5 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 11 lines

fs/io_uring.c line 238

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+{
+       struct io_kiocb *req;
+
+       /* safe to use the non tryget, as we're inside ring ref already */
+       percpu_ref_get(&ctx->refs);

Is that true? In the path io_sq_thread() -> io_submit_sqes() -> io_submit_sqe() -> io_get_req(), I don't see anything that's already holding a reference for you. Is the worker thread holding a reference somewhere that I'm missing?

Quoted 2 lines

If the thread is alive, then the ctx is alive. Before we drop the last ref to the ctx (and kill it), we wait for the thread to exit.

Where in __io_uring_register() are you waiting for the thread to exit before killing it? As far as I can tell, you come straight in from syscall context, take a mutex, and do percpu_ref_kill().

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#31 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 19 lines · Jann Horn
Quoted 18 lines · Jens Axboe
Quoted 16 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

fs/io_uring.c line 238

Quoted 6 lines
fs/io_uring.c new file +6 -0
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+{
+       struct io_kiocb *req;
+
+       /* safe to use the non tryget, as we're inside ring ref already */
+       percpu_ref_get(&ctx->refs);

Is that true? In the path io_sq_thread() -> io_submit_sqes() -> io_submit_sqe() -> io_get_req(), I don't see anything that's already holding a reference for you. Is the worker thread holding a reference somewhere that I'm missing?

Quoted 6 lines
Quoted 2 lines

If the thread is alive, then the ctx is alive. Before we drop the last ref to the ctx (and kill it), we wait for the thread to exit.

Where in __io_uring_register() are you waiting for the thread to exit before killing it? As far as I can tell, you come straight in from syscall context, take a mutex, and do percpu_ref_kill().

I was just referring to the regular shutdown path. You are right, for the later io_uring_register() more care needs to be taken. I'll just switch to the tryget, it's not like the non-try is a big cycle saver by any stretch.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#32 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe
Quoted 157 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.

Two new system calls are added for this:

io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.

io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.

With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.

For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.

Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.

Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c

---
 arch/x86/entry/syscalls/syscall_32.tbl |    2 +
 arch/x86/entry/syscalls/syscall_64.tbl |    2 +
 fs/Makefile                            |    1 +
 fs/io_uring.c                          | 1090 ++++++++++++++++++++++++
 include/linux/syscalls.h               |    6 +
 include/uapi/asm-generic/unistd.h      |    6 +-
 include/uapi/linux/io_uring.h          |   96 +++
 init/Kconfig                           |    9 +
 kernel/sys_ni.c                        |    2 +
 9 files changed, 1213 insertions(+), 1 deletion(-)
 create mode 100644 fs/io_uring.c
 create mode 100644 include/uapi/linux/io_uring.h
include/uapi/linux/io_uring.h new file +86 -0
new file mode 100644
index 000000000000..ce65db9269a8
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Header file for the io_uring interface.
+ *
+ * Copyright (C) 2019 Jens Axboe
+ * Copyright (C) 2019 Christoph Hellwig
+ */
+#ifndef LINUX_IO_URING_H
+#define LINUX_IO_URING_H
+
+#include <linux/fs.h>
+#include <linux/types.h>
+
+#define IORING_MAX_ENTRIES     4096
+
+/*
+ * IO submission data structure (Submission Queue Entry)
+ */
+struct io_uring_sqe {
+       __u8    opcode;         /* type of operation for this sqe */
+       __u8    flags;          /* as of now unused */
+       __u16   ioprio;         /* ioprio for the request */
+       __s32   fd;             /* file descriptor to do IO on */
+       __u64   off;            /* offset into file */
+       __u64   addr;           /* pointer to buffer or iovecs */
+       __u32   len;            /* buffer size or number of iovecs */
+       union {
+               __kernel_rwf_t  rw_flags;
+               __u32           __resv;
+       };
+       __u64   user_data;      /* data to be passed back at completion time */
+       __u64   __pad2[3];
+};
+
+#define IORING_OP_NOP          0
+#define IORING_OP_READV                1
+#define IORING_OP_WRITEV       2
+
+/*
+ * IO completion data structure (Completion Queue Entry)
+ */
+struct io_uring_cqe {
+       __u64   user_data;      /* sqe->data submission passed back */
+       __s32   res;            /* result code for this event */
+       __u32   flags;
+};
+
+/*
+ * Magic offsets for the application to mmap the data it needs
+ */
+#define IORING_OFF_SQ_RING             0ULL
+#define IORING_OFF_CQ_RING             0x8000000ULL
+#define IORING_OFF_SQES                        0x10000000ULL
+
+/*
+ * Filled with the offset for mmap(2)
+ */
+struct io_sqring_offsets {
+       __u32 head;
+       __u32 tail;
+       __u32 ring_mask;
+       __u32 ring_entries;
+       __u32 flags;
+       __u32 dropped;
+       __u32 array;
+       __u32 resv[3];
+};
+
+struct io_cqring_offsets {
+       __u32 head;
+       __u32 tail;
+       __u32 ring_mask;
+       __u32 ring_entries;
+       __u32 overflow;
+       __u32 cqes;
+       __u32 resv[4];
+};
+
+/*
+ * io_uring_enter(2) flags
+ */
+#define IORING_ENTER_GETEVENTS (1 << 0)
+
+/*
+ * Passed in for io_uring_setup(2). Copied back with updated info on success
+ */

include/uapi/linux/io_uring.h line 96

Quoted 10 lines
include/uapi/linux/io_uring.h new file +10 -0
+struct io_uring_params {
+       __u32 sq_entries;
+       __u32 cq_entries;
+       __u32 flags;
+       __u16 resv[10];
+       struct io_sqring_offsets sq_off;
+       struct io_cqring_offsets cq_off;
+};
+
+#endif

from a user perspective, it should always be easier if all exported symbols and macros have a common prefix. Here it seems particular worrisome, because of the missing 'u' in in the defines.

Best,
Bert

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#33 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe

* Jens Axboe:

Quoted 1 line

+#define IORING_MAX_ENTRIES 4096

Where does this constant come from? Should it really be exposed to userspace?

Quoted 19 lines
(quoted diff) +18 -0
+struct io_uring_params {
+	__u32 sq_entries;
+	__u32 cq_entries;
+	__u32 flags;
+	__u16 resv[10];
+	struct io_sqring_offsets sq_off;
+	struct io_cqring_offsets cq_off;
+};
 
+struct io_sqring_offsets {
+	__u32 head;
+	__u32 tail;
+	__u32 ring_mask;
+	__u32 ring_entries;
+	__u32 flags;
+	__u32 dropped;
+	__u32 array;
+	__u32 resv[3];
+};

Review

Quoted 10 lines
(quoted diff) +10 -0
+
+struct io_cqring_offsets {
+	__u32 head;
+	__u32 tail;
+	__u32 ring_mask;
+	__u32 ring_entries;
+	__u32 overflow;
+	__u32 cqes;
+	__u32 resv[4];
+};

Should the reserved fields include a __u64 member, to increase struct alignment on architectures that might need it in the future?

Quoted 1 line

+#define IORING_ENTER_GETEVENTS (1 << 0)

Should this be unsigned, to match the u32 flags argument? (Otherwise using 32 flags can be difficult).

Thanks,
Florian

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#34 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 6 lines · Florian Weimer

* Jens Axboe:

Quoted 1 line

+#define IORING_MAX_ENTRIES 4096

Where does this constant come from? Should it really be exposed to userspace?

Seems pretty handy for the application to know what the limit is?

Quoted 24 lines
Quoted 24 lines
(quoted diff) +23 -0
+struct io_uring_params {
+	__u32 sq_entries;
+	__u32 cq_entries;
+	__u32 flags;
+	__u16 resv[10];
+	struct io_sqring_offsets sq_off;
+	struct io_cqring_offsets cq_off;
+};
 
+struct io_sqring_offsets {
+	__u32 head;
+	__u32 tail;
+	__u32 ring_mask;
+	__u32 ring_entries;
+	__u32 flags;
+	__u32 dropped;
+	__u32 array;
+	__u32 resv[3];
+};
+
+struct io_cqring_offsets {
+	__u32 head;
+	__u32 tail;
+	__u32 ring_mask;
Quoted 8 lines

include/uapi/linux/io_uring.h line 77

Quoted 5 lines
include/uapi/linux/io_uring.h new file +5 -0
+	__u32 ring_entries;
+	__u32 overflow;
+	__u32 cqes;
+	__u32 resv[4];
+};

Should the reserved fields include a __u64 member, to increase struct alignment on architectures that might need it in the future?

Sure, I can do that.

Quoted 4 lines
Quoted 1 line

+#define IORING_ENTER_GETEVENTS (1 << 0)

Should this be unsigned, to match the u32 flags argument? (Otherwise using 32 flags can be difficult).

Good point, I've changed them all to be unsigned.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#35 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 10 lines · Jens Axboe
Quoted 6 lines · Florian Weimer

* Jens Axboe:

Quoted 1 line

+#define IORING_MAX_ENTRIES 4096

Where does this constant come from? Should it really be exposed to userspace?

Seems pretty handy for the application to know what the limit is?

The running kernel version might be different from the kernel version whose headers the userspace code was compiled with. So if this is a limit that might change at some point in the future, and you think userspace wants to know the limit, some other API might be better.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#36 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 10 lines · Jann Horn
Quoted 9 lines · Jens Axboe
Quoted 6 lines · Florian Weimer

* Jens Axboe:

Quoted 1 line

+#define IORING_MAX_ENTRIES 4096

Where does this constant come from? Should it really be exposed to userspace?

Quoted 6 lines
Quoted 1 line

Seems pretty handy for the application to know what the limit is?

The running kernel version might be different from the kernel version whose headers the userspace code was compiled with. So if this is a limit that might change at some point in the future, and you think userspace wants to know the limit, some other API might be better.

Probably best to just kill it, we'll error in io_uring_register() if it's exceeded anyway.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#37 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jann Horn
Quoted 11 lines · Jann Horn
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

+#define IORING_MAX_ENTRIES 4096

Where does this constant come from? Should it really be exposed to userspace?

Seems pretty handy for the application to know what the limit is?

The running kernel version might be different from the kernel version whose headers the userspace code was compiled with. So if this is a limit that might change at some point in the future, and you think userspace wants to know the limit, some other API might be better.

If it changes it will increase, so I'm not worried it is all that harmful. What might be useful is a specific error code if it is too large so that applications can check it. Just which one?

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

#38 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

In reply to Jens Axboe
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 4 lines
fs/io_uring.c new file +4 -0
+static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
+                          const struct io_uring_sqe *sqe,
+                          struct iovec **iovec, struct iov_iter *iter)
+{

fs/io_uring.c line 357

Quoted 10 lines
fs/io_uring.c new file +10 -0
+       void __user *buf = u64_to_user_ptr(sqe->addr);
+
+#ifdef CONFIG_COMPAT
+       if (in_compat_syscall())
+               return compat_import_iovec(rw, buf, sqe->len, UIO_FASTIOV,
+                                               iovec, iter);
+#endif
+
+       return import_iovec(rw, buf, sqe->len, UIO_FASTIOV, iovec, iter);
+}

This code can run in kthread context, right? I think in_compat_syscall() might not work if this is a kthread launched by a compat task; I don't see anything that propagates the compat flag to the kthread.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#39 lore raw

Re: [PATCH 05/18] Add io_uring IO interface

Quoted 16 lines · Jann Horn
Quoted 3 lines · Jens Axboe

The submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.

[...]

Quoted 11 lines
fs/io_uring.c new file +11 -0
+static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
+                          const struct io_uring_sqe *sqe,
+                          struct iovec **iovec, struct iov_iter *iter)
+{
+       void __user *buf = u64_to_user_ptr(sqe->addr);
+
+#ifdef CONFIG_COMPAT
+       if (in_compat_syscall())
+               return compat_import_iovec(rw, buf, sqe->len, UIO_FASTIOV,
+                                               iovec, iter);
+#endif
Quoted 8 lines

fs/io_uring.c line 357

Quoted 3 lines
fs/io_uring.c new file +3 -0
+
+       return import_iovec(rw, buf, sqe->len, UIO_FASTIOV, iovec, iter);
+}

This code can run in kthread context, right? I think in_compat_syscall() might not work if this is a kthread launched by a compat task; I don't see anything that propagates the compat flag to the kthread.

Good catch, yes it can. We should just carry this information in sqe_submit for this out-of-line purpose. I'll make that change.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#40 lore raw

PATCH 6/18 io_uring: add fsync support

In reply to Jens Axboe

From: Christoph Hellwig <[email protected]>

Add a new fsync opcode, which either syncs a range if one is passed, or the whole file if the offset and length fields are both cleared to zero. A flag is provided to use fdatasync semantics, that is only force out metadata which is required to retrieve the file data, but not others like metadata.

---
 fs/io_uring.c                 | 34 ++++++++++++++++++++++++++++++++++
 include/uapi/linux/io_uring.h |  8 +++++++-
 2 files changed, 41 insertions(+), 1 deletion(-)
fs/io_uring.c +34 -0
index 309eed3629d2..7d2e6db08b05 100644
@@ -4,6 +4,7 @@
  * supporting fast/efficient IO.
  *
  * Copyright (C) 2018-2019 Jens Axboe
+ * Copyright (c) 2018-2019 Christoph Hellwig
  */
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -466,6 +467,36 @@ static int io_nop(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	return 0;
 }
 
+static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+		    bool force_nonblock)
+{
+	struct io_ring_ctx *ctx = req->ctx;
+	loff_t end = sqe->off + sqe->len;
+	struct file *file;
+	int ret;
+
+	/* fsync always requires a blocking context */
+	if (force_nonblock)
+		return -EAGAIN;
+
+	if (unlikely(sqe->addr || sqe->ioprio))
+		return -EINVAL;
+	if (unlikely(sqe->fsync_flags & ~IORING_FSYNC_DATASYNC))
+		return -EINVAL;
+
+	file = fget(sqe->fd);
+	if (unlikely(!file))
+		return -EBADF;
+
+	ret = vfs_fsync_range(file, sqe->off, end > 0 ? end : LLONG_MAX,
+			sqe->fsync_flags & IORING_FSYNC_DATASYNC);
+
+	fput(file);
+	io_cqring_add_event(ctx, sqe->user_data, ret, 0);
+	io_free_req(req);
+	return 0;
+}
+
 static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 			   struct sqe_submit *s, bool force_nonblock)
 {
@@ -487,6 +518,9 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 	case IORING_OP_WRITEV:
 		ret = io_write(req, sqe, force_nonblock);
 		break;
+	case IORING_OP_FSYNC:
+		ret = io_fsync(req, sqe, force_nonblock);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
include/uapi/linux/io_uring.h +7 -1
index ce65db9269a8..ca503ded73e3 100644
@@ -26,7 +26,7 @@ struct io_uring_sqe {
 	__u32	len;		/* buffer size or number of iovecs */
 	union {
 		__kernel_rwf_t	rw_flags;
-		__u32		__resv;
+		__u32		fsync_flags;
 	};
 	__u64	user_data;	/* data to be passed back at completion time */
 	__u64	__pad2[3];
@@ -35,6 +35,12 @@ struct io_uring_sqe {
 #define IORING_OP_NOP		0
 #define IORING_OP_READV		1
 #define IORING_OP_WRITEV	2
+#define IORING_OP_FSYNC		3
+
+/*
+ * sqe->fsync_flags
+ */
+#define IORING_FSYNC_DATASYNC	(1 << 0)
 
 /*
  * IO completion data structure (Completion Queue Entry)
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#41 lore raw

PATCH 7/18 io_uring: support for IO polling

In reply to Jens Axboe

Add support for a polled io_uring context. When a read or write is submitted to a polled context, the application must poll for completions on the CQ ring through io_uring_enter(2). Polled IO may not generate IRQ completions, hence they need to be actively found by the application itself.

To use polling, io_uring_setup() must be used with the IORING_SETUP_IOPOLL flag being set. It is illegal to mix and match polled and non-polled IO on an io_uring.

---
 fs/io_uring.c                 | 262 ++++++++++++++++++++++++++++++++--
 include/uapi/linux/io_uring.h |   5 +
 2 files changed, 256 insertions(+), 11 deletions(-)
fs/io_uring.c +251 -11
index 7d2e6db08b05..ed5b605a1748 100644
@@ -100,6 +100,8 @@ struct io_ring_ctx {
 
 	struct {
 		spinlock_t		completion_lock;
+		bool			poll_multi_file;
+		struct list_head	poll_list;
 	} ____cacheline_aligned_in_smp;
 };
 
@@ -118,12 +120,16 @@ struct io_kiocb {
 	struct list_head	list;
 	unsigned int		flags;
 #define REQ_F_FORCE_NONBLOCK	1	/* inline submission attempt */
+#define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
+#define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
 	u64			user_data;
+	u64			res;
 
 	struct work_struct	work;
 };
 
 #define IO_PLUG_THRESHOLD		2
+#define IO_IOPOLL_BATCH			8
 
 static struct kmem_cache *req_cachep;
 
@@ -155,6 +161,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	mutex_init(&ctx->uring_lock);
 	init_waitqueue_head(&ctx->wait);
 	spin_lock_init(&ctx->completion_lock);
+	INIT_LIST_HEAD(&ctx->poll_list);
 	return ctx;
 }
 
@@ -190,8 +197,8 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)
 	return &ring->cqes[tail & ctx->cq_mask];
 }
 
-static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
-				  long res, unsigned ev_flags)
+static void io_cqring_fill_event(struct io_ring_ctx *ctx, u64 ki_user_data,
+				 long res, unsigned ev_flags)
 {
 	struct io_uring_cqe *cqe;
 
@@ -205,9 +212,15 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
 		cqe->user_data = ki_user_data;
 		cqe->res = res;
 		cqe->flags = ev_flags;
-		io_commit_cqring(ctx);
 	} else
 		ctx->cq_ring->overflow++;
+}
+
+static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
+				  long res, unsigned ev_flags)
+{
+	io_cqring_fill_event(ctx, ki_user_data, res, ev_flags);
+	io_commit_cqring(ctx);
 
 	if (waitqueue_active(&ctx->wait))
 		wake_up(&ctx->wait);
@@ -249,12 +262,158 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
 	return NULL;
 }
 
+static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
+{
+	if (*nr) {
+		kmem_cache_free_bulk(req_cachep, *nr, reqs);
+		io_ring_drop_ctx_refs(ctx, *nr);
+		*nr = 0;
+	}
+}
+
 static void io_free_req(struct io_kiocb *req)
 {
 	io_ring_drop_ctx_refs(req->ctx, 1);
 	kmem_cache_free(req_cachep, req);
 }
 
+/*
+ * Find and free completed poll iocbs
+ */
+static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
+			       struct list_head *done)
+{
+	void *reqs[IO_IOPOLL_BATCH];
+	struct io_kiocb *req;
+	int to_free = 0;
+
+	while (!list_empty(done)) {
+		req = list_first_entry(done, struct io_kiocb, list);
+		list_del(&req->list);
+
+		io_cqring_fill_event(ctx, req->user_data, req->res, 0);
+
+		reqs[to_free++] = req;
+		(*nr_events)++;
+
+		fput(req->rw.ki_filp);
+		if (to_free == ARRAY_SIZE(reqs))
+			io_free_req_many(ctx, reqs, &to_free);
+	}
+	io_commit_cqring(ctx);
+
+	if (to_free)
+		io_free_req_many(ctx, reqs, &to_free);
+}
+
+static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
+			long min)
+{
+	struct io_kiocb *req, *tmp;
+	LIST_HEAD(done);
+	bool spin;
+	int ret;
+
+	/*
+	 * Only spin for completions if we don't have multiple devices hanging
+	 * off our complete list, and we're under the requested amount.
+	 */
+	spin = !ctx->poll_multi_file && (*nr_events < min);
+
+	ret = 0;
+	list_for_each_entry_safe(req, tmp, &ctx->poll_list, list) {
+		struct kiocb *kiocb = &req->rw;
+
+		/*
+		 * Move completed entries to our local list. If we find a
+		 * request that requires polling, break out and complete
+		 * the done list first, if we have entries there.
+		 */
+		if (req->flags & REQ_F_IOPOLL_COMPLETED) {
+			list_move_tail(&req->list, &done);
+			continue;
+		}
+		if (!list_empty(&done))
+			break;
+
+		ret = kiocb->ki_filp->f_op->iopoll(kiocb, spin);
+		if (ret < 0)
+			break;
+
+		if (ret && spin)
+			spin = false;
+		ret = 0;
+	}
+
+	if (!list_empty(&done))
+		io_iopoll_complete(ctx, nr_events, &done);
+
+	return ret;
+}
+
+/*
+ * Poll for a mininum of 'min' events. Note that if min == 0 we consider that a
+ * non-spinning poll check - we'll still enter the driver poll loop, but only
+ * as a non-spinning completion check.
+ */
+static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
+				long min)
+{
+	int ret;
+
+	do {
+		if (list_empty(&ctx->poll_list))
+			return 0;
+
+		ret = io_do_iopoll(ctx, nr_events, min);
+		if (ret < 0)
+			break;
+	} while (min && *nr_events < min);
+
+	if (ret < 0)
+		return ret;
+
+	return *nr_events < min;
+}
+
+/*
+ * We can't just wait for polled events to come to us, we have to actively
+ * find and complete them.
+ */
+static void io_iopoll_reap_events(struct io_ring_ctx *ctx)
+{
+	if (!(ctx->flags & IORING_SETUP_IOPOLL))
+		return;
+
+	mutex_lock(&ctx->uring_lock);
+	while (!list_empty(&ctx->poll_list)) {
+		unsigned int nr_events = 0;
+
+		io_iopoll_getevents(ctx, &nr_events, 1);
+	}
+	mutex_unlock(&ctx->uring_lock);
+}
+
+static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events,
+			   long min)
+{
+	int ret = 0;
+
+	do {
+		int tmin = 0;
+
+		if (*nr_events < min)
+			tmin = min - *nr_events;
+
+		ret = io_iopoll_getevents(ctx, nr_events, tmin);
+		if (ret <= 0)
+			break;
+		ret = 0;
+	} while (!*nr_events || !need_resched());
+
+	return ret;
+}
+
 static void kiocb_end_write(struct kiocb *kiocb)
 {
 	if (kiocb->ki_flags & IOCB_WRITE) {
@@ -281,9 +440,60 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
 	io_free_req(req);
 }
 
+static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
+{
+	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
+
+	kiocb_end_write(kiocb);
+
+	if (unlikely(res == -EAGAIN)) {
+		req->flags |= REQ_F_IOPOLL_EAGAIN;
+	} else {
+		req->flags |= REQ_F_IOPOLL_COMPLETED;
+		req->res = res;
+	}
+}
+
+/*
+ * After the iocb has been issued, it's safe to be found on the poll list.
+ * Adding the kiocb to the list AFTER submission ensures that we don't
+ * find it from a io_iopoll_getevents() thread before the issuer is done
+ * accessing the kiocb cookie.
+ */
+static void io_iopoll_req_issued(struct io_kiocb *req)
+{
+	struct io_ring_ctx *ctx = req->ctx;
+
+	/*
+	 * Track whether we have multiple files in our lists. This will impact
+	 * how we do polling eventually, not spinning if we're on potentially
+	 * different devices.
+	 */
+	if (list_empty(&ctx->poll_list)) {
+		ctx->poll_multi_file = false;
+	} else if (!ctx->poll_multi_file) {
+		struct io_kiocb *list_req;
+
+		list_req = list_first_entry(&ctx->poll_list, struct io_kiocb,
+						list);
+		if (list_req->rw.ki_filp != req->rw.ki_filp)
+			ctx->poll_multi_file = true;
+	}
+
+	/*
+	 * For fast devices, IO may have already completed. If it has, add
+	 * it to the front so we find it first.
+	 */
+	if (req->flags & REQ_F_IOPOLL_COMPLETED)
+		list_add(&req->list, &ctx->poll_list);
+	else
+		list_add_tail(&req->list, &ctx->poll_list);
+}
+
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		      bool force_nonblock)
 {
+	struct io_ring_ctx *ctx = req->ctx;
 	struct kiocb *kiocb = &req->rw;
 	int ret;
 
@@ -309,12 +519,21 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		kiocb->ki_flags |= IOCB_NOWAIT;
 		req->flags |= REQ_F_FORCE_NONBLOCK;
 	}
-	if (kiocb->ki_flags & IOCB_HIPRI) {
-		ret = -EINVAL;
-		goto out_fput;
-	}
+	if (ctx->flags & IORING_SETUP_IOPOLL) {
+		ret = -EOPNOTSUPP;
+		if (!(kiocb->ki_flags & IOCB_DIRECT) ||
+		    !kiocb->ki_filp->f_op->iopoll)
+			goto out_fput;
 
-	kiocb->ki_complete = io_complete_rw;
+		kiocb->ki_flags |= IOCB_HIPRI;
+		kiocb->ki_complete = io_complete_rw_iopoll;
+	} else {
+		if (kiocb->ki_flags & IOCB_HIPRI) {
+			ret = -EINVAL;
+			goto out_fput;
+		}
+		kiocb->ki_complete = io_complete_rw;
+	}
 	return 0;
 out_fput:
 	fput(kiocb->ki_filp);
@@ -462,6 +681,9 @@ static int io_nop(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 
+	if (unlikely(ctx->flags & IORING_SETUP_IOPOLL))
+		return -EINVAL;
+
 	io_cqring_add_event(ctx, sqe->user_data, 0, 0);
 	io_free_req(req);
 	return 0;
@@ -479,6 +701,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	if (force_nonblock)
 		return -EAGAIN;
 
+	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+		return -EINVAL;
 	if (unlikely(sqe->addr || sqe->ioprio))
 		return -EINVAL;
 	if (unlikely(sqe->fsync_flags & ~IORING_FSYNC_DATASYNC))
@@ -526,7 +750,16 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 		break;
 	}
 
-	return ret;
+	if (ret)
+		return ret;
+
+	if (ctx->flags & IORING_SETUP_IOPOLL) {
+		if (req->flags & REQ_F_IOPOLL_EAGAIN)
+			return -EAGAIN;
+		io_iopoll_req_issued(req);
+	}
+
+	return 0;
 }
 
 static void io_sq_wq_submit_work(struct work_struct *work)
@@ -734,6 +967,8 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
 			return submitted;
 	}
 	if (flags & IORING_ENTER_GETEVENTS) {
+		unsigned nr_events = 0;
+
 		/*
 		 * The application could have included the 'to_submit' count
 		 * in how many events it wanted to wait for. If we failed to
@@ -743,7 +978,10 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
 		if (submitted < to_submit)
 			min_complete = min_t(unsigned, submitted, min_complete);
 
-		ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
+		if (ctx->flags & IORING_SETUP_IOPOLL)
+			ret = io_iopoll_check(ctx, &nr_events, min_complete);
+		else
+			ret = io_cqring_wait(ctx, min_complete, sig, sigsz);
 	}
 
 	return submitted ? submitted : ret;
@@ -842,6 +1080,7 @@ static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
 static void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
 	destroy_workqueue(ctx->sqo_wq);
+	io_iopoll_reap_events(ctx);
 
 	io_mem_free(ctx->sq_ring);
 	io_mem_free(ctx->sq_sqes);
@@ -880,6 +1119,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 	percpu_ref_kill(&ctx->refs);
 	mutex_unlock(&ctx->uring_lock);
 
+	io_iopoll_reap_events(ctx);
 	wait_for_completion(&ctx->ctx_done);
 	io_ring_ctx_free(ctx);
 }
@@ -1097,7 +1337,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 			return -EINVAL;
 	}
 
-	if (p.flags)
+	if (p.flags & ~IORING_SETUP_IOPOLL)
 		return -EINVAL;
 
 	ret = io_uring_create(entries, &p);
include/uapi/linux/io_uring.h +5 -0
index ca503ded73e3..4fc5fbd07688 100644
@@ -32,6 +32,11 @@ struct io_uring_sqe {
 	__u64	__pad2[3];
 };
 
+/*
+ * io_uring_setup() flags
+ */
+#define IORING_SETUP_IOPOLL	(1 << 0)	/* io_context is polled */
+
 #define IORING_OP_NOP		0
 #define IORING_OP_READV		1
 #define IORING_OP_WRITEV	2
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#42 lore raw

Re: [PATCH 07/18] io_uring: support for IO polling

fs/io_uring.c struct io_kiocb { · line 126

Quoted 9 lines
fs/io_uring.c +3 -0
@@ -118,12 +120,16 @@ struct io_kiocb {
 	struct list_head	list;
 	unsigned int		flags;
 #define REQ_F_FORCE_NONBLOCK	1	/* inline submission attempt */
+#define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
+#define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
 	u64			user_data;
+	u64			res;

Should this be ret or error instead? res is kinda off. A little comment describing it won't hurt either. Last but not least with the actual errno value stored here we probably don't need the REQ_F_IOPOLL_EAGAIN flag, do we?

fs/io_uring.c static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) · line 321

Quoted 5 lines
fs/io_uring.c +5 -0
+	/*
+	 * Only spin for completions if we don't have multiple devices hanging
+	 * off our complete list, and we're under the requested amount.
+	 */
+	spin = !ctx->poll_multi_file && (*nr_events < min);

no need for the braces here.

Quoted 8 lines
fs/io_uring.c +8 -0
+static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
+				long min)
+{
+	int ret;
+
+	do {
+		if (list_empty(&ctx->poll_list))
+			return 0;

fs/io_uring.c static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) · line 376

Quoted 10 lines
fs/io_uring.c +10 -0
+
+		ret = io_do_iopoll(ctx, nr_events, min);
+		if (ret < 0)
+			break;
+	} while (min && *nr_events < min);
+
+	if (ret < 0)
+		return ret;
+
+	return *nr_events < min;

The code looks a little clumsy to me. Why not:

	while (!list_empty(&ctx->poll_list)) {
		int ret = io_do_iopoll(ctx, nr_events, min);
		if (ret)
			return ret;

		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#43 lore raw

Re: [PATCH 07/18] io_uring: support for IO polling

Quoted 14 lines · Christoph Hellwig

fs/io_uring.c struct io_kiocb { · line 126

Quoted 9 lines
fs/io_uring.c +3 -0
@@ -118,12 +120,16 @@ struct io_kiocb {
 	struct list_head	list;
 	unsigned int		flags;
 #define REQ_F_FORCE_NONBLOCK	1	/* inline submission attempt */
+#define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
+#define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
 	u64			user_data;
+	u64			res;

Should this be ret or error instead? res is kinda off. A little comment describing it won't hurt either. Last but not least with the actual errno value stored here we probably don't need the REQ_F_IOPOLL_EAGAIN flag, do we?

Yes good point, that flag pre-dates us having the error in there. I'll rename the field, too.

Quoted 7 lines

fs/io_uring.c static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) · line 321

Quoted 5 lines
fs/io_uring.c +5 -0
+	/*
+	 * Only spin for completions if we don't have multiple devices hanging
+	 * off our complete list, and we're under the requested amount.
+	 */
+	spin = !ctx->poll_multi_file && (*nr_events < min);

no need for the braces here.

Killed

Quoted 21 lines
Quoted 8 lines
fs/io_uring.c +8 -0
+static int io_iopoll_getevents(struct io_ring_ctx *ctx, unsigned int *nr_events,
+				long min)
+{
+	int ret;
+
+	do {
+		if (list_empty(&ctx->poll_list))
+			return 0;

fs/io_uring.c static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx) · line 376

Quoted 10 lines
fs/io_uring.c +10 -0
+
+		ret = io_do_iopoll(ctx, nr_events, min);
+		if (ret < 0)
+			break;
+	} while (min && *nr_events < min);
+
+	if (ret < 0)
+		return ret;
+
+	return *nr_events < min;

The code looks a little clumsy to me. Why not:

Quoted 10 lines
	while (!list_empty(&ctx->poll_list)) {
		int ret = io_do_iopoll(ctx, nr_events, min);
		if (ret)
			return ret;

		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;

I think you messed up the 0/1 here, how about this:

	while (!list_empty(&ctx->poll_list)) {
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 1;
	}

	return 0;
Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#44 lore raw

Re: [PATCH 07/18] io_uring: support for IO polling

Quoted 15 lines · Jens Axboe
Quoted 12 lines

The code looks a little clumsy to me. Why not:

	while (!list_empty(&ctx->poll_list)) {
		int ret = io_do_iopoll(ctx, nr_events, min);
		if (ret)
			return ret;

		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;

I think you messed up the 0/1 here, how about this:

Quoted 11 lines
	while (!list_empty(&ctx->poll_list)) {
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 1;
	}

	return 0;

Or I did... I think yours is correct.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#45 lore raw

Re: [PATCH 07/18] io_uring: support for IO polling

Quoted 26 lines · Jens Axboe
Quoted 25 lines · Jens Axboe
Quoted 12 lines

The code looks a little clumsy to me. Why not:

	while (!list_empty(&ctx->poll_list)) {
		int ret = io_do_iopoll(ctx, nr_events, min);
		if (ret)
			return ret;

		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;

I think you messed up the 0/1 here, how about this:

	while (!list_empty(&ctx->poll_list)) {
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 1;
	}
Quoted 3 lines
Quoted 1 line
	return 0;

Or I did... I think yours is correct.

maybe document the return code? ;-)

-Jeff

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#46 lore raw

Re: [PATCH 07/18] io_uring: support for IO polling

Quoted 28 lines · Jeff Moyer
Quoted 26 lines · Jens Axboe
Quoted 25 lines · Jens Axboe
Quoted 12 lines

The code looks a little clumsy to me. Why not:

	while (!list_empty(&ctx->poll_list)) {
		int ret = io_do_iopoll(ctx, nr_events, min);
		if (ret)
			return ret;

		if (!min || *nr_events >= min)
			return 0;
	}

	return 1;

I think you messed up the 0/1 here, how about this:

	while (!list_empty(&ctx->poll_list)) {
		int ret;

		ret = io_do_iopoll(ctx, nr_events, min);
		if (ret < 0)
			return ret;
		if (!min || *nr_events >= min)
			return 1;
	}
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
	return 0;

Or I did... I think yours is correct.

maybe document the return code? ;-)

It is sort-of documented - <= 0 means "don't call me again", either because of an error (< 0) or because we have what we need. 1 means "I might have more".

But yes, maybe a comment...

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#47 lore raw

PATCH 8/18 fs: add fget_many() and fput_many()

In reply to Jens Axboe

Some uses cases repeatedly get and put references to the same file, but the only exposed interface is doing these one at the time. As each of these entail an atomic inc or dec on a shared structure, that cost can add up.

Add fget_many(), which works just like fget(), except it takes an argument for how many references to get on the file. Ditto fput_many(), which can drop an arbitrary number of references to a file.

---
 fs/file.c            | 15 ++++++++++-----
 fs/file_table.c      |  9 +++++++--
 include/linux/file.h |  2 ++
 include/linux/fs.h   |  4 +++-
 4 files changed, 22 insertions(+), 8 deletions(-)
fs/file.c +10 -5
index 3209ee271c41..97df385d6ab0 100644
@@ -705,7 +705,7 @@ void do_close_on_exec(struct files_struct *files)
 	spin_unlock(&files->file_lock);
 }
 
-static struct file *__fget(unsigned int fd, fmode_t mask)
+static struct file *__fget(unsigned int fd, fmode_t mask, unsigned int refs)
 {
 	struct files_struct *files = current->files;
 	struct file *file;
@@ -720,7 +720,7 @@ static struct file *__fget(unsigned int fd, fmode_t mask)
 		 */
 		if (file->f_mode & mask)
 			file = NULL;
-		else if (!get_file_rcu(file))
+		else if (!get_file_rcu_many(file, refs))
 			goto loop;
 	}
 	rcu_read_unlock();
@@ -728,15 +728,20 @@ static struct file *__fget(unsigned int fd, fmode_t mask)
 	return file;
 }
 
+struct file *fget_many(unsigned int fd, unsigned int refs)
+{
+	return __fget(fd, FMODE_PATH, refs);
+}
+
 struct file *fget(unsigned int fd)
 {
-	return __fget(fd, FMODE_PATH);
+	return __fget(fd, FMODE_PATH, 1);
 }
 EXPORT_SYMBOL(fget);
 
 struct file *fget_raw(unsigned int fd)
 {
-	return __fget(fd, 0);
+	return __fget(fd, 0, 1);
 }
 EXPORT_SYMBOL(fget_raw);
 
@@ -767,7 +772,7 @@ static unsigned long __fget_light(unsigned int fd, fmode_t mask)
 			return 0;
 		return (unsigned long)file;
 	} else {
-		file = __fget(fd, mask);
+		file = __fget(fd, mask, 1);
 		if (!file)
 			return 0;
 		return FDPUT_FPUT | (unsigned long)file;
fs/file_table.c +7 -2
index 5679e7fcb6b0..155d7514a094 100644
@@ -326,9 +326,9 @@ void flush_delayed_fput(void)
 
 static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput);
 
-void fput(struct file *file)
+void fput_many(struct file *file, unsigned int refs)
 {
-	if (atomic_long_dec_and_test(&file->f_count)) {
+	if (atomic_long_sub_and_test(refs, &file->f_count)) {
 		struct task_struct *task = current;
 
 		if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) {
@@ -347,6 +347,11 @@ void fput(struct file *file)
 	}
 }
 
+void fput(struct file *file)
+{
+	fput_many(file, 1);
+}
+
 /*
  * synchronous analog of fput(); for kernel threads that might be needed
  * in some umount() (and thus can't use flush_delayed_fput() without
include/linux/file.h +2 -0
index 6b2fb032416c..3fcddff56bc4 100644
@@ -13,6 +13,7 @@
 struct file;
 
 extern void fput(struct file *);
+extern void fput_many(struct file *, unsigned int);
 
 struct file_operations;
 struct vfsmount;
@@ -44,6 +45,7 @@ static inline void fdput(struct fd fd)
 }
 
 extern struct file *fget(unsigned int fd);
+extern struct file *fget_many(unsigned int fd, unsigned int refs);
 extern struct file *fget_raw(unsigned int fd);
 extern unsigned long __fdget(unsigned int fd);
 extern unsigned long __fdget_raw(unsigned int fd);
include/linux/fs.h +3 -1
index ccb0b7a63aa5..acaad78b6781 100644
@@ -952,7 +952,9 @@ static inline struct file *get_file(struct file *f)
 	atomic_long_inc(&f->f_count);
 	return f;
 }
-#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count)
+#define get_file_rcu_many(x, cnt)	\
+	atomic_long_add_unless(&(x)->f_count, (cnt), 0)
+#define get_file_rcu(x) get_file_rcu_many((x), 1)
 #define fput_atomic(x)	atomic_long_add_unless(&(x)->f_count, -1, 1)
 #define file_count(x)	atomic_long_read(&(x)->f_count)
 
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#48 lore raw

PATCH 9/18 io_uring: use fget/fput_many() for file references

In reply to Jens Axboe

Add a separate io_submit_state structure, to cache some of the things we need for IO submission.

One such example is file reference batching. io_submit_state. We get as many references as the number of sqes we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed.

On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap().

---
 fs/io_uring.c | 139 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 118 insertions(+), 21 deletions(-)
fs/io_uring.c +118 -21
index ed5b605a1748..25bdd719690d 100644
@@ -131,6 +131,19 @@ struct io_kiocb {
 #define IO_PLUG_THRESHOLD		2
 #define IO_IOPOLL_BATCH			8
 
+struct io_submit_state {
+	struct blk_plug plug;
+
+	/*
+	 * File reference cache
+	 */
+	struct file *file;
+	unsigned int fd;
+	unsigned int has_refs;
+	unsigned int used_refs;
+	unsigned int ios_left;
+};
+
 static struct kmem_cache *req_cachep;
 
 static const struct file_operations io_uring_fops;
@@ -284,9 +297,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
 			       struct list_head *done)
 {
 	void *reqs[IO_IOPOLL_BATCH];
+	int file_count, to_free;
+	struct file *file = NULL;
 	struct io_kiocb *req;
-	int to_free = 0;
 
+	file_count = to_free = 0;
 	while (!list_empty(done)) {
 		req = list_first_entry(done, struct io_kiocb, list);
 		list_del(&req->list);
@@ -296,12 +311,28 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
 		reqs[to_free++] = req;
 		(*nr_events)++;
 
-		fput(req->rw.ki_filp);
+		/*
+		 * Batched puts of the same file, to avoid dirtying the
+		 * file usage count multiple times, if avoidable.
+		 */
+		if (!file) {
+			file = req->rw.ki_filp;
+			file_count = 1;
+		} else if (file == req->rw.ki_filp) {
+			file_count++;
+		} else {
+			fput_many(file, file_count);
+			file = req->rw.ki_filp;
+			file_count = 1;
+		}
+
 		if (to_free == ARRAY_SIZE(reqs))
 			io_free_req_many(ctx, reqs, &to_free);
 	}
 	io_commit_cqring(ctx);
 
+	if (file)
+		fput_many(file, file_count);
 	if (to_free)
 		io_free_req_many(ctx, reqs, &to_free);
 }
@@ -490,14 +521,56 @@ static void io_iopoll_req_issued(struct io_kiocb *req)
 		list_add_tail(&req->list, &ctx->poll_list);
 }
 
+static void io_file_put(struct io_submit_state *state, struct file *file)
+{
+	if (!state) {
+		fput(file);
+	} else if (state->file) {
+		int diff = state->has_refs - state->used_refs;
+
+		if (diff)
+			fput_many(state->file, diff);
+		state->file = NULL;
+	}
+}
+
+/*
+ * Get as many references to a file as we have IOs left in this submission,
+ * assuming most submissions are for one file, or at least that each file
+ * has more than one submission.
+ */
+static struct file *io_file_get(struct io_submit_state *state, int fd)
+{
+	if (!state)
+		return fget(fd);
+
+	if (state->file) {
+		if (state->fd == fd) {
+			state->used_refs++;
+			state->ios_left--;
+			return state->file;
+		}
+		io_file_put(state, NULL);
+	}
+	state->file = fget_many(fd, state->ios_left);
+	if (!state->file)
+		return NULL;
+
+	state->fd = fd;
+	state->has_refs = state->ios_left;
+	state->used_refs = 1;
+	state->ios_left--;
+	return state->file;
+}
+
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
-		      bool force_nonblock)
+		      bool force_nonblock, struct io_submit_state *state)
 {
 	struct io_ring_ctx *ctx = req->ctx;
 	struct kiocb *kiocb = &req->rw;
 	int ret;
 
-	kiocb->ki_filp = fget(sqe->fd);
+	kiocb->ki_filp = io_file_get(state, sqe->fd);
 	if (unlikely(!kiocb->ki_filp))
 		return -EBADF;
 	kiocb->ki_pos = sqe->off;
@@ -536,7 +609,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	}
 	return 0;
 out_fput:
-	fput(kiocb->ki_filp);
+	io_file_put(state, kiocb->ki_filp);
 	return ret;
 }
 
@@ -577,7 +650,7 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
 }
 
 static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
-		       bool force_nonblock)
+		       bool force_nonblock, struct io_submit_state *state)
 {
 	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
 	struct kiocb *kiocb = &req->rw;
@@ -585,7 +658,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	struct file *file;
 	ssize_t ret;
 
-	ret = io_prep_rw(req, sqe, force_nonblock);
+	ret = io_prep_rw(req, sqe, force_nonblock, state);
 	if (ret)
 		return ret;
 	file = kiocb->ki_filp;
@@ -620,7 +693,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 }
 
 static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
-			bool force_nonblock)
+			bool force_nonblock, struct io_submit_state *state)
 {
 	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
 	struct kiocb *kiocb = &req->rw;
@@ -628,7 +701,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	struct file *file;
 	ssize_t ret;
 
-	ret = io_prep_rw(req, sqe, force_nonblock);
+	ret = io_prep_rw(req, sqe, force_nonblock, state);
 	if (ret)
 		return ret;
 	file = kiocb->ki_filp;
@@ -722,7 +795,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 }
 
 static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
-			   struct sqe_submit *s, bool force_nonblock)
+			   struct sqe_submit *s, bool force_nonblock,
+			   struct io_submit_state *state)
 {
 	const struct io_uring_sqe *sqe = s->sqe;
 	ssize_t ret;
@@ -737,10 +811,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 		ret = io_nop(req, sqe);
 		break;
 	case IORING_OP_READV:
-		ret = io_read(req, sqe, force_nonblock);
+		ret = io_read(req, sqe, force_nonblock, state);
 		break;
 	case IORING_OP_WRITEV:
-		ret = io_write(req, sqe, force_nonblock);
+		ret = io_write(req, sqe, force_nonblock, state);
 		break;
 	case IORING_OP_FSYNC:
 		ret = io_fsync(req, sqe, force_nonblock);
@@ -786,7 +860,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)
 	use_mm(ctx->sqo_mm);
 	set_fs(USER_DS);
 
-	ret = __io_submit_sqe(ctx, req, s, false);
+	ret = __io_submit_sqe(ctx, req, s, false, NULL);
 
 	set_fs(old_fs);
 	unuse_mm(ctx->sqo_mm);
@@ -799,7 +873,8 @@ static void io_sq_wq_submit_work(struct work_struct *work)
 	current->files = old_files;
 }
 
-static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s)
+static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
+			 struct io_submit_state *state)
 {
 	struct io_kiocb *req;
 	ssize_t ret;
@@ -812,7 +887,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s)
 	if (unlikely(!req))
 		return -EAGAIN;
 
-	ret = __io_submit_sqe(ctx, req, s, true);
+	ret = __io_submit_sqe(ctx, req, s, true, state);
 	if (ret == -EAGAIN) {
 		memcpy(&req->submit, s, sizeof(*s));
 		INIT_WORK(&req->work, io_sq_wq_submit_work);
@@ -825,6 +900,26 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s)
 	return ret;
 }
 
+/*
+ * Batched submission is done, ensure local IO is flushed out.
+ */
+static void io_submit_state_end(struct io_submit_state *state)
+{
+	blk_finish_plug(&state->plug);
+	io_file_put(state, NULL);
+}
+
+/*
+ * Start submission side cache.
+ */
+static void io_submit_state_start(struct io_submit_state *state,
+				  struct io_ring_ctx *ctx, unsigned max_ios)
+{
+	blk_start_plug(&state->plug);
+	state->file = NULL;
+	state->ios_left = max_ios;
+}
+
 static void io_commit_sqring(struct io_ring_ctx *ctx)
 {
 	struct io_sq_ring *ring = ctx->sq_ring;
@@ -879,11 +974,13 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 
 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 {
+	struct io_submit_state state, *statep = NULL;
 	int i, ret = 0, submit = 0;
-	struct blk_plug plug;
 
-	if (to_submit > IO_PLUG_THRESHOLD)
-		blk_start_plug(&plug);
+	if (to_submit > IO_PLUG_THRESHOLD) {
+		io_submit_state_start(&state, ctx, to_submit);
+		statep = &state;
+	}
 
 	for (i = 0; i < to_submit; i++) {
 		struct sqe_submit s;
@@ -891,7 +988,7 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 		if (!io_get_sqring(ctx, &s))
 			break;
 
-		ret = io_submit_sqe(ctx, &s);
+		ret = io_submit_sqe(ctx, &s, statep);
 		if (ret) {
 			io_drop_sqring(ctx);
 			break;
@@ -901,8 +998,8 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 	}
 	io_commit_sqring(ctx);
 
-	if (to_submit > IO_PLUG_THRESHOLD)
-		blk_finish_plug(&plug);
+	if (statep)
+		io_submit_state_end(statep);
 
 	return submit ? submit : ret;
 }
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#49 lore raw

Re: [PATCH 09/18] io_uring: use fget/fput_many() for file references

Quoted 15 lines · Jens Axboe

Add a separate io_submit_state structure, to cache some of the things we need for IO submission.

One such example is file reference batching. io_submit_state. We get as many references as the number of sqes we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed.

On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap().

---

[...]

Quoted 11 lines
fs/io_uring.c +11 -0
+/*
+ * Get as many references to a file as we have IOs left in this submission,
+ * assuming most submissions are for one file, or at least that each file
+ * has more than one submission.
+ */
+static struct file *io_file_get(struct io_submit_state *state, int fd)
+{
+       if (!state)
+               return fget(fd);
+
+       if (state->file) {

fs/io_uring.c static void io_iopoll_req_issued(struct io_kiocb *req) · line 557

Quoted 10 lines
fs/io_uring.c +10 -0
+               if (state->fd == fd) {
+                       state->used_refs++;
+                       state->ios_left--;
+                       return state->file;
+               }
+               io_file_put(state, NULL);
+       }
+       state->file = fget_many(fd, state->ios_left);
+       if (!state->file)
+               return NULL;

This looks wrong.

Looking at "[PATCH 05/18] Add io_uring IO interface", as far as I can tell, io_ring_submit() is called via __io_uring_enter() <- sys_io_uring_enter() with an unchecked argument "unsigned int to_submit" that is then, in this patch, stored in state->ios_left and then used here. On a 32-bit platform, file->f_count is only 32 bits wide, so I think you can then trivially overflow the reference count, leading to use-after-free. Am I missing something?

Quoted 6 lines
fs/io_uring.c +6 -0
+       state->fd = fd;
+       state->has_refs = state->ios_left;
+       state->used_refs = 1;
+       state->ios_left--;
+       return state->file;
+}

[...]

Quoted 15 lines
fs/io_uring.c +8 -0
+static void io_submit_state_start(struct io_submit_state *state,
+                                 struct io_ring_ctx *ctx, unsigned max_ios)
+{
+       blk_start_plug(&state->plug);
+       state->file = NULL;
+       state->ios_left = max_ios;
+}
+
 static void io_commit_sqring(struct io_ring_ctx *ctx)
 {
        struct io_sq_ring *ring = ctx->sq_ring;
@@ -879,11 +974,13 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 
 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 {

fs/io_uring.c static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s) · line 983

Quoted 10 lines
fs/io_uring.c +5 -3
+       struct io_submit_state state, *statep = NULL;
        int i, ret = 0, submit = 0;
-       struct blk_plug plug;
 
-       if (to_submit > IO_PLUG_THRESHOLD)
-               blk_start_plug(&plug);
+       if (to_submit > IO_PLUG_THRESHOLD) {
+               io_submit_state_start(&state, ctx, to_submit);
+               statep = &state;
+       }

[...]

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#50 lore raw

Re: [PATCH 09/18] io_uring: use fget/fput_many() for file references

Quoted 41 lines · Jann Horn
Quoted 15 lines · Jens Axboe

Add a separate io_submit_state structure, to cache some of the things we need for IO submission.

One such example is file reference batching. io_submit_state. We get as many references as the number of sqes we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed.

On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap().

---

[...]

Quoted 11 lines
fs/io_uring.c +11 -0
+/*
+ * Get as many references to a file as we have IOs left in this submission,
+ * assuming most submissions are for one file, or at least that each file
+ * has more than one submission.
+ */
+static struct file *io_file_get(struct io_submit_state *state, int fd)
+{
+       if (!state)
+               return fget(fd);
+
+       if (state->file) {

fs/io_uring.c static void io_iopoll_req_issued(struct io_kiocb *req) · line 557

Quoted 10 lines
fs/io_uring.c +10 -0
+               if (state->fd == fd) {
+                       state->used_refs++;
+                       state->ios_left--;
+                       return state->file;
+               }
+               io_file_put(state, NULL);
+       }
+       state->file = fget_many(fd, state->ios_left);
+       if (!state->file)
+               return NULL;

This looks wrong.

Quoted 7 lines

Looking at "[PATCH 05/18] Add io_uring IO interface", as far as I can tell, io_ring_submit() is called via __io_uring_enter() <- sys_io_uring_enter() with an unchecked argument "unsigned int to_submit" that is then, in this patch, stored in state->ios_left and then used here. On a 32-bit platform, file->f_count is only 32 bits wide, so I think you can then trivially overflow the reference count, leading to use-after-free. Am I missing something?

No, that is possible. Since we cap the SQ ring entries at 4k, I think we should just validate to_submit/min_complete against those numbers. That would also solve this overflow.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#51 lore raw

PATCH 10/18 io_uring: batch io_kiocb allocation

In reply to Jens Axboe

Similarly to how we use the state->ios_left to know how many references to get to a file, we can use it to allocate the io_kiocb's we need in bulk.

---
 fs/io_uring.c | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)
fs/io_uring.c +36 -3
index 25bdd719690d..a33f1b1709d0 100644
@@ -134,6 +134,13 @@ struct io_kiocb {
 struct io_submit_state {
 	struct blk_plug plug;
 
+	/*
+	 * io_kiocb alloc cache
+	 */
+	void *reqs[IO_IOPOLL_BATCH];
+	unsigned int free_reqs;
+	unsigned int cur_req;
+
 	/*
 	 * File reference cache
 	 */
@@ -257,20 +264,42 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
 		wake_up(&ctx->wait);
 }
 
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+				   struct io_submit_state *state)
 {
+	gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;
 	struct io_kiocb *req;
 
 	/* safe to use the non tryget, as we're inside ring ref already */
 	percpu_ref_get(&ctx->refs);
 
-	req = kmem_cache_alloc(req_cachep, GFP_ATOMIC | __GFP_NOWARN);
+	if (!state)
+		req = kmem_cache_alloc(req_cachep, gfp);
+	else if (!state->free_reqs) {
+		size_t sz;
+		int ret;
+
+		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz,
+						state->reqs);
+		if (ret <= 0)
+			goto out;
+		state->free_reqs = ret - 1;
+		state->cur_req = 1;
+		req = state->reqs[0];
+	} else {
+		req = state->reqs[state->cur_req];
+		state->free_reqs--;
+		state->cur_req++;
+	}
+
 	if (req) {
 		req->ctx = ctx;
 		req->flags = 0;
 		return req;
 	}
 
+out:
 	io_ring_drop_ctx_refs(ctx, 1);
 	return NULL;
 }
@@ -883,7 +912,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 	if (unlikely(s->sqe->flags))
 		return -EINVAL;
 
-	req = io_get_req(ctx);
+	req = io_get_req(ctx, state);
 	if (unlikely(!req))
 		return -EAGAIN;
 
@@ -907,6 +936,9 @@ static void io_submit_state_end(struct io_submit_state *state)
 {
 	blk_finish_plug(&state->plug);
 	io_file_put(state, NULL);
+	if (state->free_reqs)
+		kmem_cache_free_bulk(req_cachep, state->free_reqs,
+					&state->reqs[state->cur_req]);
 }
 
 /*
@@ -916,6 +948,7 @@ static void io_submit_state_start(struct io_submit_state *state,
 				  struct io_ring_ctx *ctx, unsigned max_ios)
 {
 	blk_start_plug(&state->plug);
+	state->free_reqs = 0;
 	state->file = NULL;
 	state->ios_left = max_ios;
 }
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#52 lore raw

Re: [PATCH 10/18] io_uring: batch io_kiocb allocation

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 270

Quoted 5 lines
fs/io_uring.c +3 -1
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+				   struct io_submit_state *state)
 {
+	gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;

No actually in this patch, but why do we do GFP_ATOMIC allocations here? We aren't in irq context or under a spinlock.

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 277

Quoted 2 lines
fs/io_uring.c +2 -0
+	if (!state)
+		req = kmem_cache_alloc(req_cachep, gfp);

Add a

		if (!req)
			goto out;

plus the missing braces here..

Quoted 13 lines
fs/io_uring.c +13 -0
+	else if (!state->free_reqs) {
+		size_t sz;
+		int ret;
+
+		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz,
+						state->reqs);
+		if (ret <= 0)
+			goto out;
+		state->free_reqs = ret - 1;
+		state->cur_req = 1;
+		req = state->reqs[0];
+	} else {

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 300

Quoted 10 lines
fs/io_uring.c +5 -0
+		req = state->reqs[state->cur_req];
+		state->free_reqs--;
+		state->cur_req++;
+	}
+
 	if (req) {
 		req->ctx = ctx;
 		req->flags = 0;
 		return req;
 	}

... and we don't need this conditional, which would otherwise also be in the fast path.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#53 lore raw

Re: [PATCH 10/18] io_uring: batch io_kiocb allocation

Quoted 8 lines · Christoph Hellwig

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 270

Quoted 5 lines
fs/io_uring.c +3 -1
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+				   struct io_submit_state *state)
 {
+	gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN;

No actually in this patch, but why do we do GFP_ATOMIC allocations here? We aren't in irq context or under a spinlock.

Just to avoid dipping into more expensive allocs. Probably not a big deal, I'll remove the atomic.

Quoted 28 lines

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 277

Quoted 3 lines
fs/io_uring.c +2 -0
+	if (!state)
+		req = kmem_cache_alloc(req_cachep, gfp);

Add a

		if (!req)
			goto out;

plus the missing braces here..

Quoted 18 lines
fs/io_uring.c +18 -0
+	else if (!state->free_reqs) {
+		size_t sz;
+		int ret;
+
+		sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+		ret = kmem_cache_alloc_bulk(req_cachep, gfp, sz,
+						state->reqs);
+		if (ret <= 0)
+			goto out;
+		state->free_reqs = ret - 1;
+		state->cur_req = 1;
+		req = state->reqs[0];
+	} else {
+		req = state->reqs[state->cur_req];
+		state->free_reqs--;
+		state->cur_req++;
+	}
+
Quoted 8 lines

fs/io_uring.c static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs) · line 300

Quoted 5 lines
fs/io_uring.c +0 -0
 	if (req) {
 		req->ctx = ctx;
 		req->flags = 0;
 		return req;
 	}

... and we don't need this conditional, which would otherwise also be in the fast path.

Good idea, done.

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#54 lore raw

PATCH 11/18 block: implement bio helper to add iter bvec pages to bio

In reply to Jens Axboe

For an ITER_BVEC, we can just iterate the iov and add the pages to the bio directly. This requires that the caller doesn't releases the pages on IO completion, we add a BIO_NO_PAGE_REF flag for that.

The current two callers of bio_iov_iter_get_pages() are updated to check if they need to release pages on completion. This makes them work with bvecs that contain kernel mapped pages already.

---
 block/bio.c               | 59 ++++++++++++++++++++++++++++++++-------
 fs/block_dev.c            |  5 ++--
 fs/iomap.c                |  5 ++--
 include/linux/blk_types.h |  1 +
 4 files changed, 56 insertions(+), 14 deletions(-)
block/bio.c +49 -10
index 4db1008309ed..330df572cfb8 100644
@@ -828,6 +828,23 @@ int bio_add_page(struct bio *bio, struct page *page,
 }
 EXPORT_SYMBOL(bio_add_page);
 
+static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
+{
+	const struct bio_vec *bv = iter->bvec;
+	unsigned int len;
+	size_t size;
+
+	len = min_t(size_t, bv->bv_len, iter->count);
+	size = bio_add_page(bio, bv->bv_page, len,
+				bv->bv_offset + iter->iov_offset);
+	if (size == len) {
+		iov_iter_advance(iter, size);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
 #define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
 
 /**
@@ -876,23 +893,43 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 }
 
 /**
- * bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
+ * bio_iov_iter_get_pages - add user or kernel pages to a bio
  * @bio: bio to add pages to
- * @iter: iov iterator describing the region to be mapped
+ * @iter: iov iterator describing the region to be added
+ *
+ * This takes either an iterator pointing to user memory, or one pointing to
+ * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
+ * map them into the kernel. On IO completion, the caller should put those
+ * pages. If we're adding kernel pages, we just have to add the pages to the
+ * bio directly. We don't grab an extra reference to those pages (the user
+ * should already have that), and we don't put the page on IO completion.
+ * The caller needs to check if the bio is flagged BIO_NO_PAGE_REF on IO
+ * completion. If it isn't, then pages should be released.
  *
- * Pins pages from *iter and appends them to @bio's bvec array. The
- * pages will have to be released using put_page() when done.
  * The function tries, but does not guarantee, to pin as many pages as
- * fit into the bio, or are requested in *iter, whatever is smaller.
- * If MM encounters an error pinning the requested pages, it stops.
- * Error is returned only if 0 pages could be pinned.
+ * fit into the bio, or are requested in *iter, whatever is smaller. If
+ * MM encounters an error pinning the requested pages, it stops. Error
+ * is returned only if 0 pages could be pinned.
  */
 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 {
+	const bool is_bvec = iov_iter_is_bvec(iter);
 	unsigned short orig_vcnt = bio->bi_vcnt;
 
+	/*
+	 * If this is a BVEC iter, then the pages are kernel pages. Don't
+	 * release them on IO completion.
+	 */
+	if (is_bvec)
+		bio_set_flag(bio, BIO_NO_PAGE_REF);
+
 	do {
-		int ret = __bio_iov_iter_get_pages(bio, iter);
+		int ret;
+
+		if (is_bvec)
+			ret = __bio_iov_bvec_add_pages(bio, iter);
+		else
+			ret = __bio_iov_iter_get_pages(bio, iter);
 
 		if (unlikely(ret))
 			return bio->bi_vcnt > orig_vcnt ? 0 : ret;
@@ -1634,7 +1671,8 @@ static void bio_dirty_fn(struct work_struct *work)
 		next = bio->bi_private;
 
 		bio_set_pages_dirty(bio);
-		bio_release_pages(bio);
+		if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+			bio_release_pages(bio);
 		bio_put(bio);
 	}
 }
@@ -1650,7 +1688,8 @@ void bio_check_pages_dirty(struct bio *bio)
 			goto defer;
 	}
 
-	bio_release_pages(bio);
+	if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+		bio_release_pages(bio);
 	bio_put(bio);
 	return;
 defer:
fs/block_dev.c +3 -2
index 392e2bfb636f..051ab41d1c61 100644
@@ -338,8 +338,9 @@ static void blkdev_bio_end_io(struct bio *bio)
 		struct bio_vec *bvec;
 		int i;
 
-		bio_for_each_segment_all(bvec, bio, i)
-			put_page(bvec->bv_page);
+		if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+			bio_for_each_segment_all(bvec, bio, i)
+				put_page(bvec->bv_page);
 		bio_put(bio);
 	}
 }
fs/iomap.c +3 -2
index 4ee50b76b4a1..e5c48a0b20e0 100644
@@ -1582,8 +1582,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
 		struct bio_vec *bvec;
 		int i;
 
-		bio_for_each_segment_all(bvec, bio, i)
-			put_page(bvec->bv_page);
+		if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+			bio_for_each_segment_all(bvec, bio, i)
+				put_page(bvec->bv_page);
 		bio_put(bio);
 	}
 }
include/linux/blk_types.h +1 -0
index d66bf5f32610..791fee35df88 100644
@@ -215,6 +215,7 @@ struct bio {
 /*
  * bio flags
  */
+#define BIO_NO_PAGE_REF	0	/* don't put release vec pages */
 #define BIO_SEG_VALID	1	/* bi_phys_segments valid */
 #define BIO_CLONED	2	/* doesn't own data */
 #define BIO_BOUNCED	3	/* bio is a bounce bio */
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#55 lore raw

PATCH 12/18 io_uring: add support for pre-mapped user IO buffers

In reply to Jens Axboe

If we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.

To utilize this feature, the application must call io_uring_register() after having setup an io_uring context, passing in IORING_REGISTER_BUFFERS as the opcode. The argument must be a pointer to an iovec array, and the nr_args should contain how many iovecs the application wishes to map.

If successful, these buffers are now mapped into the kernel, eligible for IO. To use these fixed buffers, the application must use the IORING_OP_READ_FIXED and IORING_OP_WRITE_FIXED opcodes, and then set sqe->index to the desired buffer index. sqe->addr..sqe->addr+seq->len must point to somewhere inside the indexed buffer.

The application may register buffers throughout the lifetime of the io_uring context. It can call io_uring_register() with IORING_UNREGISTER_BUFFERS as the opcode to unregister the current set of buffers, and then register a new set. The application need not unregister buffers explicitly before shutting down the io_uring context.

It's perfectly valid to setup a larger buffer, and then sometimes only use parts of it for an IO. As long as the range is within the originally mapped region, it will work just fine.

For now, buffers must not be file backed. If file backed buffers are passed in, the registration will fail with -1/EOPNOTSUPP. This restriction may be relaxed in the future.

RLIMIT_MEMLOCK is used to check how much memory we can pin. A somewhat arbitrary 1G per buffer size is also imposed.

---
 arch/x86/entry/syscalls/syscall_32.tbl |   1 +
 arch/x86/entry/syscalls/syscall_64.tbl |   1 +
 fs/io_uring.c                          | 356 ++++++++++++++++++++++++-
 include/linux/sched/user.h             |   2 +-
 include/linux/syscalls.h               |   2 +
 include/uapi/asm-generic/unistd.h      |   4 +-
 include/uapi/linux/io_uring.h          |  13 +-
 kernel/sys_ni.c                        |   1 +
 8 files changed, 366 insertions(+), 14 deletions(-)
arch/x86/entry/syscalls/syscall_32.tbl +1 -0
index 481c126259e9..2eefd2a7c1ce 100644
@@ -400,3 +400,4 @@
 386	i386	rseq			sys_rseq			__ia32_sys_rseq
 425	i386	io_uring_setup		sys_io_uring_setup		__ia32_sys_io_uring_setup
 426	i386	io_uring_enter		sys_io_uring_enter		__ia32_sys_io_uring_enter
+427	i386	io_uring_register	sys_io_uring_register		__ia32_sys_io_uring_register
arch/x86/entry/syscalls/syscall_64.tbl +1 -0
index 6a32a430c8e0..65c026185e61 100644
@@ -345,6 +345,7 @@
 334	common	rseq			__x64_sys_rseq
 425	common	io_uring_setup		__x64_sys_io_uring_setup
 426	common	io_uring_enter		__x64_sys_io_uring_enter
+427	common	io_uring_register	__x64_sys_io_uring_register
 
 #
 # x32-specific system call numbers start at 512 to avoid cache impact
fs/io_uring.c +345 -11

448 lines not on this page · raw message

include/linux/sched/user.h +1 -1
index 39ad98c09c58..c7b5f86b91a1 100644
@@ -40,7 +40,7 @@ struct user_struct {
 	kuid_t uid;
 
 #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \
-    defined(CONFIG_NET)
+    defined(CONFIG_NET) || defined(CONFIG_IO_URING)
 	atomic_long_t locked_vm;
 #endif
 
include/linux/syscalls.h +2 -0
index 3072dbaa7869..3681c05ac538 100644
@@ -315,6 +315,8 @@ asmlinkage long sys_io_uring_setup(u32 entries,
 asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit,
 				u32 min_complete, u32 flags,
 				const sigset_t __user *sig, size_t sigsz);
+asmlinkage long sys_io_uring_register(unsigned int fd, unsigned int op,
+				void __user *arg, unsigned int nr_args);
 
 /* fs/xattr.c */
 asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
include/uapi/asm-generic/unistd.h +3 -1
index 87871e7b7ea7..d346229a1eb0 100644
@@ -744,9 +744,11 @@ __SYSCALL(__NR_kexec_file_load,     sys_kexec_file_load)
 __SYSCALL(__NR_io_uring_setup, sys_io_uring_setup)
 #define __NR_io_uring_enter 426
 __SYSCALL(__NR_io_uring_enter, sys_io_uring_enter)
+#define __NR_io_uring_register 427
+__SYSCALL(__NR_io_uring_register, sys_io_uring_register)
 
 #undef __NR_syscalls
-#define __NR_syscalls 427
+#define __NR_syscalls 428
 
 /*
  * 32 bit systems traditionally used different
include/uapi/linux/io_uring.h +12 -1
index 4fc5fbd07688..03ce7133c3b2 100644
@@ -29,7 +29,10 @@ struct io_uring_sqe {
 		__u32		fsync_flags;
 	};
 	__u64	user_data;	/* data to be passed back at completion time */
-	__u64	__pad2[3];
+	union {
+		__u16	buf_index;	/* index into fixed buffers, if used */
+		__u64	__pad2[3];
+	};
 };
 
 /*
@@ -41,6 +44,8 @@ struct io_uring_sqe {
 #define IORING_OP_READV		1
 #define IORING_OP_WRITEV	2
 #define IORING_OP_FSYNC		3
+#define IORING_OP_READ_FIXED	4
+#define IORING_OP_WRITE_FIXED	5
 
 /*
  * sqe->fsync_flags
@@ -104,4 +109,10 @@ struct io_uring_params {
 	struct io_cqring_offsets cq_off;
 };
 
+/*
+ * io_uring_register(2) opcodes and arguments
+ */
+#define IORING_REGISTER_BUFFERS		0
+#define IORING_UNREGISTER_BUFFERS	1
+
 #endif
kernel/sys_ni.c +1 -0
index ee5e523564bb..1bb6604dc19f 100644
@@ -48,6 +48,7 @@ COND_SYSCALL_COMPAT(io_getevents);
 COND_SYSCALL_COMPAT(io_pgetevents);
 COND_SYSCALL(io_uring_setup);
 COND_SYSCALL(io_uring_enter);
+COND_SYSCALL(io_uring_register);
 
 /* fs/xattr.c */
 
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#56 lore raw

Re: [PATCH 12/18] io_uring: add support for pre-mapped user IO buffers

Quoted 3 lines · Jens Axboe

If we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.

[...]

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1766

Quoted 7 lines
fs/io_uring.c +7 -0
+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
+                              void __user *arg, unsigned nr_args)
+{
+       int ret;
+
+       /* Drop our initial ref and wait for the ctx to be fully idle */
+       percpu_ref_put(&ctx->refs);

The line above drops a reference that you just got in the caller...

Quoted 12 lines
fs/io_uring.c +12 -0
+       percpu_ref_kill(&ctx->refs);
+       wait_for_completion(&ctx->ctx_done);
+
+       switch (opcode) {
+       case IORING_REGISTER_BUFFERS:
+               ret = io_sqe_buffer_register(ctx, arg, nr_args);
+               break;
+       case IORING_UNREGISTER_BUFFERS:
+               ret = -EINVAL;
+               if (arg || nr_args)
+                       break;
+               ret = io_sqe_buffer_unregister(ctx);

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1788

Quoted 10 lines
fs/io_uring.c +10 -0
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+
+       /* bring the ctx back to life */
+       reinit_completion(&ctx->ctx_done);
+       percpu_ref_resurrect(&ctx->refs);
+       percpu_ref_get(&ctx->refs);

And then this line takes a reference that the caller will immediately drop again? Why?

Quoted 12 lines
fs/io_uring.c +12 -0
+       return ret;
+}
+
+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
+               void __user *, arg, unsigned int, nr_args)
+{
+       struct io_ring_ctx *ctx;
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1810

Quoted 10 lines
fs/io_uring.c +10 -0
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
+               goto out_fput;
+
+       ret = -ENXIO;
+       ctx = f.file->private_data;
+       if (!percpu_ref_tryget(&ctx->refs))
+               goto out_fput;

If you are holding the uring_lock of a ctx that can be accessed through a file descriptor (which you do just after this point), you know that the percpu_ref isn't zero, right? Why are you doing the tryget here?

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1821

Quoted 10 lines
fs/io_uring.c +10 -0
+       ret = -EBUSY;
+       if (mutex_trylock(&ctx->uring_lock)) {
+               ret = __io_uring_register(ctx, opcode, arg, nr_args);
+               mutex_unlock(&ctx->uring_lock);
+       }
+       io_ring_drop_ctx_refs(ctx, 1);
+out_fput:
+       fdput(f);
+       return ret;
+}

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#57 lore raw

Re: [PATCH 12/18] io_uring: add support for pre-mapped user IO buffers

Quoted 5 lines · Jann Horn
Quoted 3 lines · Jens Axboe

If we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.

[...]

Quoted 9 lines

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1766

Quoted 7 lines
fs/io_uring.c +7 -0
+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
+                              void __user *arg, unsigned nr_args)
+{
+       int ret;
+
+       /* Drop our initial ref and wait for the ctx to be fully idle */
+       percpu_ref_put(&ctx->refs);

The line above drops a reference that you just got in the caller...

Right

Quoted 17 lines
Quoted 17 lines
fs/io_uring.c +17 -0
+       percpu_ref_kill(&ctx->refs);
+       wait_for_completion(&ctx->ctx_done);
+
+       switch (opcode) {
+       case IORING_REGISTER_BUFFERS:
+               ret = io_sqe_buffer_register(ctx, arg, nr_args);
+               break;
+       case IORING_UNREGISTER_BUFFERS:
+               ret = -EINVAL;
+               if (arg || nr_args)
+                       break;
+               ret = io_sqe_buffer_unregister(ctx);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
Quoted 8 lines

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1788

Quoted 5 lines
fs/io_uring.c +5 -0
+
+       /* bring the ctx back to life */
+       reinit_completion(&ctx->ctx_done);
+       percpu_ref_resurrect(&ctx->refs);
+       percpu_ref_get(&ctx->refs);

And then this line takes a reference that the caller will immediately drop again? Why?

Just want to keep it symmetric and avoid having weird "this function drops a reference" use cases.

Quoted 20 lines
Quoted 20 lines
fs/io_uring.c +19 -0
+       return ret;
+}
+
+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
+               void __user *, arg, unsigned int, nr_args)
+{
+       struct io_ring_ctx *ctx;
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
+               goto out_fput;
+
+       ret = -ENXIO;
Quoted 8 lines

fs/io_uring.c SYSCALL_DEFINE2(io_uring_setup, u32, entries, · line 1810

Quoted 3 lines
fs/io_uring.c +3 -0
+       ctx = f.file->private_data;
+       if (!percpu_ref_tryget(&ctx->refs))
+               goto out_fput;

If you are holding the uring_lock of a ctx that can be accessed through a file descriptor (which you do just after this point), you know that the percpu_ref isn't zero, right? Why are you doing the tryget here?

Not sure I follow... We don't hold the lock at this point. I guess your point is that since the descriptor is open (or we'd fail the above check), then there's no point doing the tryget variant here? That's strictly true, that could just be a get().

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#58 lore raw

Re: [PATCH 12/18] io_uring: add support for pre-mapped user IO buffers

Quoted 76 lines · Jens Axboe
Quoted 5 lines · Jann Horn
Quoted 3 lines · Jens Axboe

If we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.

[...]

Quoted 9 lines

Review

Quoted 7 lines
(quoted diff) +7 -0
+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
+                              void __user *arg, unsigned nr_args)
+{
+       int ret;
+
+       /* Drop our initial ref and wait for the ctx to be fully idle */
+       percpu_ref_put(&ctx->refs);

The line above drops a reference that you just got in the caller...

Right

Quoted 17 lines
Quoted 17 lines
(quoted diff) +17 -0
+       percpu_ref_kill(&ctx->refs);
+       wait_for_completion(&ctx->ctx_done);
+
+       switch (opcode) {
+       case IORING_REGISTER_BUFFERS:
+               ret = io_sqe_buffer_register(ctx, arg, nr_args);
+               break;
+       case IORING_UNREGISTER_BUFFERS:
+               ret = -EINVAL;
+               if (arg || nr_args)
+                       break;
+               ret = io_sqe_buffer_unregister(ctx);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+
+       /* bring the ctx back to life */
+       reinit_completion(&ctx->ctx_done);
+       percpu_ref_resurrect(&ctx->refs);
+       percpu_ref_get(&ctx->refs);

And then this line takes a reference that the caller will immediately drop again? Why?

Just want to keep it symmetric and avoid having weird "this function drops a reference" use cases.

Quoted 20 lines
Quoted 20 lines
(quoted diff) +19 -0
+       return ret;
+}
+
+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
+               void __user *, arg, unsigned int, nr_args)
+{
+       struct io_ring_ctx *ctx;
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
+               goto out_fput;
+
+       ret = -ENXIO;
Quoted 8 lines

Review

Quoted 3 lines
(quoted diff) +3 -0
+       ctx = f.file->private_data;
+       if (!percpu_ref_tryget(&ctx->refs))
+               goto out_fput;

If you are holding the uring_lock of a ctx that can be accessed through a file descriptor (which you do just after this point), you know that the percpu_ref isn't zero, right? Why are you doing the tryget here?

Quoted 4 lines

Not sure I follow... We don't hold the lock at this point. I guess your point is that since the descriptor is open (or we'd fail the above check), then there's no point doing the tryget variant here? That's strictly true, that could just be a get().

As far as I can tell, you could do the following without breaking anything:

========================

fs/io_uring.c +0 -0
index 6916dc3222cf..c2d82765eefe 100644
@@ -2485,7 +2485,6 @@ static int __io_uring_register(struct

io_ring_ctx *ctx, unsigned opcode,

        int ret;

        /* Drop our initial ref and wait for the ctx to be fully idle */
-       percpu_ref_put(&ctx->refs);
        percpu_ref_kill(&ctx->refs);
        wait_for_completion(&ctx->ctx_done);

@@ -2516,7 +2515,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,

        /* bring the ctx back to life */
        reinit_completion(&ctx->ctx_done);
        percpu_ref_resurrect(&ctx->refs);
-       percpu_ref_get(&ctx->refs);
        return ret;

}

@@ -2535,17 +2533,13 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,

        if (f.file->f_op != &io_uring_fops)
                goto out_fput;
-       ret = -ENXIO;
        ctx = f.file->private_data;

- if (!percpu_ref_tryget(&ctx->refs))
- goto out_fput;

        ret = -EBUSY;
        if (mutex_trylock(&ctx->uring_lock)) {
                ret = __io_uring_register(ctx, opcode, arg, nr_args);
                mutex_unlock(&ctx->uring_lock);
        }
-       io_ring_drop_ctx_refs(ctx, 1);
 out_fput:
        fdput(f);
        return ret;

========================

The two functions that can drop the initial ref of the percpu refcount are:

1. io_ring_ctx_wait_and_kill(); this is only used on ->release() or on setup failure, meaning that as long as you have a reference to the file from fget()/fdget(), io_ring_ctx_wait_and_kill() can't have been called on your context
2. __io_uring_register(); this temporarily kills the percpu refcount and resurrects it, all under ctx->uring_lock, meaning that as long as you're holding ctx->uring_lock, __io_uring_register() can't have killed the percpu refcount

Therefore, I think that as long as you're in sys_io_uring_register and holding the ctx->uring_lock, you know that the percpu refcount is alive, and bumping and dropping non-initial references has no effect.

Perhaps this makes more sense when you view the percpu refcount as a read/write lock - percpu_ref_tryget() takes a read lock, the percpu_ref_kill() dance takes a write lock.

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#59 lore raw

Re: [PATCH 12/18] io_uring: add support for pre-mapped user IO buffers

Quoted 139 lines · Jann Horn
Quoted 76 lines · Jens Axboe
Quoted 5 lines · Jann Horn
Quoted 3 lines · Jens Axboe

If we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.

[...]

Quoted 9 lines

Review

Quoted 7 lines
(quoted diff) +7 -0
+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
+                              void __user *arg, unsigned nr_args)
+{
+       int ret;
+
+       /* Drop our initial ref and wait for the ctx to be fully idle */
+       percpu_ref_put(&ctx->refs);

The line above drops a reference that you just got in the caller...

Right

Quoted 17 lines
Quoted 17 lines
(quoted diff) +17 -0
+       percpu_ref_kill(&ctx->refs);
+       wait_for_completion(&ctx->ctx_done);
+
+       switch (opcode) {
+       case IORING_REGISTER_BUFFERS:
+               ret = io_sqe_buffer_register(ctx, arg, nr_args);
+               break;
+       case IORING_UNREGISTER_BUFFERS:
+               ret = -EINVAL;
+               if (arg || nr_args)
+                       break;
+               ret = io_sqe_buffer_unregister(ctx);
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+
+       /* bring the ctx back to life */
+       reinit_completion(&ctx->ctx_done);
+       percpu_ref_resurrect(&ctx->refs);
+       percpu_ref_get(&ctx->refs);

And then this line takes a reference that the caller will immediately drop again? Why?

Just want to keep it symmetric and avoid having weird "this function drops a reference" use cases.

Quoted 20 lines
Quoted 20 lines
(quoted diff) +19 -0
+       return ret;
+}
+
+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
+               void __user *, arg, unsigned int, nr_args)
+{
+       struct io_ring_ctx *ctx;
+       long ret = -EBADF;
+       struct fd f;
+
+       f = fdget(fd);
+       if (!f.file)
+               return -EBADF;
+
+       ret = -EOPNOTSUPP;
+       if (f.file->f_op != &io_uring_fops)
+               goto out_fput;
+
+       ret = -ENXIO;
Quoted 8 lines

Review

Quoted 3 lines
(quoted diff) +3 -0
+       ctx = f.file->private_data;
+       if (!percpu_ref_tryget(&ctx->refs))
+               goto out_fput;

If you are holding the uring_lock of a ctx that can be accessed through a file descriptor (which you do just after this point), you know that the percpu_ref isn't zero, right? Why are you doing the tryget here?

Quoted 4 lines

Not sure I follow... We don't hold the lock at this point. I guess your point is that since the descriptor is open (or we'd fail the above check), then there's no point doing the tryget variant here? That's strictly true, that could just be a get().

As far as I can tell, you could do the following without breaking anything:

========================

fs/io_uring.c +0 -6
index 6916dc3222cf..c2d82765eefe 100644
@@ -2485,7 +2485,6 @@ static int __io_uring_register(struct
 io_ring_ctx *ctx, unsigned opcode,
        int ret;
 
        /* Drop our initial ref and wait for the ctx to be fully idle */
-       percpu_ref_put(&ctx->refs);
        percpu_ref_kill(&ctx->refs);
        wait_for_completion(&ctx->ctx_done);
@@ -2516,7 +2515,6 @@ static int __io_uring_register(struct
 io_ring_ctx *ctx, unsigned opcode,
        /* bring the ctx back to life */
        reinit_completion(&ctx->ctx_done);
        percpu_ref_resurrect(&ctx->refs);
-       percpu_ref_get(&ctx->refs);
        return ret;
 }
@@ -2535,17 +2533,13 @@ SYSCALL_DEFINE4(io_uring_register, unsigned
 int, fd, unsigned int, opcode,
        if (f.file->f_op != &io_uring_fops)
                goto out_fput;
 
-       ret = -ENXIO;
        ctx = f.file->private_data;
-       if (!percpu_ref_tryget(&ctx->refs))
-               goto out_fput;
 
        ret = -EBUSY;
        if (mutex_trylock(&ctx->uring_lock)) {
                ret = __io_uring_register(ctx, opcode, arg, nr_args);
                mutex_unlock(&ctx->uring_lock);
        }
-       io_ring_drop_ctx_refs(ctx, 1);
 out_fput:
        fdput(f);
        return ret;

========================

The two functions that can drop the initial ref of the percpu refcount are:

1. io_ring_ctx_wait_and_kill(); this is only used on ->release() or on setup failure, meaning that as long as you have a reference to the file from fget()/fdget(), io_ring_ctx_wait_and_kill() can't have been called on your context
2. __io_uring_register(); this temporarily kills the percpu refcount and resurrects it, all under ctx->uring_lock, meaning that as long as you're holding ctx->uring_lock, __io_uring_register() can't have killed the percpu refcount

Quoted 7 lines

Therefore, I think that as long as you're in sys_io_uring_register and holding the ctx->uring_lock, you know that the percpu refcount is alive, and bumping and dropping non-initial references has no effect.

Perhaps this makes more sense when you view the percpu refcount as a read/write lock - percpu_ref_tryget() takes a read lock, the percpu_ref_kill() dance takes a write lock.

This looks good, I'll fold it in. Thanks!

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#60 lore raw

PATCH 13/18 io_uring: add file set registration

In reply to Jens Axboe

We normally have to fget/fput for each IO we do on a file. Even with the batching we do, the cost of the atomic inc/dec of the file usage count adds up.

This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes for the io_uring_register(2) system call. The arguments passed in must be an array of __s32 holding file descriptors, and nr_args should hold the number of file descriptors the application wishes to pin for the duration of the io_uring context (or until IORING_UNREGISTER_FILES is called).

When used, the application must set IOSQE_FIXED_FILE in the sqe->flags member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd to the index in the array passed in to IORING_REGISTER_FILES.

Files are automatically unregistered when the io_uring context is torn down. An application need only unregister if it wishes to register a few set of fds.

---
 fs/io_uring.c                 | 125 +++++++++++++++++++++++++++++-----
 include/uapi/linux/io_uring.h |   9 ++-
 2 files changed, 116 insertions(+), 18 deletions(-)
fs/io_uring.c +108 -17
index 682714d6f217..77993972879b 100644
@@ -98,6 +98,10 @@ struct io_ring_ctx {
 		struct fasync_struct	*cq_fasync;
 	} ____cacheline_aligned_in_smp;
 
+	/* if used, fixed file set */
+	struct file		**user_files;
+	unsigned		nr_user_files;
+
 	/* if used, fixed mapped user buffers */
 	unsigned		nr_user_bufs;
 	struct io_mapped_ubuf	*user_bufs;
@@ -135,6 +139,7 @@ struct io_kiocb {
 #define REQ_F_FORCE_NONBLOCK	1	/* inline submission attempt */
 #define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
 #define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
+#define REQ_F_FIXED_FILE	8	/* ctx owns file */
 	u64			user_data;
 	u64			res;
 
@@ -357,15 +362,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
 		 * Batched puts of the same file, to avoid dirtying the
 		 * file usage count multiple times, if avoidable.
 		 */
-		if (!file) {
-			file = req->rw.ki_filp;
-			file_count = 1;
-		} else if (file == req->rw.ki_filp) {
-			file_count++;
-		} else {
-			fput_many(file, file_count);
-			file = req->rw.ki_filp;
-			file_count = 1;
+		if (!(req->flags & REQ_F_FIXED_FILE)) {
+			if (!file) {
+				file = req->rw.ki_filp;
+				file_count = 1;
+			} else if (file == req->rw.ki_filp) {
+				file_count++;
+			} else {
+				fput_many(file, file_count);
+				file = req->rw.ki_filp;
+				file_count = 1;
+			}
 		}
 
 		if (to_free == ARRAY_SIZE(reqs))
@@ -502,13 +509,19 @@ static void kiocb_end_write(struct kiocb *kiocb)
 	}
 }
 
+static void io_fput(struct io_kiocb *req)
+{
+	if (!(req->flags & REQ_F_FIXED_FILE))
+		fput(req->rw.ki_filp);
+}
+
 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
 {
 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
 
 	kiocb_end_write(kiocb);
 
-	fput(kiocb->ki_filp);
+	io_fput(req);
 	io_cqring_add_event(req->ctx, req->user_data, res, 0);
 	io_free_req(req);
 }
@@ -612,7 +625,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	struct kiocb *kiocb = &req->rw;
 	int ret;
 
-	kiocb->ki_filp = io_file_get(state, sqe->fd);
+	if (sqe->flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+			return -EBADF;
+		kiocb->ki_filp = ctx->user_files[sqe->fd];
+		req->flags |= REQ_F_FIXED_FILE;
+	} else {
+		kiocb->ki_filp = io_file_get(state, sqe->fd);
+	}
 	if (unlikely(!kiocb->ki_filp))
 		return -EBADF;
 	kiocb->ki_pos = sqe->off;
@@ -651,7 +671,8 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	}
 	return 0;
 out_fput:
-	io_file_put(state, kiocb->ki_filp);
+	if (!(sqe->flags & IOSQE_FIXED_FILE))
+		io_file_put(state, kiocb->ki_filp);
 	return ret;
 }
 
@@ -769,7 +790,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	kfree(iovec);
 out_fput:
 	if (unlikely(ret))
-		fput(file);
+		io_fput(req);
 	return ret;
 }
 
@@ -824,7 +845,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	kfree(iovec);
 out_fput:
 	if (unlikely(ret))
-		fput(file);
+		io_fput(req);
 	return ret;
 }
 
@@ -862,14 +883,23 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	if (unlikely(sqe->fsync_flags & ~IORING_FSYNC_DATASYNC))
 		return -EINVAL;
 
-	file = fget(sqe->fd);
+	if (sqe->flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+			return -EBADF;
+		file = ctx->user_files[sqe->fd];
+	} else {
+		file = fget(sqe->fd);
+	}
+
 	if (unlikely(!file))
 		return -EBADF;
 
 	ret = vfs_fsync_range(file, sqe->off, end > 0 ? end : LLONG_MAX,
 			sqe->fsync_flags & IORING_FSYNC_DATASYNC);
 
-	fput(file);
+	if (!(sqe->flags & IOSQE_FIXED_FILE))
+		fput(file);
+
 	io_cqring_add_event(ctx, sqe->user_data, ret, 0);
 	io_free_req(req);
 	return 0;
@@ -987,7 +1017,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 	ssize_t ret;
 
 	/* enforce forwards compatibility on users */
-	if (unlikely(s->sqe->flags))
+	if (unlikely(s->sqe->flags & ~IOSQE_FIXED_FILE))
 		return -EINVAL;
 
 	req = io_get_req(ctx, state);
@@ -1195,6 +1225,57 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
 	return submitted ? submitted : ret;
 }
 
+static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
+{
+	int i;
+
+	if (!ctx->user_files)
+		return -ENXIO;
+
+	for (i = 0; i < ctx->nr_user_files; i++)
+		fput(ctx->user_files[i]);
+
+	kfree(ctx->user_files);
+	ctx->user_files = NULL;
+	ctx->nr_user_files = 0;
+	return 0;
+}
+
+static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
+				 unsigned nr_args)
+{
+	__s32 __user *fds = (__s32 __user *) arg;
+	int fd, i, ret = 0;
+
+	if (ctx->user_files)
+		return -EBUSY;
+	if (!nr_args)
+		return -EINVAL;
+
+	ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
+	if (!ctx->user_files)
+		return -ENOMEM;
+
+	for (i = 0; i < nr_args; i++) {
+		ret = -EFAULT;
+		if (copy_from_user(&fd, &fds[i], sizeof(fd)))
+			break;
+
+		ctx->user_files[i] = fget(fd);
+
+		ret = -EBADF;
+		if (!ctx->user_files[i])
+			break;
+		ctx->nr_user_files++;
+		ret = 0;
+	}
+
+	if (ret)
+		io_sqe_files_unregister(ctx);
+
+	return ret;
+}
+
 static int io_sq_offload_start(struct io_ring_ctx *ctx)
 {
 	int ret;
@@ -1482,6 +1563,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	destroy_workqueue(ctx->sqo_wq);
 	io_iopoll_reap_events(ctx);
 	io_sqe_buffer_unregister(ctx);
+	io_sqe_files_unregister(ctx);
 
 	io_mem_free(ctx->sq_ring);
 	io_mem_free(ctx->sq_sqes);
@@ -1777,6 +1859,15 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
 			break;
 		ret = io_sqe_buffer_unregister(ctx);
 		break;
+	case IORING_REGISTER_FILES:
+		ret = io_sqe_files_register(ctx, arg, nr_args);
+		break;
+	case IORING_UNREGISTER_FILES:
+		ret = -EINVAL;
+		if (arg || nr_args)
+			break;
+		ret = io_sqe_files_unregister(ctx);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
include/uapi/linux/io_uring.h +8 -1
index 03ce7133c3b2..8323320077ec 100644
@@ -18,7 +18,7 @@
  */
 struct io_uring_sqe {
 	__u8	opcode;		/* type of operation for this sqe */
-	__u8	flags;		/* as of now unused */
+	__u8	flags;		/* IOSQE_ flags */
 	__u16	ioprio;		/* ioprio for the request */
 	__s32	fd;		/* file descriptor to do IO on */
 	__u64	off;		/* offset into file */
@@ -35,6 +35,11 @@ struct io_uring_sqe {
 	};
 };
 
+/*
+ * sqe->flags
+ */
+#define IOSQE_FIXED_FILE	(1 << 0)	/* use fixed fileset */
+
 /*
  * io_uring_setup() flags
  */
@@ -114,5 +119,7 @@ struct io_uring_params {
  */
 #define IORING_REGISTER_BUFFERS		0
 #define IORING_UNREGISTER_BUFFERS	1
+#define IORING_REGISTER_FILES		2
+#define IORING_UNREGISTER_FILES		3
 
 #endif
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
#61 lore raw

Re: [PATCH 13/18] io_uring: add file set registration

Quoted 11 lines · Jens Axboe

We normally have to fget/fput for each IO we do on a file. Even with the batching we do, the cost of the atomic inc/dec of the file usage count adds up.

This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes for the io_uring_register(2) system call. The arguments passed in must be an array of __s32 holding file descriptors, and nr_args should hold the number of file descriptors the application wishes to pin for the duration of the io_uring context (or until IORING_UNREGISTER_FILES is called).

Quoted 7 lines

When used, the application must set IOSQE_FIXED_FILE in the sqe->flags member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd to the index in the array passed in to IORING_REGISTER_FILES.

Files are automatically unregistered when the io_uring context is torn down. An application need only unregister if it wishes to register a few set of fds.

s/few/new/ ?

Quoted 11 lines
---
 fs/io_uring.c                 | 125 +++++++++++++++++++++++++++++-----
 include/uapi/linux/io_uring.h |   9 ++-
 2 files changed, 116 insertions(+), 18 deletions(-)
fs/io_uring.c +0 -0
index 682714d6f217..77993972879b 100644

fs/io_uring.c struct io_ring_ctx { · line 103

Quoted 6 lines
fs/io_uring.c +3 -0
@@ -98,6 +98,10 @@ struct io_ring_ctx {
                struct fasync_struct    *cq_fasync;
        } ____cacheline_aligned_in_smp;
 
+       /* if used, fixed file set */
+       struct file             **user_files;
+       unsigned                nr_user_files;

It'd be nice if you could add a comment about locking rules here - something like "writers must ensure that ->refs is dead, readers must ensure that ->refs is alive as long as the file* is used".

[...]

fs/io_uring.c static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · line 631

Quoted 9 lines
fs/io_uring.c +4 -1
@@ -612,7 +625,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
        struct kiocb *kiocb = &req->rw;
        int ret;
 
-       kiocb->ki_filp = io_file_get(state, sqe->fd);
+       if (sqe->flags & IOSQE_FIXED_FILE) {
+               if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+                       return -EBADF;
+               kiocb->ki_filp = ctx->user_files[sqe->fd];

It doesn't really matter as long as ctx->nr_user_files<=INT_MAX, but it'd be nice if you could explicitly cast sqe->fd to unsigned here.

Quoted 4 lines
fs/io_uring.c +4 -0
+               req->flags |= REQ_F_FIXED_FILE;
+       } else {
+               kiocb->ki_filp = io_file_get(state, sqe->fd);
+       }

[...]

Quoted 9 lines
fs/io_uring.c +9 -0
+static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
+                                unsigned nr_args)
+{
+       __s32 __user *fds = (__s32 __user *) arg;
+       int fd, i, ret = 0;
+
+       if (ctx->user_files)
+               return -EBUSY;
+       if (!nr_args)

fs/io_uring.c static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit, · line 1262

Quoted 10 lines
fs/io_uring.c +10 -0
+               return -EINVAL;
+
+       ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
+       if (!ctx->user_files)
+               return -ENOMEM;
+
+       for (i = 0; i < nr_args; i++) {
+               ret = -EFAULT;
+               if (copy_from_user(&fd, &fds[i], sizeof(fd)))
+                       break;

"i" is signed, but "nr_args" is unsigned. You can't get through that kcalloc() call with nr_args>=0x80000000 on a normal kernel, someone would have to set CONFIG_FORCE_MAX_ZONEORDER really high for that, but still, in theory, if you reach this copy_to_user(..., &fds[i], ...) with a negative "i", that'd be bad. You might want to make "i" unsigned here and check that it's at least smaller than UINT_MAX...

Quoted 5 lines
fs/io_uring.c +5 -0
+               ctx->user_files[i] = fget(fd);
+
+               ret = -EBADF;
+               if (!ctx->user_files[i])
+                       break;

fs/io_uring.c static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit, · line 1278

Quoted 10 lines
fs/io_uring.c +10 -0
+               ctx->nr_user_files++;
+               ret = 0;
+       }
+
+       if (ret)
+               io_sqe_files_unregister(ctx);
+
+       return ret;
+}
+

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#62 lore raw

Re: [PATCH 13/18] io_uring: add file set registration

Quoted 16 lines · Jann Horn
Quoted 15 lines · Jens Axboe

We normally have to fget/fput for each IO we do on a file. Even with the batching we do, the cost of the atomic inc/dec of the file usage count adds up.

This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes for the io_uring_register(2) system call. The arguments passed in must be an array of __s32 holding file descriptors, and nr_args should hold the number of file descriptors the application wishes to pin for the duration of the io_uring context (or until IORING_UNREGISTER_FILES is called).

When used, the application must set IOSQE_FIXED_FILE in the sqe->flags member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd to the index in the array passed in to IORING_REGISTER_FILES.

Quoted 5 lines
Quoted 3 lines

Files are automatically unregistered when the io_uring context is torn down. An application need only unregister if it wishes to register a few set of fds.

s/few/new/ ?

Indeed, thanks.

Quoted 8 lines
Quoted 8 lines
fs/io_uring.c +0 -0
index 682714d6f217..77993972879b 100644
@@ -98,6 +98,10 @@ struct io_ring_ctx {
                struct fasync_struct    *cq_fasync;
        } ____cacheline_aligned_in_smp;
 
Quoted 7 lines

fs/io_uring.c struct io_ring_ctx { · line 103

Quoted 3 lines
fs/io_uring.c +3 -0
+       /* if used, fixed file set */
+       struct file             **user_files;
+       unsigned                nr_user_files;

It'd be nice if you could add a comment about locking rules here - something like "writers must ensure that ->refs is dead, readers must ensure that ->refs is alive as long as the file* is used".

Will add.

Quoted 5 lines

[...]

Quoted 4 lines
fs/io_uring.c +0 -0
@@ -612,7 +625,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
        struct kiocb *kiocb = &req->rw;
        int ret;
 
Quoted 8 lines

fs/io_uring.c static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, · line 631

Quoted 5 lines
fs/io_uring.c +4 -1
-       kiocb->ki_filp = io_file_get(state, sqe->fd);
+       if (sqe->flags & IOSQE_FIXED_FILE) {
+               if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+                       return -EBADF;
+               kiocb->ki_filp = ctx->user_files[sqe->fd];

It doesn't really matter as long as ctx->nr_user_files<=INT_MAX, but it'd be nice if you could explicitly cast sqe->fd to unsigned here.

OK, will do.

Quoted 23 lines
Quoted 4 lines
fs/io_uring.c +4 -0
+               req->flags |= REQ_F_FIXED_FILE;
+       } else {
+               kiocb->ki_filp = io_file_get(state, sqe->fd);
+       }

[...]

Quoted 18 lines
fs/io_uring.c +18 -0
+static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
+                                unsigned nr_args)
+{
+       __s32 __user *fds = (__s32 __user *) arg;
+       int fd, i, ret = 0;
+
+       if (ctx->user_files)
+               return -EBUSY;
+       if (!nr_args)
+               return -EINVAL;
+
+       ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
+       if (!ctx->user_files)
+               return -ENOMEM;
+
+       for (i = 0; i < nr_args; i++) {
+               ret = -EFAULT;
+               if (copy_from_user(&fd, &fds[i], sizeof(fd)))
Quoted 8 lines
Quoted 1 line

+ break;

"i" is signed, but "nr_args" is unsigned. You can't get through that kcalloc() call with nr_args>=0x80000000 on a normal kernel, someone would have to set CONFIG_FORCE_MAX_ZONEORDER really high for that, but still, in theory, if you reach this copy_to_user(..., &fds[i], ...) with a negative "i", that'd be bad. You might want to make "i" unsigned here and check that it's at least smaller than UINT_MAX...

Done.

Thanks for your review!

Signature · 7 lines
Jens Axboe

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#63 lore raw

PATCH 14/18 io_uring: add submission polling

In reply to Jens Axboe

This enables an application to do IO, without ever entering the kernel. By using the SQ ring to fill in new sqes and watching for completions on the CQ ring, we can submit and reap IOs without doing a single system call. The kernel side thread will poll for new submissions, and in case of HIPRI/polled IO, it'll also poll for completions.

By default, we allow 1 second of active spinning. This can by changed by passing in a different grace period at io_uring_register(2) time. If the thread exceeds this idle time without having any work to do, it will set:

sq_ring->flags |= IORING_SQ_NEED_WAKEUP.

The application will have to call io_uring_enter() to start things back up again. If IO is kept busy, that will never be needed. Basically an application that has this feature enabled will guard it's io_uring_enter(2) call with:

read_barrier();
if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP)

	io_uring_enter(fd, 0, 0, IORING_ENTER_SQ_WAKEUP);

instead of calling it unconditionally.

---
 fs/io_uring.c                 | 232 +++++++++++++++++++++++++++++++++-
 include/uapi/linux/io_uring.h |  12 +-
 2 files changed, 237 insertions(+), 7 deletions(-)
fs/io_uring.c +226 -6
index 77993972879b..34e786eb2e2d 100644
@@ -24,6 +24,7 @@
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/workqueue.h>
+#include <linux/kthread.h>
 #include <linux/blkdev.h>
 #include <linux/bvec.h>
 #include <linux/anon_inodes.h>
@@ -79,14 +80,16 @@ struct io_ring_ctx {
 		unsigned		cached_sq_head;
 		unsigned		sq_entries;
 		unsigned		sq_mask;
-		unsigned		sq_thread_cpu;
+		unsigned		sq_thread_idle;
 		struct io_uring_sqe	*sq_sqes;
 	} ____cacheline_aligned_in_smp;
 
 	/* IO offload */
 	struct workqueue_struct	*sqo_wq;
+	struct task_struct	*sqo_thread;	/* if using sq thread polling */
 	struct mm_struct	*sqo_mm;
 	struct files_struct	*sqo_files;
+	wait_queue_head_t	sqo_wait;
 
 	struct {
 		/* CQ ring */
@@ -262,6 +265,8 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
 
 	if (waitqueue_active(&ctx->wait))
 		wake_up(&ctx->wait);
+	if (waitqueue_active(&ctx->sqo_wait))
+		wake_up(&ctx->sqo_wait);
 }
 
 static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
@@ -1113,6 +1118,168 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 	return false;
 }
 
+static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
+			  unsigned int nr, bool mm_fault)
+{
+	struct io_submit_state state, *statep = NULL;
+	int ret, i, submitted = 0;
+
+	if (nr > IO_PLUG_THRESHOLD) {
+		io_submit_state_start(&state, ctx, nr);
+		statep = &state;
+	}
+
+	for (i = 0; i < nr; i++) {
+		if (unlikely(mm_fault))
+			ret = -EFAULT;
+		else
+			ret = io_submit_sqe(ctx, &sqes[i], statep);
+		if (!ret) {
+			submitted++;
+			continue;
+		}
+
+		io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret, 0);
+	}
+
+	if (statep)
+		io_submit_state_end(&state);
+
+	return submitted;
+}
+
+static int io_sq_thread(void *data)
+{
+	struct sqe_submit sqes[IO_IOPOLL_BATCH];
+	struct io_ring_ctx *ctx = data;
+	struct mm_struct *cur_mm = NULL;
+	struct files_struct *old_files;
+	mm_segment_t old_fs;
+	DEFINE_WAIT(wait);
+	unsigned inflight;
+	unsigned long timeout;
+
+	old_files = current->files;
+	current->files = ctx->sqo_files;
+
+	old_fs = get_fs();
+	set_fs(USER_DS);
+
+	timeout = inflight = 0;
+	while (!kthread_should_stop()) {
+		bool all_fixed, mm_fault = false;
+		int i;
+
+		if (inflight) {
+			unsigned nr_events = 0;
+
+			if (ctx->flags & IORING_SETUP_IOPOLL) {
+				/*
+				 * We disallow the app entering submit/complete
+				 * with polling, so no need to lock the ring.
+				 */
+				io_iopoll_check(ctx, &nr_events, 0);
+			} else {
+				/*
+				 * Normal IO, just pretend everything completed.
+				 * We don't have to poll completions for that.
+				 */
+				nr_events = inflight;
+			}
+
+			inflight -= nr_events;
+			if (!inflight)
+				timeout = jiffies + ctx->sq_thread_idle;
+		}
+
+		if (!io_get_sqring(ctx, &sqes[0])) {
+			/*
+			 * We're polling. If we're within the defined idle
+			 * period, then let us spin without work before going
+			 * to sleep.
+			 */
+			if (inflight || !time_after(jiffies, timeout)) {
+				cpu_relax();
+				continue;
+			}
+
+			/*
+			 * Drop cur_mm before scheduling, we can't hold it for
+			 * long periods (or over schedule()). Do this before
+			 * adding ourselves to the waitqueue, as the unuse/drop
+			 * may sleep.
+			 */
+			if (cur_mm) {
+				unuse_mm(cur_mm);
+				mmput(cur_mm);
+				cur_mm = NULL;
+			}
+
+			prepare_to_wait(&ctx->sqo_wait, &wait,
+						TASK_INTERRUPTIBLE);
+
+			/* Tell userspace we may need a wakeup call */
+			ctx->sq_ring->flags |= IORING_SQ_NEED_WAKEUP;
+			smp_wmb();
+
+			if (!io_get_sqring(ctx, &sqes[0])) {
+				if (kthread_should_park())
+					kthread_parkme();
+				if (kthread_should_stop()) {
+					finish_wait(&ctx->sqo_wait, &wait);
+					break;
+				}
+				if (signal_pending(current))
+					flush_signals(current);
+				schedule();
+				finish_wait(&ctx->sqo_wait, &wait);
+
+				ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+				smp_wmb();
+				continue;
+			}
+			finish_wait(&ctx->sqo_wait, &wait);
+
+			ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+			smp_wmb();
+		}
+
+		i = 0;
+		all_fixed = true;
+		do {
+			if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
+				all_fixed = false;
+
+			i++;
+			if (i == ARRAY_SIZE(sqes))
+				break;
+		} while (io_get_sqring(ctx, &sqes[i]));
+
+		io_commit_sqring(ctx);
+
+		/* Unless all new commands are FIXED regions, grab mm */
+		if (!all_fixed && !cur_mm) {
+			mm_fault = !mmget_not_zero(ctx->sqo_mm);
+			if (!mm_fault) {
+				use_mm(ctx->sqo_mm);
+				cur_mm = ctx->sqo_mm;
+			}
+		}
+
+		inflight += io_submit_sqes(ctx, sqes, i, mm_fault);
+	}
+
+	io_iopoll_reap_events(ctx);
+
+	current->files = old_files;
+	set_fs(old_fs);
+	if (cur_mm) {
+		unuse_mm(cur_mm);
+		mmput(cur_mm);
+	}
+	return 0;
+}
+
 static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
 {
 	struct io_submit_state state, *statep = NULL;
@@ -1198,6 +1365,17 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,
 {
 	int submitted, ret;
 
+	/*
+	 * For SQ polling, the thread will do all submissions and completions.
+	 * Just return the requested submit count, and wake the thread if
+	 * we were asked to.
+	 */
+	if (ctx->flags & IORING_SETUP_SQPOLL) {
+		if (flags & IORING_ENTER_SQ_WAKEUP)
+			wake_up(&ctx->sqo_wait);
+		return to_submit;
+	}
+
 	submitted = ret = 0;
 	if (to_submit) {
 		submitted = io_ring_submit(ctx, to_submit);
@@ -1276,10 +1454,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
 	return ret;
 }
 
-static int io_sq_offload_start(struct io_ring_ctx *ctx)
+static int io_sq_offload_start(struct io_ring_ctx *ctx,
+			       struct io_uring_params *p)
 {
 	int ret;
 
+	init_waitqueue_head(&ctx->sqo_wait);
 	ctx->sqo_mm = current->mm;
 
 	/*
@@ -1292,6 +1472,34 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)
 	if (!ctx->sqo_files)
 		goto err;
 
+	ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
+	if (!ctx->sq_thread_idle)
+		ctx->sq_thread_idle = HZ;
+
+	if (ctx->flags & IORING_SETUP_SQPOLL) {
+		if (p->flags & IORING_SETUP_SQ_AFF) {
+			int cpu;
+
+			cpu = array_index_nospec(p->sq_thread_cpu, NR_CPUS);
+			ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
+							ctx, cpu,
+							"io_uring-sq");
+		} else {
+			ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
+							"io_uring-sq");
+		}
+		if (IS_ERR(ctx->sqo_thread)) {
+			ret = PTR_ERR(ctx->sqo_thread);
+			ctx->sqo_thread = NULL;
+			goto err;
+		}
+		wake_up_process(ctx->sqo_thread);
+	} else if (p->flags & IORING_SETUP_SQ_AFF) {
+		/* Can't have SQ_AFF without SQPOLL */
+		ret = -EINVAL;
+		goto err;
+	}
+
 	/* Do QD, or 2 * CPUS, whatever is smallest */
 	ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
 			min(ctx->sq_entries - 1, 2 * num_online_cpus()));
@@ -1302,6 +1510,11 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)
 
 	return 0;
 err:
+	if (ctx->sqo_thread) {
+		kthread_park(ctx->sqo_thread);
+		kthread_stop(ctx->sqo_thread);
+		ctx->sqo_thread = NULL;
+	}
 	if (ctx->sqo_files)
 		ctx->sqo_files = NULL;
 	ctx->sqo_mm = NULL;
@@ -1560,8 +1773,13 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
 
 static void io_ring_ctx_free(struct io_ring_ctx *ctx)
 {
+	if (ctx->sqo_thread) {
+		kthread_park(ctx->sqo_thread);
+		kthread_stop(ctx->sqo_thread);
+	}
 	destroy_workqueue(ctx->sqo_wq);
-	io_iopoll_reap_events(ctx);
+	if (!(ctx->flags & IORING_SETUP_SQPOLL))
+		io_iopoll_reap_events(ctx);
 	io_sqe_buffer_unregister(ctx);
 	io_sqe_files_unregister(ctx);
 
@@ -1602,7 +1820,8 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 	percpu_ref_kill(&ctx->refs);
 	mutex_unlock(&ctx->uring_lock);
 
-	io_iopoll_reap_events(ctx);
+	if (!(ctx->flags & IORING_SETUP_SQPOLL))
+		io_iopoll_reap_events(ctx);
 	wait_for_completion(&ctx->ctx_done);
 	io_ring_ctx_free(ctx);
 }
@@ -1771,7 +1990,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
 	if (ret)
 		goto err;
 
-	ret = io_sq_offload_start(ctx);
+	ret = io_sq_offload_start(ctx, p);
 	if (ret)
 		goto err;
 
@@ -1820,7 +2039,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 			return -EINVAL;
 	}
 
-	if (p.flags & ~IORING_SETUP_IOPOLL)
+	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
+			IORING_SETUP_SQ_AFF))
 		return -EINVAL;
 
 	ret = io_uring_create(entries, &p);
include/uapi/linux/io_uring.h +11 -1
index 8323320077ec..cb3592d618fe 100644
@@ -44,6 +44,8 @@ struct io_uring_sqe {
  * io_uring_setup() flags
  */
 #define IORING_SETUP_IOPOLL	(1 << 0)	/* io_context is polled */
+#define IORING_SETUP_SQPOLL	(1 << 1)	/* SQ poll thread */
+#define IORING_SETUP_SQ_AFF	(1 << 2)	/* sq_thread_cpu is valid */
 
 #define IORING_OP_NOP		0
 #define IORING_OP_READV		1
@@ -87,6 +89,11 @@ struct io_sqring_offsets {
 	__u32 resv[3];
 };
 
+/*
+ * sq_ring->flags
+ */
+#define IORING_SQ_NEED_WAKEUP	(1 << 0) /* needs io_uring_enter wakeup */
+
 struct io_cqring_offsets {
 	__u32 head;
 	__u32 tail;
@@ -101,6 +108,7 @@ struct io_cqring_offsets {
  * io_uring_enter(2) flags
  */
 #define IORING_ENTER_GETEVENTS	(1 << 0)
+#define IORING_ENTER_SQ_WAKEUP	(1 << 1)
 
 /*
  * Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -109,7 +117,9 @@ struct io_uring_params {
 	__u32 sq_entries;
 	__u32 cq_entries;
 	__u32 flags;
-	__u16 resv[10];
+	__u16 sq_thread_cpu;
+	__u16 sq_thread_idle;
+	__u16 resv[8];
 	struct io_sqring_offsets sq_off;
 	struct io_cqring_offsets cq_off;
 };
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#64 lore raw

PATCH 15/18 io_uring: add io_kiocb ref count

In reply to Jens Axboe

We'll use this for the POLL implementation. Regular requests will NOT be using references, so initialize it to 0. Any real use of the io_kiocb ref will initialize it to at least 2.

---
 fs/io_uring.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
fs/io_uring.c +6 -2
index 34e786eb2e2d..71bf890e25f2 100644
@@ -139,6 +139,7 @@ struct io_kiocb {
 	struct io_ring_ctx	*ctx;
 	struct list_head	list;
 	unsigned int		flags;
+	refcount_t		refs;
 #define REQ_F_FORCE_NONBLOCK	1	/* inline submission attempt */
 #define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
 #define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
@@ -319,6 +320,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
 	if (req) {
 		req->ctx = ctx;
 		req->flags = 0;
+		refcount_set(&req->refs, 0);
 		return req;
 	}
 
@@ -338,8 +340,10 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
 
 static void io_free_req(struct io_kiocb *req)
 {
-	io_ring_drop_ctx_refs(req->ctx, 1);
-	kmem_cache_free(req_cachep, req);
+	if (!refcount_read(&req->refs) || refcount_dec_and_test(&req->refs)) {
+		io_ring_drop_ctx_refs(req->ctx, 1);
+		kmem_cache_free(req_cachep, req);
+	}
 }
 
 /*
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Reviewed-by on v1 15/18
#65 lore raw

Re: [PATCH 15/18] io_uring: add io_kiocb ref count

Quoted 3 lines · Jens Axboe

We'll use this for the POLL implementation. Regular requests will NOT be using references, so initialize it to 0. Any real use of the io_kiocb ref will initialize it to at least 2.

Looks good,

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>

author maintainer
#66 lore raw

PATCH 16/18 io_uring: add support for IORING_OP_POLL

In reply to Jens Axboe

This is basically a direct port of bfe4037e722e, which implements a one-shot poll command through aio. Description below is based on that commit as well. However, instead of adding a POLL command and relying on io_cancel(2) to remove it, we mimic the epoll(2) interface of having a command to add a poll notification, IORING_OP_POLL_ADD, and one to remove it again, IORING_OP_POLL_REMOVE.

To poll for a file descriptor the application should submit an sqe of type IORING_OP_POLL. It will poll the fd for the events specified in the poll_events field.

Unlike poll or epoll without EPOLLONESHOT this interface always works in one shot mode, that is once the sqe is completed, it will have to be resubmitted.

Based-on-code-from: Christoph Hellwig <[email protected]>

---
 fs/io_uring.c                 | 245 ++++++++++++++++++++++++++++++++++
 include/uapi/linux/io_uring.h |   3 +
 2 files changed, 248 insertions(+)
fs/io_uring.c +245 -0
index 71bf890e25f2..13e9bb0ce44e 100644
@@ -122,6 +122,7 @@ struct io_ring_ctx {
 		spinlock_t		completion_lock;
 		bool			poll_multi_file;
 		struct list_head	poll_list;
+		struct list_head	cancel_list;
 	} ____cacheline_aligned_in_smp;
 };
 
@@ -130,9 +131,19 @@ struct sqe_submit {
 	unsigned			index;
 };
 
+struct io_poll_iocb {
+	struct file *file;
+	struct wait_queue_head *head;
+	__poll_t events;
+	bool woken;
+	bool canceled;
+	struct wait_queue_entry wait;
+};
+
 struct io_kiocb {
 	union {
 		struct kiocb		rw;
+		struct io_poll_iocb	poll;
 		struct sqe_submit	submit;
 	};
 
@@ -204,6 +215,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	init_waitqueue_head(&ctx->wait);
 	spin_lock_init(&ctx->completion_lock);
 	INIT_LIST_HEAD(&ctx->poll_list);
+	INIT_LIST_HEAD(&ctx->cancel_list);
 	return ctx;
 }
 
@@ -914,6 +926,232 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	return 0;
 }
 
+static void io_poll_remove_one(struct io_kiocb *req)
+{
+	struct io_poll_iocb *poll = &req->poll;
+
+	spin_lock(&poll->head->lock);
+	WRITE_ONCE(poll->canceled, true);
+	if (!list_empty(&poll->wait.entry)) {
+		list_del_init(&poll->wait.entry);
+		queue_work(req->ctx->sqo_wq, &req->work);
+	}
+	spin_unlock(&poll->head->lock);
+
+	list_del_init(&req->list);
+}
+
+static void io_poll_remove_all(struct io_ring_ctx *ctx)
+{
+	struct io_kiocb *req;
+
+	spin_lock_irq(&ctx->completion_lock);
+	while (!list_empty(&ctx->cancel_list)) {
+		req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
+		io_poll_remove_one(req);
+	}
+	spin_unlock_irq(&ctx->completion_lock);
+}
+
+/*
+ * Find a running poll command that matches one specified in sqe->addr,
+ * and remove it if found.
+ */
+static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+	struct io_ring_ctx *ctx = req->ctx;
+	struct io_kiocb *poll_req, *next;
+	int ret = -ENOENT;
+
+	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+		return -EINVAL;
+	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
+	    sqe->poll_events)
+		return -EINVAL;
+
+	spin_lock_irq(&ctx->completion_lock);
+	list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
+		if (sqe->addr == poll_req->user_data) {
+			io_poll_remove_one(poll_req);
+			ret = 0;
+			break;
+		}
+	}
+	spin_unlock_irq(&ctx->completion_lock);
+
+	io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
+	io_free_req(req);
+	return 0;
+}
+
+static void io_poll_complete(struct io_kiocb *req, __poll_t mask)
+{
+	io_cqring_add_event(req->ctx, req->user_data, mangle_poll(mask), 0);
+	io_fput(req);
+	io_free_req(req);
+}
+
+static void io_poll_complete_work(struct work_struct *work)
+{
+	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+	struct io_poll_iocb *poll = &req->poll;
+	struct poll_table_struct pt = { ._key = poll->events };
+	struct io_ring_ctx *ctx = req->ctx;
+	__poll_t mask = 0;
+
+	if (!READ_ONCE(poll->canceled))
+		mask = vfs_poll(poll->file, &pt) & poll->events;
+
+	/*
+	 * Note that ->ki_cancel callers also delete iocb from active_reqs after
+	 * calling ->ki_cancel.  We need the ctx_lock roundtrip here to
+	 * synchronize with them.  In the cancellation case the list_del_init
+	 * itself is not actually needed, but harmless so we keep it in to
+	 * avoid further branches in the fast path.
+	 */
+	spin_lock_irq(&ctx->completion_lock);
+	if (!mask && !READ_ONCE(poll->canceled)) {
+		add_wait_queue(poll->head, &poll->wait);
+		spin_unlock_irq(&ctx->completion_lock);
+		return;
+	}
+	list_del_init(&req->list);
+	spin_unlock_irq(&ctx->completion_lock);
+
+	io_poll_complete(req, mask);
+}
+
+static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
+			void *key)
+{
+	struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
+							wait);
+	struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
+	struct io_ring_ctx *ctx = req->ctx;
+	__poll_t mask = key_to_poll(key);
+
+	poll->woken = true;
+
+	/* for instances that support it check for an event match first: */
+	if (mask) {
+		if (!(mask & poll->events))
+			return 0;
+
+		/* try to complete the iocb inline if we can: */
+		if (spin_trylock(&ctx->completion_lock)) {
+			list_del(&req->list);
+			spin_unlock(&ctx->completion_lock);
+
+			list_del_init(&poll->wait.entry);
+			io_poll_complete(req, mask);
+			return 1;
+		}
+	}
+
+	list_del_init(&poll->wait.entry);
+	queue_work(ctx->sqo_wq, &req->work);
+	return 1;
+}
+
+struct io_poll_table {
+	struct poll_table_struct pt;
+	struct io_kiocb *req;
+	int error;
+};
+
+static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
+			       struct poll_table_struct *p)
+{
+	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
+
+	if (unlikely(pt->req->poll.head)) {
+		pt->error = -EINVAL;
+		return;
+	}
+
+	pt->error = 0;
+	pt->req->poll.head = head;
+	add_wait_queue(head, &pt->req->poll.wait);
+}
+
+static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+	struct io_poll_iocb *poll = &req->poll;
+	struct io_ring_ctx *ctx = req->ctx;
+	struct io_poll_table ipt;
+	__poll_t mask;
+
+	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+		return -EINVAL;
+	if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
+		return -EINVAL;
+
+	INIT_WORK(&req->work, io_poll_complete_work);
+	poll->events = demangle_poll(sqe->poll_events) | EPOLLERR | EPOLLHUP;
+
+	if (sqe->flags & IOSQE_FIXED_FILE) {
+		if (unlikely(!ctx->user_files || sqe->fd >= ctx->nr_user_files))
+			return -EBADF;
+		poll->file = ctx->user_files[sqe->fd];
+		req->flags |= REQ_F_FIXED_FILE;
+	} else {
+		poll->file = fget(sqe->fd);
+	}
+	if (unlikely(!poll->file))
+		return -EBADF;
+
+	poll->head = NULL;
+	poll->woken = false;
+	poll->canceled = false;
+
+	ipt.pt._qproc = io_poll_queue_proc;
+	ipt.pt._key = poll->events;
+	ipt.req = req;
+	ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
+
+	/* initialized the list so that we can do list_empty checks */
+	INIT_LIST_HEAD(&poll->wait.entry);
+	init_waitqueue_func_entry(&poll->wait, io_poll_wake);
+
+	/* one for removal from waitqueue, one for this function */
+	refcount_set(&req->refs, 2);
+
+	mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
+	if (unlikely(!poll->head)) {
+		/* we did not manage to set up a waitqueue, done */
+		goto out;
+	}
+
+	spin_lock_irq(&ctx->completion_lock);
+	spin_lock(&poll->head->lock);
+	if (poll->woken) {
+		/* wake_up context handles the rest */
+		mask = 0;
+		ipt.error = 0;
+	} else if (mask || ipt.error) {
+		/* if we get an error or a mask we are done */
+		WARN_ON_ONCE(list_empty(&poll->wait.entry));
+		list_del_init(&poll->wait.entry);
+	} else {
+		/* actually waiting for an event */
+		list_add_tail(&req->list, &ctx->cancel_list);
+	}
+	spin_unlock(&poll->head->lock);
+	spin_unlock_irq(&ctx->completion_lock);
+
+out:
+	if (unlikely(ipt.error)) {
+		if (!(sqe->flags & IOSQE_FIXED_FILE))
+			fput(poll->file);
+		return ipt.error;
+	}
+
+	if (mask)
+		io_poll_complete(req, mask);
+	io_free_req(req);
+	return 0;
+}
+
 static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 			   struct sqe_submit *s, bool force_nonblock,
 			   struct io_submit_state *state)
@@ -949,6 +1187,12 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 	case IORING_OP_FSYNC:
 		ret = io_fsync(req, sqe, force_nonblock);
 		break;
+	case IORING_OP_POLL_ADD:
+		ret = io_poll_add(req, sqe);
+		break;
+	case IORING_OP_POLL_REMOVE:
+		ret = io_poll_remove(req, sqe);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
@@ -1824,6 +2068,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
 	percpu_ref_kill(&ctx->refs);
 	mutex_unlock(&ctx->uring_lock);
 
+	io_poll_remove_all(ctx);
 	if (!(ctx->flags & IORING_SETUP_SQPOLL))
 		io_iopoll_reap_events(ctx);
 	wait_for_completion(&ctx->ctx_done);
include/uapi/linux/io_uring.h +3 -0
index cb3592d618fe..39d3d3336dce 100644
@@ -27,6 +27,7 @@ struct io_uring_sqe {
 	union {
 		__kernel_rwf_t	rw_flags;
 		__u32		fsync_flags;
+		__u16		poll_events;
 	};
 	__u64	user_data;	/* data to be passed back at completion time */
 	union {
@@ -53,6 +54,8 @@ struct io_uring_sqe {
 #define IORING_OP_FSYNC		3
 #define IORING_OP_READ_FIXED	4
 #define IORING_OP_WRITE_FIXED	5
+#define IORING_OP_POLL_ADD	6
+#define IORING_OP_POLL_REMOVE	7
 
 /*
  * sqe->fsync_flags
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#67 lore raw

PATCH 17/18 io_uring: allow workqueue item to handle multiple buffered requests

In reply to Jens Axboe

Right now we punt any buffered request that ends up triggering an -EAGAIN to an async workqueue. This works fine in terms of providing async execution of them, but it also can create quite a lot of work queue items. For sequentially buffered IO, it's advantageous to serialize the issue of them. For reads, the first one will trigger a read-ahead, and subsequent request merely end up waiting on later pages to complete. For writes, devices usually respond better to streamed sequential writes.

Add state to track the last buffered request we punted to a work queue, and if the next one is sequential to the previous, attempt to get the previous work item to handle it. We limit the number of sequential add-ons to the a multiple (8) of the max read-ahead size of the file. This should be a good number for both reads and wries, as it defines the max IO size the device can do directly.

This drastically cuts down on the number of context switches we need to handle buffered sequential IO, and a basic test case of copying a big file with io_uring sees a 5x speedup.

---
 fs/io_uring.c | 231 ++++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 194 insertions(+), 37 deletions(-)
fs/io_uring.c +194 -37
index 13e9bb0ce44e..0240e0a04f6a 100644
@@ -67,6 +67,16 @@ struct io_mapped_ubuf {
 	unsigned int	nr_bvecs;
 };
 
+struct async_list {
+	spinlock_t		lock;
+	atomic_t		cnt;
+	struct list_head	list;
+
+	struct file		*file;
+	off_t			io_end;
+	size_t			io_pages;
+};
+
 struct io_ring_ctx {
 	struct {
 		struct percpu_ref	refs;
@@ -124,6 +134,8 @@ struct io_ring_ctx {
 		struct list_head	poll_list;
 		struct list_head	cancel_list;
 	} ____cacheline_aligned_in_smp;
+
+	struct async_list	pending_async[2];
 };
 
 struct sqe_submit {
@@ -155,6 +167,7 @@ struct io_kiocb {
 #define REQ_F_IOPOLL_COMPLETED	2	/* polled IO has completed */
 #define REQ_F_IOPOLL_EAGAIN	4	/* submission got EAGAIN */
 #define REQ_F_FIXED_FILE	8	/* ctx owns file */
+#define REQ_F_SEQ_PREV		16	/* sequential with previous */
 	u64			user_data;
 	u64			res;
 
@@ -198,6 +211,7 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref)
 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 {
 	struct io_ring_ctx *ctx;
+	int i;
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
@@ -213,6 +227,11 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	init_completion(&ctx->ctx_done);
 	mutex_init(&ctx->uring_lock);
 	init_waitqueue_head(&ctx->wait);
+	for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) {
+		spin_lock_init(&ctx->pending_async[i].lock);
+		INIT_LIST_HEAD(&ctx->pending_async[i].list);
+		atomic_set(&ctx->pending_async[i].cnt, 0);
+	}
 	spin_lock_init(&ctx->completion_lock);
 	INIT_LIST_HEAD(&ctx->poll_list);
 	INIT_LIST_HEAD(&ctx->cancel_list);
@@ -772,6 +791,39 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
 	return import_iovec(rw, buf, sqe->len, UIO_FASTIOV, iovec, iter);
 }
 
+static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
+{
+	struct async_list *async_list = &req->ctx->pending_async[rw];
+	struct kiocb *kiocb = &req->rw;
+	struct file *filp = kiocb->ki_filp;
+	off_t io_end = kiocb->ki_pos + len;
+
+	if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
+		unsigned long max_pages;
+
+		/* Use 8x RA size as a decent limiter for both reads/writes */
+		max_pages = filp->f_ra.ra_pages;
+		if (!max_pages)
+			max_pages = VM_MAX_READAHEAD >> (PAGE_SHIFT - 10);
+		max_pages *= 8;
+
+		len >>= PAGE_SHIFT;
+		if (async_list->io_pages + len <= max_pages) {
+			req->flags |= REQ_F_SEQ_PREV;
+			async_list->io_pages += len;
+		} else {
+			io_end = 0;
+			async_list->io_pages = 0;
+		}
+	}
+
+	if (async_list->file != filp) {
+		async_list->io_pages = 0;
+		async_list->file = filp;
+	}
+	async_list->io_end = io_end;
+}
+
 static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		       bool force_nonblock, struct io_submit_state *state)
 {
@@ -779,6 +831,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	struct kiocb *kiocb = &req->rw;
 	struct iov_iter iter;
 	struct file *file;
+	size_t iov_count;
 	ssize_t ret;
 
 	ret = io_prep_rw(req, sqe, force_nonblock, state);
@@ -797,16 +850,19 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	if (ret)
 		goto out_fput;
 
-	ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+	iov_count = iov_iter_count(&iter);
+	ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
 	if (!ret) {
 		ssize_t ret2;
 
 		/* Catch -EAGAIN return for forced non-blocking submission */
 		ret2 = call_read_iter(file, kiocb, &iter);
-		if (!force_nonblock || ret2 != -EAGAIN)
+		if (!force_nonblock || ret2 != -EAGAIN) {
 			io_rw_done(kiocb, ret2);
-		else
+		} else {
+			io_async_list_note(READ, req, iov_count);
 			ret = -EAGAIN;
+		}
 	}
 	kfree(iovec);
 out_fput:
@@ -822,6 +878,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	struct kiocb *kiocb = &req->rw;
 	struct iov_iter iter;
 	struct file *file;
+	size_t iov_count;
 	ssize_t ret;
 
 	ret = io_prep_rw(req, sqe, force_nonblock, state);
@@ -829,10 +886,6 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		return ret;
 	file = kiocb->ki_filp;
 
-	ret = -EAGAIN;
-	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
-		goto out_fput;
-
 	ret = -EBADF;
 	if (unlikely(!(file->f_mode & FMODE_WRITE)))
 		goto out_fput;
@@ -844,8 +897,15 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 	if (ret)
 		goto out_fput;
 
-	ret = rw_verify_area(WRITE, file, &kiocb->ki_pos,
-				iov_iter_count(&iter));
+	iov_count = iov_iter_count(&iter);
+
+	ret = -EAGAIN;
+	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT)) {
+		io_async_list_note(WRITE, req, iov_count);
+		goto out_free;
+	}
+
+	ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
 	if (!ret) {
 		/*
 		 * Open-code file_start_write here to grab freeze protection,
@@ -863,6 +923,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,
 		kiocb->ki_flags |= IOCB_WRITE;
 		io_rw_done(kiocb, call_write_iter(file, kiocb, &iter));
 	}
+out_free:
 	kfree(iovec);
 out_fput:
 	if (unlikely(ret))
@@ -1210,6 +1271,21 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
 	return 0;
 }
 
+static struct async_list *io_async_list_from_sqe(struct io_ring_ctx *ctx,
+						 const struct io_uring_sqe *sqe)
+{
+	switch (sqe->opcode) {
+	case IORING_OP_READV:
+	case IORING_OP_READ_FIXED:
+		return &ctx->pending_async[READ];
+	case IORING_OP_WRITEV:
+	case IORING_OP_WRITE_FIXED:
+		return &ctx->pending_async[WRITE];
+	default:
+		return NULL;
+	}
+}
+
 static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
 {
 	return !(sqe->opcode == IORING_OP_READ_FIXED ||
@@ -1219,50 +1295,124 @@ static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
 static void io_sq_wq_submit_work(struct work_struct *work)
 {
 	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
-	struct sqe_submit *s = &req->submit;
-	u64 user_data = s->sqe->user_data;
 	struct io_ring_ctx *ctx = req->ctx;
+	struct mm_struct *cur_mm = NULL;
 	struct files_struct *old_files;
+	struct async_list *async_list;
+	LIST_HEAD(req_list);
 	mm_segment_t old_fs;
-	bool needs_user;
 	int ret;
 
-	 /* Ensure we clear previously set forced non-block flag */
-	req->flags &= ~REQ_F_FORCE_NONBLOCK;
-
 	old_files = current->files;
 	current->files = ctx->sqo_files;
 
+	async_list = io_async_list_from_sqe(ctx, req->submit.sqe);
+restart:
+	do {
+		struct sqe_submit *s = &req->submit;
+		u64 user_data = s->sqe->user_data;
+
+		/* Ensure we clear previously set forced non-block flag */
+		req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+		ret = 0;
+		if (io_sqe_needs_user(s->sqe) && !cur_mm) {
+			if (!mmget_not_zero(ctx->sqo_mm)) {
+				ret = -EFAULT;
+			} else {
+				cur_mm = ctx->sqo_mm;;
+				use_mm(ctx->sqo_mm);
+				old_fs = get_fs();
+				set_fs(USER_DS);
+			}
+		}
+
+		if (!ret)
+			ret = __io_submit_sqe(ctx, req, s, false, NULL);
+		if (ret) {
+			io_cqring_add_event(ctx, user_data, ret, 0);
+			io_free_req(req);
+		}
+		if (!async_list)
+			break;
+		if (!list_empty(&req_list)) {
+			req = list_first_entry(&req_list, struct io_kiocb,
+						list);
+			list_del(&req->list);
+			continue;
+		}
+		if (list_empty(&async_list->list))
+			break;
+
+		req = NULL;
+		spin_lock(&async_list->lock);
+		if (list_empty(&async_list->list)) {
+			spin_unlock(&async_list->lock);
+			break;
+		}
+		list_splice_init(&async_list->list, &req_list);
+		spin_unlock(&async_list->lock);
+
+		req = list_first_entry(&req_list, struct io_kiocb, list);
+		list_del(&req->list);
+	} while (req);
+
 	/*
-	 * If we're doing IO to fixed buffers, we don't need to get/set
-	 * user context
+	 * Rare case of racing with a submitter. If we find the count has
+	 * dropped to zero AND we have pending work items, then restart
+	 * the processing. This is a tiny race window.
 	 */
-	needs_user = io_sqe_needs_user(s->sqe);
-	if (needs_user) {
-		if (!mmget_not_zero(ctx->sqo_mm)) {
-			ret = -EFAULT;
-			goto err;
+	ret = atomic_dec_return(&async_list->cnt);
+	while (!ret && !list_empty(&async_list->list)) {
+		spin_lock(&async_list->lock);
+		atomic_inc(&async_list->cnt);
+		list_splice_init(&async_list->list, &req_list);
+		spin_unlock(&async_list->lock);
+
+		if (!list_empty(&req_list)) {
+			req = list_first_entry(&req_list, struct io_kiocb,
+						list);
+			list_del(&req->list);
+			goto restart;
 		}
-		use_mm(ctx->sqo_mm);
-		old_fs = get_fs();
-		set_fs(USER_DS);
+		ret = atomic_dec_return(&async_list->cnt);
 	}
 
-	ret = __io_submit_sqe(ctx, req, s, false, NULL);
-
-	if (needs_user) {
+	if (cur_mm) {
 		set_fs(old_fs);
-		unuse_mm(ctx->sqo_mm);
-		mmput(ctx->sqo_mm);
-	}
-err:
-	if (ret) {
-		io_cqring_add_event(ctx, user_data, ret, 0);
-		io_free_req(req);
+		unuse_mm(cur_mm);
+		mmput(cur_mm);
 	}
 	current->files = old_files;
 }
 
+/*
+ * See if we can piggy back onto previously submitted work, that is still
+ * running. We currently only allow this if the new request is sequential
+ * to the previous one we punted.
+ */
+static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
+{
+	bool ret = false;
+
+	if (!list)
+		return false;
+	if (!(req->flags & REQ_F_SEQ_PREV))
+		return false;
+	if (!atomic_read(&list->cnt))
+		return false;
+
+	ret = true;
+	spin_lock(&list->lock);
+	list_add_tail(&req->list, &list->list);
+	if (!atomic_read(&list->cnt)) {
+		list_del_init(&req->list);
+		ret = false;
+	}
+	spin_unlock(&list->lock);
+	return ret;
+}
+
 static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 			 struct io_submit_state *state)
 {
@@ -1279,9 +1429,16 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
 
 	ret = __io_submit_sqe(ctx, req, s, true, state);
 	if (ret == -EAGAIN) {
+		struct async_list *list;
+
+		list = io_async_list_from_sqe(ctx, s->sqe);
 		memcpy(&req->submit, s, sizeof(*s));
-		INIT_WORK(&req->work, io_sq_wq_submit_work);
-		queue_work(ctx->sqo_wq, &req->work);
+		if (!io_add_to_prev_work(list, req)) {
+			if (list)
+				atomic_inc(&list->cnt);
+			INIT_WORK(&req->work, io_sq_wq_submit_work);
+			queue_work(ctx->sqo_wq, &req->work);
+		}
 		ret = 0;
 	}
 	if (ret)
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
author maintainer
#68 lore raw

PATCH 18/18 io_uring: add io_uring_event cache hit information

In reply to Jens Axboe

Add hint on whether a read was served out of the page cache, or if it hit media. This is useful for buffered async IO, O_DIRECT reads would never have this set (for obvious reasons).

If the read hit page cache, cqe->flags will have IOCQE_FLAG_CACHEHIT set.

---
 fs/io_uring.c                 | 7 ++++++-
 include/uapi/linux/io_uring.h | 5 +++++
 2 files changed, 11 insertions(+), 1 deletion(-)
fs/io_uring.c +6 -1
index 0240e0a04f6a..9b4b4f9d06a5 100644
@@ -558,11 +558,16 @@ static void io_fput(struct io_kiocb *req)
 static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
 {
 	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
+	unsigned ev_flags = 0;
 
 	kiocb_end_write(kiocb);
 
 	io_fput(req);
-	io_cqring_add_event(req->ctx, req->user_data, res, 0);
+
+	if (res > 0 && (req->flags & REQ_F_FORCE_NONBLOCK))
+		ev_flags = IOCQE_FLAG_CACHEHIT;
+
+	io_cqring_add_event(req->ctx, req->user_data, res, ev_flags);
 	io_free_req(req);
 }
 
include/uapi/linux/io_uring.h +5 -0
index 39d3d3336dce..589b6402081d 100644
@@ -71,6 +71,11 @@ struct io_uring_cqe {
 	__u32	flags;
 };
 
+/*
+ * io_uring_event->flags
+ */
+#define IOCQE_FLAG_CACHEHIT	(1 << 0)	/* IO did not hit media */
+
 /*
  * Magic offsets for the application to mmap the data it needs
  */
Signature · 7 lines
2.17.1

--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to [email protected].  For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>