io_uring IO interface
Subsystems (12)
Jens Axboe
maintainerPATCH v7 1/1 io_uring IO interface
Here's v7 of the io_uring interface. Not a lot of changes this time around, mostly corner cases and error handling that hadn't been exercised yet.
Only new "feature" is an improvement in sequential buffered IO performance. This is done by piggy backing on to existing async work if we're sequential to the previous work item, and it hasn't finished yet. This greatly reduces the overhead, context switches, and increases the performance a lot. Using io_uring-cp, as test program that does cp with io_uring, I saw a 5x improvement in speed when copying a large file. It's now faster than plan 'cp.
The x86/x86-64 system calls were renumbered, so if you are testing, be sure to pull apply the renumbering. The fio and liburing repos have been updated.
The liburing git repo now has a man page for io_uring_register(2), thanks to Jeff Moyer. Outside of that, not much activity there since last time. Clone the liburing repo here:
git://git.kernel.dk/liburing
Patches are against 5.0-rc3, and can also be found in my io_uring branch here:
git://git.kernel.dk/linux-block io_uring
Changes since v6:
- Fix leak of iovec in write path for larger writes
- Fix CQ event filling for error handling of workqueue buffered items
- Improve sequential buffered IO performance
- Fix leak of accounted mem in error path for io_uring_create() (Roman)
- Rebase on v5.0-rc3
- Check for entries == 0 in io_uring_create (Jeff)
- Renumber system calls so they are ready for Arnds unification
Documentation/filesystems/vfs.txt | 3 + arch/x86/entry/syscalls/syscall_32.tbl | 3 + arch/x86/entry/syscalls/syscall_64.tbl | 3 + block/bio.c | 59 +- fs/Makefile | 1 + fs/block_dev.c | 19 +- fs/file.c | 15 +- fs/file_table.c | 9 +- fs/gfs2/file.c | 2 + fs/io_uring.c | 2540 ++++++++++++++++++++++++ fs/iomap.c | 48 +- fs/xfs/xfs_file.c | 1 + include/linux/bio.h | 14 + include/linux/blk_types.h | 1 + include/linux/file.h | 2 + include/linux/fs.h | 6 +- include/linux/iomap.h | 1 + include/linux/sched/user.h | 2 +- include/linux/syscalls.h | 7 + include/uapi/asm-generic/unistd.h | 8 +- include/uapi/linux/io_uring.h | 141 ++ init/Kconfig | 9 + kernel/sys_ni.c | 4 + 23 files changed, 2857 insertions(+), 41 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/18 fs: add an iopoll method to struct file_operations
From: Christoph Hellwig <[email protected]>
This new methods is used to explicitly poll for I/O completion for an iocb. It must be called for any iocb submitted asynchronously (that is with a non-null ki_complete) which has the IOCB_HIPRI flag set.
The method is assisted by a new ki_cookie field in struct iocb to store the polling cookie.
TODO: we can probably union ki_cookie with the existing hint and I/O priority fields to avoid struct kiocb growth.
--- Documentation/filesystems/vfs.txt | 3 +++ include/linux/fs.h | 2 ++ 2 files changed, 5 insertions(+)
Documentation/filesystems/vfs.txt +3 -0
@@ -857,6 +857,7 @@ struct file_operations { ssize_t (*write) (struct file *, const char __user *, size_t, loff_t *); ssize_t (*read_iter) (struct kiocb *, struct iov_iter *); ssize_t (*write_iter) (struct kiocb *, struct iov_iter *); + int (*iopoll)(struct kiocb *kiocb, bool spin); int (*iterate) (struct file *, struct dir_context *); int (*iterate_shared) (struct file *, struct dir_context *); __poll_t (*poll) (struct file *, struct poll_table_struct *);
@@ -902,6 +903,8 @@ otherwise noted. write_iter: possibly asynchronous write with iov_iter as source + iopoll: called when aio wants to poll for completions on HIPRI iocbs + iterate: called when the VFS needs to read the directory contents iterate_shared: called when the VFS needs to read the directory contents
include/linux/fs.h +2 -0
@@ -310,6 +310,7 @@ struct kiocb {int ki_flags;u16 ki_hint;u16 ki_ioprio;/* See linux/ioprio.h */ +unsigned int ki_cookie;/* for ->iopoll */ } __randomize_layout;static inline bool is_sync_kiocb (struct kiocb *kiocb)
@@ -1786,6 +1787,7 @@ struct file_operations {ssize_t (*write) (struct file *,const char __user *,size_t ,loff_t *);ssize_t (*read_iter) (struct kiocb *,struct iov_iter *);ssize_t (*write_iter) (struct kiocb *,struct iov_iter *); +int (*iopoll)(struct kiocb *kiocb,bool spin);int (*iterate) (struct file *,struct dir_context *);int (*iterate_shared) (struct file *,struct dir_context *);__poll_t (*poll) (struct file *,struct poll_table_struct *);
Signature · 1 line
2.17.1
Re: [PATCH 01/18] fs: add an iopoll method to struct file_operations
Quoted 2 lines · Jens Axboe
TODO: we can probably union
ki_cookiewith the existing hint and I/O priority fields to avoid struct kiocb growth.
Please kill this comment. While this should be doable the semantics would be a bit nasty, so unless we have to I'd rather skip this work for now.
Jens Axboe
maintainerRe: [PATCH 01/18] fs: add an iopoll method to struct file_operations
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Jens Axboe
TODO: we can probably union
ki_cookiewith the existing hint and I/O priority fields to avoid struct kiocb growth.Please kill this comment. While this should be doable the semantics would be a bit nasty, so unless we have to I'd rather skip this work for now.
Agree, killed.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 2/18 block: wire up block device iopoll method
In reply to Jens AxboeFrom: Christoph Hellwig <[email protected]>
Just call blk_poll on the iocb cookie, we can derive the block device from the inode trivially.
--- fs/block_dev.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
fs/block_dev.c +10 -0
@@ -293,6 +293,14 @@ struct blkdev_dio {static struct bio_set blkdev_dio_pool; +static int blkdev_iopoll (struct kiocb *kiocb,bool wait) +{ +struct block_device *bdev =I_BDEV (kiocb->ki_filp->f_mapping->host); +struct request_queue *q =bdev_get_queue (bdev); + +return blk_poll (q,READ_ONCE (kiocb->ki_cookie), wait); +} +static void blkdev_bio_end_io (struct bio *bio) {struct blkdev_dio *dio = bio->bi_private;
@@ -410,6 +418,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) bio->bi_opf |=REQ_HIPRI ; qc =submit_bio (bio); +WRITE_ONCE (iocb->ki_cookie, qc);break ; }
@@ -2076,6 +2085,7 @@ const struct file_operations def_blk_fops = { .llseek = block_llseek, .read_iter = blkdev_read_iter, .write_iter = blkdev_write_iter, + .iopoll = blkdev_iopoll, .mmap = generic_file_mmap, .fsync = blkdev_fsync, .unlocked_ioctl = block_ioctl,
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 3/18 block: add bio_set_polled() helper
In reply to Jens AxboeFor the upcoming async polled IO, we can't sleep allocating requests. If we do, then we introduce a deadlock where the submitter already has async polled IO in-flight, but can't wait for them to complete since polled requests must be active found and reaped.
Utilize the helper in the blockdev DIRECT_IO code.
--- fs/block_dev.c | 4 ++-- include/linux/bio.h | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-)
fs/block_dev.c +2 -2
@@ -247,7 +247,7 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,task_io_account_write (ret); }if (iocb->ki_flags &IOCB_HIPRI ) - bio.bi_opf |=REQ_HIPRI ; +bio_set_polled (&bio, iocb); qc =submit_bio (&bio);for (;;) {
@@ -415,7 +415,7 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) nr_pages =iov_iter_npages (iter,BIO_MAX_PAGES );if (!nr_pages) {if (iocb->ki_flags &IOCB_HIPRI ) - bio->bi_opf |=REQ_HIPRI ; +bio_set_polled (bio, iocb); qc =submit_bio (bio);WRITE_ONCE (iocb->ki_cookie, qc);
include/linux/bio.h +14 -0
@@ -823,5 +823,19 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page,#endif /* CONFIG_BLK_DEV_INTEGRITY */ +/* +* Mark a bio as polled. Note that for async polled IO, the caller must +* expect -EWOULDBLOCK if we cannot allocate a request (or other resources). +* We cannot block waiting for requests on polled IO, as those completions +* must be found by the caller. This is different than IRQ driven IO, where +* it's safe to wait for IO to complete. +*/ +static inline void bio_set_polled (struct bio *bio,struct kiocb *kiocb) +{ + bio->bi_opf |=REQ_HIPRI ; +if (!is_sync_kiocb (kiocb)) + bio->bi_opf |=REQ_NOWAIT ; +} +#endif /* CONFIG_BLOCK */ #endif /* __LINUX_BIO_H */
Signature · 1 line
2.17.1
Re: [PATCH 03/18] block: add bio_set_polled() helper
Quoted 8 lines · Jens Axboe
For the upcoming async polled IO, we can't sleep allocating requests. If we do, then we introduce a deadlock where the submitter already has async polled IO in-flight, but can't wait for them to complete since polled requests must be active found and reaped.
Utilize the helper in the blockdev
DIRECT_IOcode.Signed-off-by Jens Axboe <[email protected]>
Looks good,
Jens Axboe
maintainerPATCH 4/18 iomap: wire up the iopoll method
In reply to Jens AxboeFrom: Christoph Hellwig <[email protected]>
Store the request queue the last bio was submitted to in the iocb private data in addition to the cookie so that we find the right block device. Also refactor the common direct I/O bio submission code into a nice little helper.
Modified to use bio_set_polled().
--- fs/gfs2/file.c | 2 ++ fs/iomap.c | 43 ++++++++++++++++++++++++++++--------------- fs/xfs/xfs_file.c | 1 + include/linux/iomap.h | 1 + 4 files changed, 32 insertions(+), 15 deletions(-)
fs/gfs2/file.c +2 -0
@@ -1280,6 +1280,7 @@ const struct file_operations gfs2_file_fops = { .llseek = gfs2_llseek, .read_iter = gfs2_file_read_iter, .write_iter = gfs2_file_write_iter, + .iopoll = iomap_dio_iopoll, .unlocked_ioctl = gfs2_ioctl, .mmap = gfs2_mmap, .open = gfs2_open,
@@ -1310,6 +1311,7 @@ const struct file_operations gfs2_file_fops_nolock = { .llseek = gfs2_llseek, .read_iter = gfs2_file_read_iter, .write_iter = gfs2_file_write_iter, + .iopoll = iomap_dio_iopoll, .unlocked_ioctl = gfs2_ioctl, .mmap = gfs2_mmap, .open = gfs2_open,
fs/iomap.c +28 -15
82 lines not on this page · raw message
fs/xfs/xfs_file.c +1 -0
@@ -1203,6 +1203,7 @@ const struct file_operations xfs_file_operations = { .write_iter = xfs_file_write_iter, .splice_read = generic_file_splice_read, .splice_write = iter_file_splice_write, + .iopoll = iomap_dio_iopoll, .unlocked_ioctl = xfs_file_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = xfs_file_compat_ioctl,
include/linux/iomap.h +1 -0
@@ -162,6 +162,7 @@ typedef int (iomap_dio_end_io_t)(struct kiocb *iocb, ssize_t ret, unsigned flags);ssize_t iomap_dio_rw (struct kiocb *iocb,struct iov_iter *iter,const struct iomap_ops *ops,iomap_dio_end_io_t end_io); +int iomap_dio_iopoll (struct kiocb *kiocb,bool spin);#ifdef CONFIG_SWAP struct file ;
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 5/18 Add io_uring IO interface
In reply to Jens AxboeThe submission queue (SQ) and completion queue (CQ) rings are shared between the application and the kernel. This eliminates the need to copy data back and forth to submit and complete IO.
IO submissions use the io_uring_sqe data structure, and completions are generated in the form of io_uring_sqe data structures. The SQ ring is an index into the io_uring_sqe array, which makes it possible to submit a batch of IOs without them being contiguous in the ring. The CQ ring is always contiguous, as completion events are inherently unordered, and hence any io_uring_cqe entry can point back to an arbitrary submission.
Two new system calls are added for this:
io_uring_setup(entries, params)
Sets up a context for doing async IO. On success, returns a file descriptor that the application can mmap to gain access to the SQ ring, CQ ring, and io_uring_sqes.
io_uring_enter(fd, to_submit, min_complete, flags)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll try and submit new IO. If IORING_ENTER_GETEVENTS is set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to set IORING_ENTER_GETEVENTS and 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.
With this setup, it's possible to do async IO with a single system call. Future developments will enable polled IO with this interface, and polled submission as well. The latter will enable an application to do IO without doing ANY system calls at all.
For IRQ driven IO, an application only needs to enter the kernel for completions if it wants to wait for them to occur.
Each io_uring is backed by a workqueue, to support buffered async IO as well. We will only punt to an async context if the command would need to wait for IO on the device side. Any data that can be accessed directly in the page cache is done inline. This avoids the slowness issue of usual threadpools, since cached data is accessed as quickly as a sync interface.
Sample application: http://git.kernel.dk/cgit/fio/plain/t/io_uring.c
--- arch/x86/entry/syscalls/syscall_32.tbl | 2 + arch/x86/entry/syscalls/syscall_64.tbl | 2 + fs/Makefile | 1 + fs/io_uring.c | 1091 ++++++++++++++++++++++++ include/linux/syscalls.h | 5 + include/uapi/asm-generic/unistd.h | 6 +- include/uapi/linux/io_uring.h | 96 +++ init/Kconfig | 9 + kernel/sys_ni.c | 3 + 9 files changed, 1214 insertions(+), 1 deletion(-) create mode 100644 fs/io_uring.c create mode 100644 include/uapi/linux/io_uring.h
arch/x86/entry/syscalls/syscall_32.tbl +2 -0
@@ -398,3 +398,5 @@ 384 i386 arch_prctl sys_arch_prctl __ia32_compat_sys_arch_prctl 385 i386 io_pgetevents sys_io_pgetevents __ia32_compat_sys_io_pgetevents 386 i386 rseq sys_rseq __ia32_sys_rseq +425 i386 io_uring_setup sys_io_uring_setup __ia32_compat_sys_io_uring_setup +426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter
arch/x86/entry/syscalls/syscall_64.tbl +2 -0
@@ -343,6 +343,8 @@ 332 common statx __x64_sys_statx 333 common io_pgetevents __x64_sys_io_pgetevents 334 common rseq __x64_sys_rseq +425 common io_uring_setup __x64_sys_io_uring_setup +426 common io_uring_enter __x64_sys_io_uring_enter # # x32-specific system call numbers start at 512 to avoid cache impact
fs/Makefile +1 -0
@@ -30,6 +30,7 @@ obj-$(CONFIG_TIMERFD) += timerfd.o obj-$(CONFIG_EVENTFD) += eventfd.o obj-$(CONFIG_USERFAULTFD) += userfaultfd.o obj-$(CONFIG_AIO) += aio.o +obj-$(CONFIG_IO_URING) += io_uring.o obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ obj-$(CONFIG_FILE_LOCKING) += locks.o
fs/io_uring.c new file +1091 -0
1,091 lines not on this page · raw message
include/linux/syscalls.h +5 -0
@@ -69,6 +69,7 @@ struct file_handle;struct sigaltstack ;struct rseq ;union bpf_attr ; +struct io_uring_params ;#include <linux/types.h> #include <linux/aio_abi.h>
@@ -309,6 +310,10 @@ asmlinkage long sys_io_pgetevents_time32(aio_context_t ctx_id,struct io_event __user *events, struct old_timespec32 __user *timeout, const struct __aio_sigset *sig); +asmlinkage long sys_io_uring_setup (u32 entries, +struct io_uring_params __user *p); +asmlinkage long sys_io_uring_enter (unsigned int fd,u32 to_submit, +u32 min_complete,u32 flags);/* fs/xattr.c */ asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
include/uapi/asm-generic/unistd.h +5 -1
@@ -740,9 +740,13 @@ __SC_COMP(__NR_io_pgetevents, sys_io_pgetevents, compat_sys_io_pgetevents) __SYSCALL(__NR_rseq, sys_rseq)#define __NR_kexec_file_load 294 __SYSCALL(__NR_kexec_file_load, sys_kexec_file_load) +#define __NR_io_uring_setup 425 +__SYSCALL(__NR_io_uring_setup, sys_io_uring_setup) +#define __NR_io_uring_enter 426 +__SYSCALL(__NR_io_uring_enter, sys_io_uring_enter)#undef __NR_syscalls -#define __NR_syscalls 295 +#define __NR_syscalls 427/* * 32 bit systems traditionally used different
include/uapi/linux/io_uring.h new file +96 -0
96 lines not on this page · raw message
init/Kconfig +9 -0
@@ -1403,6 +1403,15 @@ config AIO by some high performance threaded applications. Disabling this option saves about 7k. +config IO_URING + bool "Enable IO uring support" if EXPERT + select ANON_INODES + default y + help + This option enables support for the io_uring interface, enabling + applications to submit and completion IO through submission and + completion rings that are shared between the kernel and application. + config ADVISE_SYSCALLS bool "Enable madvise/fadvise syscalls" if EXPERT default y
kernel/sys_ni.c +3 -0
@@ -46,6 +46,9 @@ COND_SYSCALL(io_getevents);COND_SYSCALL (io_pgetevents);COND_SYSCALL_COMPAT (io_getevents);COND_SYSCALL_COMPAT (io_pgetevents); +COND_SYSCALL (io_uring_setup); +COND_SYSCALL_COMPAT (io_uring_setup); +COND_SYSCALL (io_uring_enter);/* fs/xattr.c */
Signature · 1 line
2.17.1
Re: [PATCH 05/18] Add io_uring IO interface
[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
Quoted 4 lines
io_uring_enter(fd, to_submit, min_complete, flags)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'll
Quoted 8 lines
try and submit new IO. If
IORING_ENTER_GETEVENTSis set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to setIORING_ENTER_GETEVENTSand 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
fs/io_uring.c line 109
Quoted 4 lines
fs/io_uring.c new file +4
-0+struct sqe_submit { +const struct io_uring_sqe *sqe; +unsigned index; +};
Can you make sure all the structs use tab indentation for their field names? Maybe even the same for all structs just to be nice to my eyes?
fs/io_uring.c line 355
Quoted 11 lines
fs/io_uring.c new file +11
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT +if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif
I think we can just check in_compat_syscall() here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.
fs/io_uring.c line 464
Quoted 8 lines
fs/io_uring.c new file +8
-0+/* +* IORING_OP_NOP just posts a completion event, nothing else. +*/ +static int io_nop (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_ring_ctx *ctx = req->ctx; + +__io_cqring_add_event (ctx, sqe->user_data,0 ,0 );
Can you explain why not taking the completion lock is safe here? And why we want to have such a somewhat dangerous special case just for the no-op benchmarking aid?
fs/io_uring.c line 588
Quoted 9 lines
fs/io_uring.c new file +9
-0+static bool io_get_sqring (struct io_ring_ctx *ctx,struct sqe_submit *s) +{ +struct io_sq_ring *ring = ctx->sq_ring; +unsigned head; + + head = ctx->cached_sq_head; +smp_rmb (); +if (head ==READ_ONCE (ring->r.tail)) +return false;
Do we really need to optimize the sq_head == tail case so much? Or am I missing why we are using the cached sq head case here? Maybe add some more comments for a start.
Quoted 5 lines
fs/io_uring.c new file +5
-0+static int __io_uring_enter (struct io_ring_ctx *ctx,unsigned to_submit, +unsigned min_complete,unsigned flags) +{ +int ret =0 ; +
fs/io_uring.c line 684
Quoted 10 lines
fs/io_uring.c new file +10
-0+if (to_submit) { + ret =io_ring_submit (ctx, to_submit); +if (ret <0 ) +return ret; + } +if (flags &IORING_ENTER_GETEVENTS ) { +int get_ret; + +if (!ret && to_submit) + min_complete =0 ;
Why do we have this special case? Does it need some documentation?
fs/io_uring.c line 691
Quoted 7 lines
fs/io_uring.c new file +7
-0+ + get_ret =io_cqring_wait (ctx, min_complete); +if (get_ret <0 && !ret) + ret = get_ret; + } + +return ret;
Maybe using different names and slightly different semantics for the return values would clear some of this up?
if (to_submit) {
submitted = io_ring_submit(ctx, to_submit);
if (submitted < 0)
return submitted;
}
if (flags & IORING_ENTER_GETEVENTS) {
...
ret = io_cqring_wait(ctx, min_complete);
}
return submitted ? submitted : ret;fs/io_uring.c line 726
Quoted 3 lines
fs/io_uring.c new file +2
-0+static int io_sq_offload_start(struct io_ring_ctx *ctx) +static void io_sq_offload_stop(struct io_ring_ctx *ctx)
Can we just merge these two functions into the callers? Currently the flow is a little odd with these helpers that don't seem to be too clear about their responsibilities.
Quoted 4 lines
fs/io_uring.c new file +4
-0+static void io_free_scq_urings (struct io_ring_ctx *ctx) +{ +if (ctx->sq_ring) { +page_frag_free (ctx->sq_ring);
fs/io_uring.c line 789
Quoted 10 lines
fs/io_uring.c new file +10
-0+ ctx->sq_ring =NULL ; + } +if (ctx->sq_sqes) { +page_frag_free (ctx->sq_sqes); + ctx->sq_sqes =NULL ; + } +if (ctx->cq_ring) { +page_frag_free (ctx->cq_ring); + ctx->cq_ring =NULL ; + }
Why is this using the page_frag helpers? Also the callers just free these ctx structure, so there isn't much of a point zeroing them out.
Also I'd be tempted to open code the freeing in io_allocate_scq_urings instead of caling the helper, which would avoid the NULL checks and make the error handling code a little more obvious.
fs/io_uring.c line 895
Quoted 2 lines
fs/io_uring.c new file +2
-0+if (mutex_trylock (&ctx->uring_lock)) { + ret =__io_uring_enter (ctx, to_submit, min_complete, flags);
do we even need the separate __io_uring_enter helper?
Quoted 1 line
+static void
io_fill_offsets(struct io_uring_params *p)
Do we really need this as a separate helper?
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 2 lines · Christoph Hellwig
[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
I already did review with Arnd on those parts, I'll add linux-api and linux-man for the next posting.
Quoted 13 lines
Quoted 4 lines
io_uring_enter(fd, to_submit, min_complete, flags)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'llQuoted 8 lines
try and submit new IO. If
IORING_ENTER_GETEVENTSis set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to setIORING_ENTER_GETEVENTSand 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
I guess we do.
Quoted 8 lines
fs/io_uring.c line 109
Quoted 4 lines
fs/io_uring.c new file +4
-0+struct sqe_submit { +const struct io_uring_sqe *sqe; +unsigned index; +};Can you make sure all the structs use tab indentation for their field names? Maybe even the same for all structs just to be nice to my eyes?
Sure, fixed.
Quoted 7 lines
Quoted 7 lines
fs/io_uring.c new file +7
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT
Quoted 8 lines
fs/io_uring.c line 355
Quoted 4 lines
fs/io_uring.c new file +4
-0+if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif I think we can just check
in_compat_syscall()here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.
Good point, I'll switch to using that so we don't have to track it.
Quoted 4 lines
Quoted 4 lines
fs/io_uring.c new file +4
-0+/* +* IORING_OP_NOP just posts a completion event, nothing else. +*/ +static int io_nop (struct io_kiocb *req,const struct io_uring_sqe *sqe)
Quoted 8 lines
fs/io_uring.c line 464
Quoted 4 lines
fs/io_uring.c new file +4
-0+{ +struct io_ring_ctx *ctx = req->ctx; + +__io_cqring_add_event (ctx, sqe->user_data,0 ,0 );Can you explain why not taking the completion lock is safe here? And why we want to have such a somewhat dangerous special case just for the no-op benchmarking aid?
Was going to say it's safe because we always fill the ring inside the ring mutex, but that won't work if we intermingle with non-polled IO. I'll switch it to using the normal locked variant.
Quoted 5 lines
Quoted 5 lines
fs/io_uring.c new file +5
-0+static bool io_get_sqring (struct io_ring_ctx *ctx,struct sqe_submit *s) +{ +struct io_sq_ring *ring = ctx->sq_ring; +unsigned head; +
Quoted 8 lines
fs/io_uring.c line 588
Quoted 4 lines
fs/io_uring.c new file +4
-0+ head = ctx->cached_sq_head; +smp_rmb (); +if (head ==READ_ONCE (ring->r.tail)) +return false;Do we really need to optimize the
sq_head== tail case so much? Or am I missing why we are using the cached sq head case here? Maybe add some more comments for a start.
It basically serves two purposes:
1) When we grab multiple events, only have to update the actual ring tail once. This is a big deal, especially on archs where the barriers are more expensive.
2) It means the kernel tracks the sq tail and cq head, instead of completely relying on the application. This seems a much saner choice.
Quoted 9 lines
Quoted 9 lines
fs/io_uring.c new file +9
-0+static int __io_uring_enter (struct io_ring_ctx *ctx,unsigned to_submit, +unsigned min_complete,unsigned flags) +{ +int ret =0 ; + +if (to_submit) { + ret =io_ring_submit (ctx, to_submit); +if (ret <0 ) +return ret;
Quoted 8 lines
fs/io_uring.c line 684
Quoted 6 lines
fs/io_uring.c new file +6
-0+ } +if (flags &IORING_ENTER_GETEVENTS ) { +int get_ret; + +if (!ret && to_submit) + min_complete =0 ;Why do we have this special case? Does it need some documentation?
At least for polled IO, if we don't submit what we were asked to, then we can't reliably poll for the passed in number of events. The min_complete from the application could very well include the expectation that to_submit were submitted as well.
I'll add a comment.
Quoted 11 lines
fs/io_uring.c line 691
Quoted 7 lines
fs/io_uring.c new file +7
-0+ + get_ret =io_cqring_wait (ctx, min_complete); +if (get_ret <0 && !ret) + ret = get_ret; + } + +return ret;Maybe using different names and slightly different semantics for the return values would clear some of this up?
Quoted 11 lines
if (to_submit) {
submitted = io_ring_submit(ctx, to_submit);
if (submitted < 0)
return submitted;
}
if (flags & IORING_ENTER_GETEVENTS) {
...
ret = io_cqring_wait(ctx, min_complete);
}
return submitted ? submitted : ret;That would probably make it more readable, I'll make this change.
Quoted 7 lines
fs/io_uring.c line 726
Quoted 3 lines
fs/io_uring.c new file +2
-0+static int io_sq_offload_start(struct io_ring_ctx *ctx) +static void io_sq_offload_stop(struct io_ring_ctx *ctx)Can we just merge these two functions into the callers? Currently the flow is a little odd with these helpers that don't seem to be too clear about their responsibilities.
In the initial patch I agree, but with the later thread addition, I like having it in a separate helper. At least for the start, the top side is more trivial.
Quoted 9 lines
Quoted 9 lines
fs/io_uring.c new file +9
-0+static void io_free_scq_urings (struct io_ring_ctx *ctx) +{ +if (ctx->sq_ring) { +page_frag_free (ctx->sq_ring); + ctx->sq_ring =NULL ; + } +if (ctx->sq_sqes) { +page_frag_free (ctx->sq_sqes); + ctx->sq_sqes =NULL ;
Quoted 8 lines
fs/io_uring.c line 789
Quoted 5 lines
fs/io_uring.c new file +5
-0+ } +if (ctx->cq_ring) { +page_frag_free (ctx->cq_ring); + ctx->cq_ring =NULL ; + }Why is this using the
page_fraghelpers? Also the callers just free these ctx structure, so there isn't much of a point zeroing them out.
Why not use the page frag helpers? No point in open-coding it. I can kill the zeroing, double call would be a bug anyway.
Quoted 3 lines
Also I'd be tempted to open code the freeing in
io_allocate_scq_uringsinstead of caling the helper, which would avoid the NULL checks and make the error handling code a little more obvious.
OK
Quoted 4 lines
fs/io_uring.c line 895
Quoted 2 lines
fs/io_uring.c new file +2
-0+if (mutex_trylock (&ctx->uring_lock)) { + ret =__io_uring_enter (ctx, to_submit, min_complete, flags);do we even need the separate
__io_uring_enterhelper?
I like having it nicely separated.
Quoted 3 lines
Quoted 1 line
+static void
io_fill_offsets(struct io_uring_params *p)Do we really need this as a separate helper?
That does seem pointless, folded in.
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Re: [PATCH 05/18] Add io_uring IO interface
Quoted 18 lines · Jens Axboe
Quoted 9 lines
Quoted 9 lines
fs/io_uring.c new file +9
-0+static void io_free_scq_urings (struct io_ring_ctx *ctx) +{ +if (ctx->sq_ring) { +page_frag_free (ctx->sq_ring); + ctx->sq_ring =NULL ; + } +if (ctx->sq_sqes) { +page_frag_free (ctx->sq_sqes); + ctx->sq_sqes =NULL ;Quoted 8 lines
fs/io_uring.c line 789
Quoted 5 lines
fs/io_uring.c new file +5
-0+ } +if (ctx->cq_ring) { +page_frag_free (ctx->cq_ring); + ctx->cq_ring =NULL ; + }Why is this using the
page_fraghelpers? Also the callers just free these ctx structure, so there isn't much of a point zeroing them out.
Quoted 2 lines
Why not use the page frag helpers? No point in open-coding it. I can kill the zeroing, double call would be a bug anyway.
Because they are at a different level of abstraction, and someone might change the implementation, and is unlikely to catch the io_uring mix of interfaces. If you think this is really useful we should also export the helpers under a different name and with documentation. (and add a __get_free_pages version that returns a pointer..)
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 19 lines · Christoph Hellwig
Quoted 18 lines · Jens Axboe
Quoted 17 lines
Quoted 4 lines
fs/io_uring.c new file +4
-0+static void io_free_scq_urings (struct io_ring_ctx *ctx) +{ +if (ctx->sq_ring) { +page_frag_free (ctx->sq_ring);fs/io_uring.c line 789
Quoted 10 lines
fs/io_uring.c new file +10
-0+ ctx->sq_ring =NULL ; + } +if (ctx->sq_sqes) { +page_frag_free (ctx->sq_sqes); + ctx->sq_sqes =NULL ; + } +if (ctx->cq_ring) { +page_frag_free (ctx->cq_ring); + ctx->cq_ring =NULL ; + }Why is this using the
page_fraghelpers? Also the callers just free these ctx structure, so there isn't much of a point zeroing them out.
Quoted 8 lines
Quoted 2 lines
Why not use the page frag helpers? No point in open-coding it. I can kill the zeroing, double call would be a bug anyway.
Because they are at a different level of abstraction, and someone might change the implementation, and is unlikely to catch the
io_uringmix of interfaces. If you think this is really useful we should also export the helpers under a different name and with documentation. (and add a__get_free_pagesversion that returns a pointer..)
Fair enough, I'll avoid using the page_frag_free().
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
In reply to Christoph HellwigQuoted 16 lines · Christoph Hellwig
[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
Quoted 4 lines
io_uring_enter(fd, to_submit, min_complete, flags)
Initiates IO against the rings mapped to this fd, or waits for them to complete, or both. The behavior is controlled by the parameters passed in. If 'to_submit' is non-zero, then we'llQuoted 8 lines
try and submit new IO. If
IORING_ENTER_GETEVENTSis set, the kernel will wait for 'min_complete' events, if they aren't already available. It's valid to setIORING_ENTER_GETEVENTSand 'min_complete' == 0 at the same time, this allows the kernel to return already completed events without waiting for them. This is useful only for polling, as for IRQ driven IO, the application can just check the CQ ring without entering the kernel.
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Is there any way to avoid passing in the sigset_t size? If it's just a 32-bit/64-bit thing, surely the in_compat_syscall() could cover it? Or are there other cases that need to be catered to?
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Re: [PATCH 05/18] Add io_uring IO interface
Quoted 7 lines · Jens Axboe
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Is there any way to avoid passing in the
sigset_tsize? If it's just a 32-bit/64-bit thing, surely thein_compat_syscall()could cover it? Or are there other cases that need to be catered to?
As far as I can tell we never look at it, never looked at it and don't have any plans to look at it anytime soon. But when I tried to omit it for io_pgetevents I got stong pushback and thus had to add the crazy double indirection calling convention.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Arnd Bergmann
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 6 lines · Christoph Hellwig
Quoted 4 lines · Jens Axboe
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Quoted 8 lines
Quoted 3 lines
Is there any way to avoid passing in the
sigset_tsize? If it's just a 32-bit/64-bit thing, surely thein_compat_syscall()could cover it? Or are there other cases that need to be catered to?As far as I can tell we never look at it, never looked at it and don't have any plans to look at it anytime soon. But when I tried to omit it for
io_pgeteventsI got stong pushback and thus had to add the crazy double indirection calling convention.
Deepa has recently reworked the handling for the sigset_t handling to be more consistent. As I understand it, we only ever check the size argument to ensure that user and kernel space agree on the size, as it there had been some historic differences.
If you pass a signal mask to a syscall, you should now just use the set_user_sigmask()/set_compat_user_sigmask()/restore_user_sigmask() helpers. The compat version is required for the incompatible bit order between 32-bit and 64-bit big-endian architectures (on little-endian, compat and native signal masks are compatible), and to deal with the one architecture that has an _NSIG defined to something other than 64:
MIPS uses 128 because of a historic accident.
I think Deepa originally suggested combining set_user_sigmask() and set_compat_user_sigmask(), using an in_compat_syscall() check. This would let us simplify a number of compat syscalls (ppoll, ppoll_time32, pselect6, pselect6_time32, epoll_pwait() io_pgetevents_time64, io_pgetevents_time32). I advised against changing it at the time for consistency with other compat syscalls, but it's something we can still do now.
There was a recent discussion about the size of sigset_t in glibc, which is 1024 bits there instead of 64 bits in the kernel, the idea being that the kernel might eventually grow more signals at some point in the future, as we did when we extended from 32 to 64 a long time ago with the addition of the rt_sig* signals; see the thread around
https://www.spinics.net/lists/linux-snps-arc/msg04860.html
Arnd
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 29 lines · Arnd Bergmann
Quoted 6 lines · Christoph Hellwig
Quoted 4 lines · Jens Axboe
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Quoted 8 lines
Quoted 3 lines
Is there any way to avoid passing in the
sigset_tsize? If it's just a 32-bit/64-bit thing, surely thein_compat_syscall()could cover it? Or are there other cases that need to be catered to?As far as I can tell we never look at it, never looked at it and don't have any plans to look at it anytime soon. But when I tried to omit it for
io_pgeteventsI got stong pushback and thus had to add the crazy double indirection calling convention.Deepa has recently reworked the handling for the
sigset_thandling to be more consistent. As I understand it, we only ever check the size argument to ensure that user and kernel space agree on the size, as it there had been some historic differences.If you pass a signal mask to a syscall, you should now just use the set_user_sigmask()/set_compat_user_sigmask()/restore_user_sigmask() helpers. The compat version is required for the incompatible bit order between 32-bit and 64-bit big-endian architectures (on little-endian, compat and native signal masks are compatible), and to deal with the one architecture that has an
_NSIGdefined to something other than 64:
MIPS uses 128 because of a historic accident.
Quoted 7 lines
I think Deepa originally suggested combining
set_user_sigmask()andset_compat_user_sigmask(), using anin_compat_syscall()check. This would let us simplify a number of compat syscalls (ppoll,ppoll_time32, pselect6,pselect6_time32,epoll_pwait()io_pgetevents_time64,io_pgetevents_time32). I advised against changing it at the time for consistency with other compat syscalls, but it's something we can still do now.
That's good info. I am currently using set_user_sigmask() for it. I'd really like to avoid having to pass in a sigset_t size for the system call, however. What's the best way of achieving that? Can I get away with doing something like this:
if (in_compat_syscall()) {
const compat_sigset_t __user *compat_sig;
compat_sig = (const compat_sigset_t __user *) sig;
ret = set_compat_user_sigmask(compat_sig, &ksigmask,
&sigsaved, _NSIG_WORDS);
} else {
ret = set_user_sigmask(sig, &ksigmask, &sigsaved,
_NSIG_WORDS);
}Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Arnd Bergmann
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 11 lines · Jens Axboe
Quoted 10 lines · Arnd Bergmann
Quoted 9 lines · Christoph Hellwig
Quoted 8 lines · Jens Axboe
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Is there any way to avoid passing in the
sigset_tsize? If it's just a 32-bit/64-bit thing, surely thein_compat_syscall()could cover it? Or are there other cases that need to be catered to?
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
As far as I can tell we never look at it, never looked at it and don't have any plans to look at it anytime soon. But when I tried to omit it for
io_pgeteventsI got stong pushback and thus had to add the crazy double indirection calling convention.That's good info. I am currently using
set_user_sigmask()for it. I'd really like to avoid having to pass in asigset_tsize for the system call, however.
I really wouldn't do it, given that all other signal handling interfaces are prepared for longer signal masks. You /could/ probably extend it later with a flags bit to signify a longer mask instead of using the entire register to hold the bit length, it just seems really inconsistent with all other system calls.
Arnd
Quoted 13 lines
What's the best way of achieving that? Can I get
away with doing something like this:if (in_compat_syscall()) { const compat_sigset_t __user *compat_sig; compat_sig = (const compat_sigset_t __user *) sig; ret = set_compat_user_sigmask(compat_sig, &ksigmask, &sigsaved, _NSIG_WORDS); } else { ret = set_user_sigmask(sig, &ksigmask, &sigsaved, _NSIG_WORDS); }
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 21 lines · Arnd Bergmann
Quoted 11 lines · Jens Axboe
Quoted 10 lines · Arnd Bergmann
Quoted 9 lines · Christoph Hellwig
Quoted 8 lines · Jens Axboe
Quoted 3 lines
Especially with poll support now in the series, don't we need a ѕigmask argument similar to pselect/ppoll/io_pgetevents now to deal with signal blocking during waiting for events?
Is there any way to avoid passing in the
sigset_tsize? If it's just a 32-bit/64-bit thing, surely thein_compat_syscall()could cover it? Or are there other cases that need to be catered to?Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
As far as I can tell we never look at it, never looked at it and don't have any plans to look at it anytime soon. But when I tried to omit it for
io_pgeteventsI got stong pushback and thus had to add the crazy double indirection calling convention.That's good info. I am currently using
set_user_sigmask()for it. I'd really like to avoid having to pass in asigset_tsize for the system call, however.
Quoted 5 lines
I really wouldn't do it, given that all other signal handling interfaces are prepared for longer signal masks. You /could/ probably extend it later with a flags bit to signify a longer mask instead of using the entire register to hold the bit length, it just seems really inconsistent with all other system calls.
Damnit! OK, I'll keep what I currently have then.
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Arnd Bergmann
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 6 lines · Jens Axboe
Quoted 5 lines · Arnd Bergmann
Quoted 3 lines · Jens Axboe
That's good info. I am currently using
set_user_sigmask()for it. I'd really like to avoid having to pass in asigset_tsize for the system call, however.
Quoted 7 lines
Quoted 5 lines
I really wouldn't do it, given that all other signal handling interfaces are prepared for longer signal masks. You /could/ probably extend it later with a flags bit to signify a longer mask instead of using the entire register to hold the bit length, it just seems really inconsistent with all other system calls.
Damnit! OK, I'll keep what I currently have then.
As long as you stay within the 6-argument syscall contraints, the cost of passing the length is basically free, right?
Is there anything else you are worried about?
Arnd
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 13 lines · Arnd Bergmann
Quoted 12 lines · Jens Axboe
Quoted 10 lines · Arnd Bergmann
Quoted 3 lines · Jens Axboe
That's good info. I am currently using
set_user_sigmask()for it. I'd really like to avoid having to pass in asigset_tsize for the system call, however.I really wouldn't do it, given that all other signal handling interfaces are prepared for longer signal masks. You /could/ probably extend it later with a flags bit to signify a longer mask instead of using the entire register to hold the bit length, it just seems really inconsistent with all other system calls.
Quoted 6 lines
Quoted 1 line
Damnit! OK, I'll keep what I currently have then.
As long as you stay within the 6-argument syscall contraints, the cost of passing the length is basically free, right?
Is there anything else you are worried about?
My main worry is not the extra argument, more that we're at capacity for the system call. If we wanted to add a timeout parameter, then we'd need to bundle them up, which sucks.
But I think we're fine, I'll go with what I have.
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Arnd Bergmann
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 17 lines · Jens Axboe
Quoted 15 lines · Arnd Bergmann
Quoted 13 lines · Jens Axboe
Quoted 10 lines · Arnd Bergmann
Quoted 3 lines · Jens Axboe
That's good info. I am currently using
set_user_sigmask()for it. I'd really like to avoid having to pass in asigset_tsize for the system call, however.I really wouldn't do it, given that all other signal handling interfaces are prepared for longer signal masks. You /could/ probably extend it later with a flags bit to signify a longer mask instead of using the entire register to hold the bit length, it just seems really inconsistent with all other system calls.
Damnit! OK, I'll keep what I currently have then.
Quoted 8 lines
Quoted 4 lines
As long as you stay within the 6-argument syscall contraints, the cost of passing the length is basically free, right?
Is there anything else you are worried about?
My main worry is not the extra argument, more that we're at capacity for the system call. If we wanted to add a timeout parameter, then we'd need to bundle them up, which sucks.
Ok, got it. If it turns out that we do need the timeout argument after all, you could avoid one indirection level by grouping the sigset_t with min_complete, timeout_ns and/or sigset_size, as long as the sigset_t comes last (similar to struct sigaction).
Another (also awkward) trick might be to combine min_complete and sigset_size into a 32-bit integer argument, as each of them can fit into 16 bits.
Arnd
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Andy Lutomirski
maintainerRe: [PATCH 05/18] Add io_uring IO interface
In reply to Christoph HellwigQuoted 4 lines · Christoph Hellwig
[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
Quoted 16 lines
fs/io_uring.c line 355
Quoted 11 lines
fs/io_uring.c new file +11
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT +if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif I think we can just check
in_compat_syscall()here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.
Since this whole API is new, I don't suppose you could introduce a struct iovec64 or similar and just make the ABI be identical for 64-bit and 32-bit code?
--Andy
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 22 lines · Andy Lutomirski
Quoted 4 lines · Christoph Hellwig
[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
Quoted 16 lines
fs/io_uring.c line 355
Quoted 11 lines
fs/io_uring.c new file +11
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT +if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif I think we can just check
in_compat_syscall()here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.
Quoted 3 lines
Since this whole API is new, I don't suppose you could introduce a struct iovec64 or similar and just make the ABI be identical for 64-bit and 32-bit code?
Sure, that would be straight forward. Is there a strong reason to do so outside of "that would be nice"? It's not like it's a huge amount of code.
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Re: [PATCH 05/18] Add io_uring IO interface
Quoted 3 lines · Jens Axboe
Sure, that would be straight forward. Is there a strong reason to do so outside of "that would be nice"? It's not like it's a huge amount of code.
And it would be really painful for userspace. Because now you can't pass struct iovec through from a higher level, but will instead of to copy the iovec to a different type in the submission path.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Arnd Bergmann
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 9 lines · Christoph Hellwig
Quoted 3 lines · Jens Axboe
Sure, that would be straight forward. Is there a strong reason to do so outside of "that would be nice"? It's not like it's a huge amount of code.
And it would be really painful for userspace. Because now you can't pass struct iovec through from a higher level, but will instead of to copy the iovec to a different type in the submission path.
Agreed. However, if we decide to add the in_compat_syscall() check to set_user_sigmask()/set_compat_user_sigmask(), we probably want to do the same thing in import_iovec()/compat_import_iovec() and rw_copy_check_uvector()/compat_rw_copy_check_uvector().
Arnd
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Andy Lutomirski
maintainerRe: [PATCH 05/18] Add io_uring IO interface
In reply to Jens AxboeQuoted 24 lines
Quoted 24 lines
Quoted 23 lines · Jens Axboe
On 1/28/19 5:47 PM, Andy Lutomirski wrote:
On Mon, Jan 28, 2019 at 6:57 AM Christoph Hellwig <[email protected]> wrote:[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
fs/io_uring.c line 355
Quoted 11 lines
fs/io_uring.c new file +11
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT +if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif I think we can just check
in_compat_syscall()here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.
Quoted 7 lines
Quoted 3 lines
Since this whole API is new, I don't suppose you could introduce a struct iovec64 or similar and just make the ABI be identical for 64-bit and 32-bit code?
Sure, that would be straight forward. Is there a strong reason to do so outside of "that would be nice"? It's not like it's a huge amount of code.
Here are some minor-ish benefits:
- It avoids having a code path that is only used with 32 bit code on
64 bit kernels and is therefore rarely tested. (In this particular case, the code path doesn't diverge much, but for most compat syscalls, it's almost an entirely separate implementation of the main syscall code.)
- It makes life easier for tools like strace.
- It minimizes the chance of making a giant mess on x32, which isn't
really native or compat.
--Andy
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerRe: [PATCH 05/18] Add io_uring IO interface
Quoted 40 lines · Andy Lutomirski
Quoted 24 lines
Quoted 24 lines
Quoted 23 lines · Jens Axboe
On 1/28/19 5:47 PM, Andy Lutomirski wrote:
On Mon, Jan 28, 2019 at 6:57 AM Christoph Hellwig <[email protected]> wrote:[please make sure linux-api and linux-man are CCed on new syscalls so that we get API experts to review them]
fs/io_uring.c line 355
Quoted 11 lines
fs/io_uring.c new file +11
-0+static int io_import_iovec (struct io_ring_ctx *ctx,int rw, +const struct io_uring_sqe *sqe, +struct iovec **iovec,struct iov_iter *iter) +{ +void __user *buf =u64_to_user_ptr (sqe->addr); + +#ifdef CONFIG_COMPAT +if (ctx->compat) +return compat_import_iovec (rw, buf, sqe->len,UIO_FASTIOV , + iovec, iter); +#endif I think we can just check
in_compat_syscall()here, which means we can kill the ->compat member, and the separate compat version of the setup syscall.Quoted 7 lines
Quoted 3 lines
Since this whole API is new, I don't suppose you could introduce a struct iovec64 or similar and just make the ABI be identical for 64-bit and 32-bit code?
Sure, that would be straight forward. Is there a strong reason to do so outside of "that would be nice"? It's not like it's a huge amount of code.
Here are some minor-ish benefits:
- It avoids having a code path that is only used with 32 bit code on
64 bit kernels and is therefore rarely tested. (In this particular case, the code path doesn't diverge much, but for most compat syscalls, it's almost an entirely separate implementation of the main syscall code.)
Quoted 4 lines
- It makes life easier for tools like strace.
- It minimizes the chance of making a giant mess on x32, which isn't
really native or compat.
Not really anything major here, at least not to the extent that suffering the pain of having a different iovec for this is warranted.
Signature · 7 lines
Jens Axboe -- To unsubscribe, send a message with 'unsubscribe linux-aio' in the body to [email protected]. For more info on Linux AIO, see: http://www.kvack.org/aio/ Don't email: <a href=mailto:"[email protected]">[email protected]</a>
Jens Axboe
maintainerPATCH 6/18 io_uring: add fsync support
In reply to Jens AxboeFrom: Christoph Hellwig <[email protected]>
Add a new fsync opcode, which either syncs a range if one is passed, or the whole file if the offset and length fields are both cleared to zero. A flag is provided to use fdatasync semantics, that is only force out metadata which is required to retrieve the file data, but not others like metadata.
--- fs/io_uring.c | 34 ++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 8 +++++++- 2 files changed, 41 insertions(+), 1 deletion(-)
fs/io_uring.c +34 -0
@@ -4,6 +4,7 @@* supporting fast/efficient IO. * * Copyright (C) 2018-2019 Jens Axboe +* Copyright (c) 2018-2019 Christoph Hellwig */ #include <linux/kernel.h> #include <linux/init.h>
@@ -466,6 +467,36 @@ static int io_nop(struct io_kiocb *req, const struct io_uring_sqe *sqe)return 0 ; } +static int io_fsync (struct io_kiocb *req,const struct io_uring_sqe *sqe, +bool force_nonblock) +{ +struct io_ring_ctx *ctx = req->ctx; +loff_t end = sqe->off + sqe->len; +struct file *file; +int ret; + +/* fsync always requires a blocking context */ +if (force_nonblock) +return -EAGAIN ; + +if (unlikely (sqe->addr || sqe->ioprio)) +return -EINVAL ; +if (unlikely (sqe->fsync_flags & ~IORING_FSYNC_DATASYNC )) +return -EINVAL ; + + file =fget (sqe->fd); +if (unlikely (!file)) +return -EBADF ; + + ret =vfs_fsync_range (file, sqe->off, end >0 ? end :LLONG_MAX , + sqe->fsync_flags &IORING_FSYNC_DATASYNC ); + +fput (file); +io_cqring_add_event (ctx, sqe->user_data, ret,0 ); +io_free_req (req); +return 0 ; +} +static int __io_submit_sqe (struct io_ring_ctx *ctx,struct io_kiocb *req,struct sqe_submit *s,bool force_nonblock) {
@@ -487,6 +518,9 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,case IORING_OP_WRITEV : ret =io_write (req, sqe, force_nonblock);break ; +case IORING_OP_FSYNC : + ret =io_fsync (req, sqe, force_nonblock); +break ;default : ret = -EINVAL ;break ;
include/uapi/linux/io_uring.h +7 -1
@@ -26,7 +26,7 @@ struct io_uring_sqe {__u32 len;/* buffer size or number of iovecs */ union {__kernel_rwf_t rw_flags; -__u32 __resv; +__u32 fsync_flags; };__u64 user_data;/* data to be passed back at completion time */ __u64 __pad2[3 ];
@@ -35,6 +35,12 @@ struct io_uring_sqe {#define IORING_OP_NOP 0#define IORING_OP_READV 1#define IORING_OP_WRITEV 2 +#define IORING_OP_FSYNC 3 + +/* +* sqe->fsync_flags +*/ +#define IORING_FSYNC_DATASYNC (1 << 0)/* * IO completion data structure (Completion Queue Entry)
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 7/13 io_uring: add support for pre-mapped user IO buffers
In reply to Jens AxboeIf we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.
To utilize this feature, the application must call io_uring_register() after having setup an io_uring context, passing in IORING_REGISTER_BUFFERS as the opcode. The argument must be a pointer to an iovec array, and the nr_args should contain how many iovecs the application wishes to map.
If successful, these buffers are now mapped into the kernel, eligible for IO. To use these fixed buffers, the application must use the IORING_OP_READ_FIXED and IORING_OP_WRITE_FIXED opcodes, and then set sqe->index to the desired buffer index. sqe->addr..sqe->addr+seq->len must point to somewhere inside the indexed buffer.
The application may register buffers throughout the lifetime of the io_uring context. It can call io_uring_register() with IORING_UNREGISTER_BUFFERS as the opcode to unregister the current set of buffers, and then register a new set. The application need not unregister buffers explicitly before shutting down the io_uring context.
It's perfectly valid to setup a larger buffer, and then sometimes only use parts of it for an IO. As long as the range is within the originally mapped region, it will work just fine.
For now, buffers must not be file backed. If file backed buffers are passed in, the registration will fail with -1/EOPNOTSUPP. This restriction may be relaxed in the future.
RLIMIT_MEMLOCK is used to check how much memory we can pin. A somewhat arbitrary 1G per buffer size is also imposed.
--- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + fs/io_uring.c | 357 ++++++++++++++++++++++++- include/linux/sched/user.h | 2 +- include/linux/syscalls.h | 2 + include/uapi/linux/io_uring.h | 13 +- kernel/sys_ni.c | 1 + 7 files changed, 364 insertions(+), 13 deletions(-)
arch/x86/entry/syscalls/syscall_32.tbl +1 -0
@@ -400,3 +400,4 @@ 386 i386 rseq sys_rseq __ia32_sys_rseq 425 i386 io_uring_setup sys_io_uring_setup __ia32_compat_sys_io_uring_setup 426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter +427 i386 io_uring_register sys_io_uring_register __ia32_sys_io_uring_register
arch/x86/entry/syscalls/syscall_64.tbl +1 -0
@@ -345,6 +345,7 @@ 334 common rseq __x64_sys_rseq 425 common io_uring_setup __x64_sys_io_uring_setup 426 common io_uring_enter __x64_sys_io_uring_enter +427 common io_uring_register __x64_sys_io_uring_register # # x32-specific system call numbers start at 512 to avoid cache impact
fs/io_uring.c +346 -11
451 lines not on this page · raw message
include/linux/sched/user.h +1 -1
@@ -40,7 +40,7 @@ struct user_struct {kuid_t uid;#if defined(CONFIG_PERF_EVENTS ) || defined(CONFIG_BPF_SYSCALL ) || \ - defined(CONFIG_NET ) + defined(CONFIG_NET ) || defined(CONFIG_IO_URING )atomic_long_t locked_vm;#endif
include/linux/syscalls.h +2 -0
@@ -314,6 +314,8 @@ asmlinkage long sys_io_uring_setup(u32 entries, struct io_uring_params __user *p);asmlinkage long sys_io_uring_enter (unsigned int fd,u32 to_submit,u32 min_complete,u32 flags); +asmlinkage long sys_io_uring_register (unsigned int fd,unsigned int op, +void __user *arg,unsigned int nr_args);/* fs/xattr.c */ asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
include/uapi/linux/io_uring.h +12 -1
@@ -29,7 +29,10 @@ struct io_uring_sqe {__u32 fsync_flags; };__u64 user_data;/* data to be passed back at completion time */ -__u64 __pad2[3 ]; +union { +__u16 buf_index;/* index into fixed buffers, if used */ +__u64 __pad2[3 ]; + }; };/*
@@ -41,6 +44,8 @@ struct io_uring_sqe {#define IORING_OP_READV 1#define IORING_OP_WRITEV 2#define IORING_OP_FSYNC 3 +#define IORING_OP_READ_FIXED 4 +#define IORING_OP_WRITE_FIXED 5/* * sqe->fsync_flags
@@ -104,4 +109,10 @@ struct io_uring_params { struct io_cqring_offsets cq_off; }; +/* +* io_uring_register(2) opcodes and arguments +*/ +#define IORING_REGISTER_BUFFERS 0 +#define IORING_UNREGISTER_BUFFERS 1 +#endif
kernel/sys_ni.c +1 -0
@@ -49,6 +49,7 @@ COND_SYSCALL_COMPAT(io_pgetevents);COND_SYSCALL (io_uring_setup);COND_SYSCALL_COMPAT (io_uring_setup);COND_SYSCALL (io_uring_enter); +COND_SYSCALL (io_uring_register);/* fs/xattr.c */
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 7/18 io_uring: support for IO polling
In reply to Jens AxboeAdd support for a polled io_uring context. When a read or write is submitted to a polled context, the application must poll for completions on the CQ ring through io_uring_enter(2). Polled IO may not generate IRQ completions, hence they need to be actively found by the application itself.
To use polling, io_uring_setup() must be used with the IORING_SETUP_IOPOLL flag being set. It is illegal to mix and match polled and non-polled IO on an io_uring.
--- fs/io_uring.c | 263 ++++++++++++++++++++++++++++++++-- include/uapi/linux/io_uring.h | 5 + 2 files changed, 257 insertions(+), 11 deletions(-)
fs/io_uring.c +252 -11
@@ -101,6 +101,8 @@ struct io_ring_ctx {struct {spinlock_t completion_lock; +unsigned poll_multi_file; +struct list_head poll_list; }____cacheline_aligned_in_smp ; };
@@ -119,12 +121,16 @@ struct io_kiocb {struct list_head list;unsigned int flags;#define REQ_F_FORCE_NONBLOCK 1/* inline submission attempt */ +#define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ +#define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */ u64 user_data; +u64 res;struct work_struct work; };#define IO_PLUG_THRESHOLD 2 +#define IO_IOPOLL_BATCH 8static struct kmem_cache *req_cachep;
@@ -156,6 +162,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)mutex_init (&ctx->uring_lock);init_waitqueue_head (&ctx->wait);spin_lock_init (&ctx->completion_lock); +INIT_LIST_HEAD (&ctx->poll_list);return ctx; }
@@ -191,8 +198,8 @@ static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx)return &ring->cqes[tail & ctx->cq_mask]; } -static void __io_cqring_add_event (struct io_ring_ctx *ctx,u64 ki_user_data, -long res,unsigned ev_flags) +static void io_cqring_fill_event (struct io_ring_ctx *ctx,u64 ki_user_data, +long res,unsigned ev_flags) {struct io_uring_cqe *cqe;
@@ -206,9 +213,15 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, cqe->user_data = ki_user_data; cqe->res = res; cqe->flags = ev_flags; -io_commit_cqring (ctx); }else ctx->cq_ring->overflow++; +} + +static void __io_cqring_add_event (struct io_ring_ctx *ctx,u64 ki_user_data, +long res,unsigned ev_flags) +{ +io_cqring_fill_event (ctx, ki_user_data, res, ev_flags); +io_commit_cqring (ctx);if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait);
@@ -250,12 +263,158 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)return NULL ; } +static void io_free_req_many (struct io_ring_ctx *ctx,void **reqs,int *nr) +{ +if (*nr) { +kmem_cache_free_bulk (req_cachep, *nr, reqs); +io_ring_drop_ctx_refs (ctx, *nr); + *nr =0 ; + } +} +static void io_free_req (struct io_kiocb *req) {io_ring_drop_ctx_refs (req->ctx,1 );kmem_cache_free (req_cachep, req); } +/* +* Find and free completed poll iocbs +*/ +static void io_iopoll_complete (struct io_ring_ctx *ctx,unsigned int *nr_events, +struct list_head *done) +{ +void *reqs[IO_IOPOLL_BATCH ]; +struct io_kiocb *req; +int to_free =0 ; + +while (!list_empty (done)) { + req = list_first_entry(done, struct io_kiocb, list); +list_del (&req->list); + +io_cqring_fill_event (ctx, req->user_data, req->res,0 ); + + reqs[to_free++] = req; + (*nr_events)++; + +fput (req->rw.ki_filp); +if (to_free ==ARRAY_SIZE (reqs)) +io_free_req_many (ctx, reqs, &to_free); + } +io_commit_cqring (ctx); + +if (to_free) +io_free_req_many (ctx, reqs, &to_free); +} + +static int io_do_iopoll (struct io_ring_ctx *ctx,unsigned int *nr_events, +long min) +{ +struct io_kiocb *req, *tmp; +LIST_HEAD (done); +bool spin; +int ret; + +/* +* Only spin for completions if we don't have multiple devices hanging +* off our complete list, and we're under the requested amount. +*/ + spin = !ctx->poll_multi_file && (*nr_events < min); + + ret =0 ; +list_for_each_entry_safe (req, tmp, &ctx->poll_list, list) { +struct kiocb *kiocb = &req->rw; + +/* +* Move completed entries to our local list. If we find a +* request that requires polling, break out and complete +* the done list first, if we have entries there. +*/ +if (req->flags &REQ_F_IOPOLL_COMPLETED ) { +list_move_tail (&req->list, &done); +continue ; + } +if (!list_empty (&done)) +break ; + + ret = kiocb->ki_filp->f_op->iopoll (kiocb, spin); +if (ret <0 ) +break ; + +if (ret && spin) + spin = false; + ret =0 ; + } + +if (!list_empty (&done)) +io_iopoll_complete (ctx, nr_events, &done); + +return ret; +} + +/* +* Poll for a mininum of 'min' events. Note that if min == 0 we consider that a +* non-spinning poll check - we'll still enter the driver poll loop, but only +* as a non-spinning completion check. +*/ +static int io_iopoll_getevents (struct io_ring_ctx *ctx,unsigned int *nr_events, +long min) +{ +int ret; + +do { +if (list_empty (&ctx->poll_list)) +return 0 ; + + ret =io_do_iopoll (ctx, nr_events, min); +if (ret <0 ) +break ; + }while (min && *nr_events < min); + +if (ret <0 ) +return ret; + +return *nr_events < min; +} + +/* +* We can't just wait for polled events to come to us, we have to actively +* find and complete them. +*/ +static void io_iopoll_reap_events (struct io_ring_ctx *ctx) +{ +if (!(ctx->flags &IORING_SETUP_IOPOLL )) +return ; + +mutex_lock (&ctx->uring_lock); +while (!list_empty (&ctx->poll_list)) { +unsigned int nr_events =0 ; + +io_iopoll_getevents (ctx, &nr_events,1 ); + } +mutex_unlock (&ctx->uring_lock); +} + +static int io_iopoll_check (struct io_ring_ctx *ctx,unsigned *nr_events, +long min) +{ +int ret =0 ; + +while (!*nr_events || !need_resched ()) { +int tmin =0 ; + +if (*nr_events < min) + tmin = min - *nr_events; + + ret =io_iopoll_getevents (ctx, nr_events, tmin); +if (ret <=0 ) +break ; + ret =0 ; + } + +return ret; +} +static void kiocb_end_write (struct kiocb *kiocb) {if (kiocb->ki_flags &IOCB_WRITE ) {
@@ -282,9 +441,60 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2)io_free_req (req); } +static void io_complete_rw_iopoll (struct kiocb *kiocb,long res,long res2) +{ + struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw); + +kiocb_end_write (kiocb); + +if (unlikely (res == -EAGAIN )) { + req->flags |=REQ_F_IOPOLL_EAGAIN ; + }else { + req->flags |=REQ_F_IOPOLL_COMPLETED ; + req->res = res; + } +} + +/* +* After the iocb has been issued, it's safe to be found on the poll list. +* Adding the kiocb to the list AFTER submission ensures that we don't +* find it from a io_iopoll_getevents() thread before the issuer is done +* accessing the kiocb cookie. +*/ +static void io_iopoll_req_issued (struct io_kiocb *req) +{ +struct io_ring_ctx *ctx = req->ctx; + +/* +* Track whether we have multiple files in our lists. This will impact +* how we do polling eventually, not spinning if we're on potentially +* different devices. +*/ +if (list_empty (&ctx->poll_list)) { + ctx->poll_multi_file =0 ; + }else if (!ctx->poll_multi_file) { +struct io_kiocb *list_req; + + list_req = list_first_entry(&ctx->poll_list, struct io_kiocb, + list); +if (list_req->rw.ki_filp != req->rw.ki_filp) + ctx->poll_multi_file =1 ; + } + +/* +* For fast devices, IO may have already completed. If it has, add +* it to the front so we find it first. +*/ +if (req->flags &REQ_F_IOPOLL_COMPLETED ) +list_add (&req->list, &ctx->poll_list); +else +list_add_tail (&req->list, &ctx->poll_list); +} +static int io_prep_rw (struct io_kiocb *req,const struct io_uring_sqe *sqe,bool force_nonblock) { +struct io_ring_ctx *ctx = req->ctx;struct kiocb *kiocb = &req->rw;int ret;
@@ -310,12 +520,21 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_flags |=IOCB_NOWAIT ; req->flags |=REQ_F_FORCE_NONBLOCK ; } -if (kiocb->ki_flags &IOCB_HIPRI ) { - ret = -EINVAL ; - goto out_fput; - } +if (ctx->flags &IORING_SETUP_IOPOLL ) { + ret = -EOPNOTSUPP ; +if (!(kiocb->ki_flags &IOCB_DIRECT ) || + !kiocb->ki_filp->f_op->iopoll) + goto out_fput; - kiocb->ki_complete = io_complete_rw; + kiocb->ki_flags |=IOCB_HIPRI ; + kiocb->ki_complete = io_complete_rw_iopoll; + }else { +if (kiocb->ki_flags &IOCB_HIPRI ) { + ret = -EINVAL ; + goto out_fput; + } + kiocb->ki_complete = io_complete_rw; + }return 0 ; out_fput:fput (kiocb->ki_filp);
@@ -462,6 +681,9 @@ static int io_nop(struct io_kiocb *req, const struct io_uring_sqe *sqe) {struct io_ring_ctx *ctx = req->ctx; +if (unlikely (ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; +__io_cqring_add_event (ctx, sqe->user_data,0 ,0 );io_free_req (req);return 0 ;
@@ -479,6 +701,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (force_nonblock)return -EAGAIN ; +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ;if (unlikely (sqe->addr || sqe->ioprio))return -EINVAL ;if (unlikely (sqe->fsync_flags & ~IORING_FSYNC_DATASYNC ))
@@ -526,7 +750,16 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,break ; } -return ret; +if (ret) +return ret; + +if (ctx->flags &IORING_SETUP_IOPOLL ) { +if (req->flags &REQ_F_IOPOLL_EAGAIN ) +return -EAGAIN ; +io_iopoll_req_issued (req); + } + +return 0 ; }static void io_sq_wq_submit_work (struct work_struct *work)
@@ -712,12 +945,18 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,return ret; }if (flags &IORING_ENTER_GETEVENTS ) { +unsigned nr_events =0 ;int get_ret;if (!ret && to_submit) min_complete =0 ; - get_ret =io_cqring_wait (ctx, min_complete); +if (ctx->flags &IORING_SETUP_IOPOLL ) + get_ret =io_iopoll_check (ctx, &nr_events, + min_complete); +else + get_ret =io_cqring_wait (ctx, min_complete); +if (get_ret <0 && !ret) ret = get_ret; }
@@ -826,6 +1065,7 @@ static void io_free_scq_urings(struct io_ring_ctx *ctx)static void io_ring_ctx_free (struct io_ring_ctx *ctx) {io_sq_offload_stop (ctx); +io_iopoll_reap_events (ctx);io_free_scq_urings (ctx);percpu_ref_exit (&ctx->refs);io_unaccount_mem (ctx,ring_pages (ctx->sq_entries, ctx->cq_entries));
@@ -860,6 +1100,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)percpu_ref_kill (&ctx->refs);mutex_unlock (&ctx->uring_lock); +io_iopoll_reap_events (ctx);wait_for_completion (&ctx->ctx_done);io_ring_ctx_free (ctx); }
@@ -1090,7 +1331,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params,return -EINVAL ; } -if (p.flags) +if (p.flags & ~IORING_SETUP_IOPOLL )return -EINVAL ; ret =io_uring_create (entries, &p, compat);
include/uapi/linux/io_uring.h +5 -0
@@ -32,6 +32,11 @@ struct io_uring_sqe {__u64 __pad2[3 ]; }; +/* +* io_uring_setup() flags +*/ +#define IORING_SETUP_IOPOLL (1 << 0)/* io_context is polled */ +#define IORING_OP_NOP 0#define IORING_OP_READV 1#define IORING_OP_WRITEV 2
Signature · 1 line
2.17.1
Re: [PATCH 07/18] io_uring: support for IO polling
fs/io_uring.c struct io_ring_ctx { · line 104
Quoted 3 lines
fs/io_uring.c +1
-0struct {spinlock_t completion_lock; +unsigned poll_multi_file;
This seems to be used as a bool.
Quoted 9 lines
fs/io_uring.c +2
-2-static void __io_cqring_add_event (struct io_ring_ctx *ctx,u64 ki_user_data, -long res,unsigned ev_flags) +static void io_cqring_fill_event (struct io_ring_ctx *ctx,u64 ki_user_data, +long res,unsigned ev_flags) {struct io_uring_cqe *cqe;
fs/io_uring.c static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, · line 218
Quoted 7 lines
fs/io_uring.c +1
-1@@ -206,9 +213,15 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, cqe->user_data = ki_user_data; cqe->res = res; cqe->flags = ev_flags; -io_commit_cqring (ctx); }else ctx->cq_ring->overflow++; +}
Maybe we should have created this low-level helper already in the patch introducing the code?
Jens Axboe
maintainerRe: [PATCH 07/18] io_uring: support for IO polling
Quoted 5 lines · Christoph Hellwig
fs/io_uring.c struct io_ring_ctx { · line 104
Quoted 3 lines
fs/io_uring.c +1
-0struct {spinlock_t completion_lock; +unsigned poll_multi_file;This seems to be used as a bool.
Fixed
Quoted 7 lines
Quoted 7 lines
fs/io_uring.c +2
-2-static void __io_cqring_add_event (struct io_ring_ctx *ctx,u64 ki_user_data, -long res,unsigned ev_flags) +static void io_cqring_fill_event (struct io_ring_ctx *ctx,u64 ki_user_data, +long res,unsigned ev_flags) {struct io_uring_cqe *cqe;
Quoted 11 lines
fs/io_uring.c static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, · line 218
Quoted 8 lines
fs/io_uring.c +1
-1@@ -206,9 +213,15 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, cqe->user_data = ki_user_data; cqe->res = res; cqe->flags = ev_flags; -io_commit_cqring (ctx); }else ctx->cq_ring->overflow++; +}Maybe we should have created this low-level helper already in the patch introducing the code?
I think that can go both ways. If we create it early, it'll just be a pointless helper. Don't really care too much about this particular case.
Signature · 1 line
Jens Axboe
Re: [PATCH 07/18] io_uring: support for IO polling
Quoted 2 lines · Jens Axboe
I think that can go both ways. If we create it early, it'll just be a pointless helper. Don't really care too much about this particular case.
Without the no-op locking fix we don't really need the current __io_cqring_add_event anymore, so we'd still have two somewhat reasonably defined helpers.
Jens Axboe
maintainerRe: [PATCH 07/18] io_uring: support for IO polling
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Jens Axboe
I think that can go both ways. If we create it early, it'll just be a pointless helper. Don't really care too much about this particular case.
Without the no-op locking fix we don't really need the current
__io_cqring_add_eventanymore, so we'd still have two somewhat reasonably defined helpers.
Yeah, killed it, which also fixes your complaint about adding a new helper in the poll patch.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 8/18 fs: add fget_many() and fput_many()
In reply to Jens AxboeSome uses cases repeatedly get and put references to the same file, but the only exposed interface is doing these one at the time. As each of these entail an atomic inc or dec on a shared structure, that cost can add up.
Add fget_many(), which works just like fget(), except it takes an argument for how many references to get on the file. Ditto fput_many(), which can drop an arbitrary number of references to a file.
--- fs/file.c | 15 ++++++++++----- fs/file_table.c | 9 +++++++-- include/linux/file.h | 2 ++ include/linux/fs.h | 4 +++- 4 files changed, 22 insertions(+), 8 deletions(-)
fs/file.c +10 -5
@@ -705,7 +705,7 @@ void do_close_on_exec(struct files_struct *files)spin_unlock (&files->file_lock); } -static struct file *__fget (unsigned int fd,fmode_t mask) +static struct file *__fget (unsigned int fd,fmode_t mask,unsigned int refs) {struct files_struct *files = current->files;struct file *file;
@@ -720,7 +720,7 @@ static struct file *__fget(unsigned int fd, fmode_t mask)*/ if (file->f_mode & mask) file =NULL ; -else if (!get_file_rcu (file)) +else if (!get_file_rcu_many (file, refs)) goto loop; }rcu_read_unlock ();
@@ -728,15 +728,20 @@ static struct file *__fget(unsigned int fd, fmode_t mask)return file; } +struct file *fget_many (unsigned int fd,unsigned int refs) +{ +return __fget (fd,FMODE_PATH , refs); +} +struct file *fget (unsigned int fd) { -return __fget (fd,FMODE_PATH ); +return fget_many (fd,1 ); }EXPORT_SYMBOL (fget);struct file *fget_raw (unsigned int fd) { -return __fget (fd,0 ); +return __fget (fd,0 ,1 ); }EXPORT_SYMBOL (fget_raw);
@@ -767,7 +772,7 @@ static unsigned long __fget_light(unsigned int fd, fmode_t mask)return 0 ;return (unsigned long )file; }else { - file =__fget (fd, mask); + file =__fget (fd, mask,1 );if (!file)return 0 ;return FDPUT_FPUT | (unsigned long )file;
fs/file_table.c +7 -2
@@ -326,9 +326,9 @@ void flush_delayed_fput(void) static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput); -void fput (struct file *file) +void fput_many (struct file *file,unsigned int refs) { -if (atomic_long_dec_and_test (&file->f_count)) { +if (atomic_long_sub_and_test (refs, &file->f_count)) {struct task_struct *task = current;if (likely (!in_interrupt () && !(task->flags &PF_KTHREAD ))) {
@@ -347,6 +347,11 @@ void fput(struct file *file) } } +void fput (struct file *file) +{ +fput_many (file,1 ); +} +/* * synchronous analog of fput(); for kernel threads that might be needed * in some umount() (and thus can't use flush_delayed_fput() without
include/linux/file.h +2 -0
@@ -13,6 +13,7 @@struct file ;extern void fput (struct file *); +extern void fput_many (struct file *,unsigned int );struct file_operations ;struct vfsmount ;
@@ -44,6 +45,7 @@ static inline void fdput(struct fd fd) }extern struct file *fget (unsigned int fd); +extern struct file *fget_many (unsigned int fd,unsigned int refs);extern struct file *fget_raw (unsigned int fd);extern unsigned long __fdget (unsigned int fd);extern unsigned long __fdget_raw (unsigned int fd);
include/linux/fs.h +3 -1
@@ -952,7 +952,9 @@ static inline struct file *get_file(struct file *f)atomic_long_inc (&f->f_count);return f; } -#define get_file_rcu (x) atomic_long_inc_not_zero(&(x)->f_count) +#define get_file_rcu_many (x, cnt) \ + atomic_long_add_unless(&(x)->f_count, (cnt), 0) +#define get_file_rcu (x) get_file_rcu_many((x), 1)#define fput_atomic (x) atomic_long_add_unless(&(x)->f_count, -1, 1)#define file_count (x) atomic_long_read(&(x)->f_count)
Signature · 1 line
2.17.1
Re: [PATCH 08/18] fs: add fget_many() and fput_many()
fs/file.c static struct file *__fget(unsigned int fd, fmode_t mask) · line 738
Quoted 9 lines
fs/file.c +6
-1+struct file *fget_many (unsigned int fd,unsigned int refs) +{ +return __fget (fd,FMODE_PATH , refs); +} +struct file *fget (unsigned int fd) { -return __fget (fd,FMODE_PATH ); +return fget_many (fd,1 );
I'd still prefer removing the double indirection in this call, but I know you disagree.
fs/file.c static struct file *__fget(unsigned int fd, fmode_t mask) · line 746
Quoted 9 lines
}
EXPORT_SYMBOL(fget);struct file *fget_raw(unsigned int fd)
{fs/file.c +1
-1-return __fget (fd,0 ); +return __fget (fd,0 ,1 ); }EXPORT_SYMBOL (fget_raw);
Especially as we already avoid it here..
Otherwise looks fine to me:
Jens Axboe
maintainerRe: [PATCH 08/18] fs: add fget_many() and fput_many()
Quoted 4 lines · Christoph Hellwig
Quoted 4 lines
fs/file.c +4
-0+struct file *fget_many (unsigned int fd,unsigned int refs) +{ +return __fget (fd,FMODE_PATH , refs); +}
Quoted 8 lines
fs/file.c static struct file *__fget(unsigned int fd, fmode_t mask) · line 738
Quoted 5 lines
fs/file.c +2
-1+struct file *fget (unsigned int fd) { -return __fget (fd,FMODE_PATH ); +return fget_many (fd,1 );I'd still prefer removing the double indirection in this call, but I know you disagree.
Not a big deal, I can kill it.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 8/13 io_uring: add file set registration
In reply to Jens AxboeWe normally have to fget/fput for each IO we do on a file. Even with the batching we do, the cost of the atomic inc/dec of the file usage count adds up.
This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes for the io_uring_register(2) system call. The arguments passed in must be an array of __s32 holding file descriptors, and nr_args should hold the number of file descriptors the application wishes to pin for the duration of the io_uring context (or until IORING_UNREGISTER_FILES is called).
When used, the application must set IOSQE_FIXED_FILE in the sqe->flags member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd to the index in the array passed in to IORING_REGISTER_FILES.
Files are automatically unregistered when the io_uring context is torn down. An application need only unregister if it wishes to register a few set of fds.
--- fs/io_uring.c | 125 +++++++++++++++++++++++++++++----- include/uapi/linux/io_uring.h | 9 ++- 2 files changed, 116 insertions(+), 18 deletions(-)
fs/io_uring.c +108 -17
@@ -100,6 +100,10 @@ struct io_ring_ctx {struct fasync_struct *cq_fasync; }____cacheline_aligned_in_smp ; +/* if used, fixed file set */ +struct file **user_files; +unsigned nr_user_files; +/* if used, fixed mapped user buffers */ unsigned nr_user_bufs;struct io_mapped_ubuf *user_bufs;
@@ -137,6 +141,7 @@ struct io_kiocb {#define REQ_F_FORCE_NONBLOCK 1/* inline submission attempt */ #define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */ +#define REQ_F_FIXED_FILE 8/* ctx owns file */ u64 user_data;u64 res;
@@ -359,15 +364,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,* Batched puts of the same file, to avoid dirtying the * file usage count multiple times, if avoidable. */ -if (!file) { - file = req->rw.ki_filp; - file_count =1 ; - }else if (file == req->rw.ki_filp) { - file_count++; - }else { -fput_many (file, file_count); - file = req->rw.ki_filp; - file_count =1 ; +if (!(req->flags &REQ_F_FIXED_FILE )) { +if (!file) { + file = req->rw.ki_filp; + file_count =1 ; + }else if (file == req->rw.ki_filp) { + file_count++; + }else { +fput_many (file, file_count); + file = req->rw.ki_filp; + file_count =1 ; + } }if (to_free ==ARRAY_SIZE (reqs))
@@ -504,13 +511,19 @@ static void kiocb_end_write(struct kiocb *kiocb) } } +static void io_fput (struct io_kiocb *req) +{ +if (!(req->flags &REQ_F_FIXED_FILE )) +fput (req->rw.ki_filp); +} +static void io_complete_rw (struct kiocb *kiocb,long res,long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);kiocb_end_write (kiocb); -fput (kiocb->ki_filp); +io_fput (req);io_cqring_add_event (req->ctx, req->user_data, res,0 );io_free_req (req); }
@@ -614,7 +627,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;int ret; - kiocb->ki_filp =io_file_get (state, sqe->fd); +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + kiocb->ki_filp = ctx->user_files[sqe->fd]; + req->flags |=REQ_F_FIXED_FILE ; + }else { + kiocb->ki_filp =io_file_get (state, sqe->fd); + }if (unlikely (!kiocb->ki_filp))return -EBADF ; kiocb->ki_pos = sqe->off;
@@ -653,7 +673,8 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, }return 0 ; out_fput: -io_file_put (state, kiocb->ki_filp); +if (!(sqe->flags &IOSQE_FIXED_FILE )) +io_file_put (state, kiocb->ki_filp);return ret; }
@@ -770,7 +791,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,kfree (iovec); out_fput:if (unlikely (ret)) -fput (file); +io_fput (req);return ret; }
@@ -825,7 +846,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,kfree (iovec); out_fput:if (unlikely (ret)) -fput (file); +io_fput (req);return ret; }
@@ -863,14 +884,23 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (unlikely (sqe->fsync_flags & ~IORING_FSYNC_DATASYNC ))return -EINVAL ; - file =fget (sqe->fd); +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + file = ctx->user_files[sqe->fd]; + }else { + file =fget (sqe->fd); + } +if (unlikely (!file))return -EBADF ; ret =vfs_fsync_range (file, sqe->off, end >0 ? end :LLONG_MAX , sqe->fsync_flags &IORING_FSYNC_DATASYNC ); -fput (file); +if (!(sqe->flags &IOSQE_FIXED_FILE )) +fput (file); +io_cqring_add_event (ctx, sqe->user_data, ret,0 );io_free_req (req);return 0 ;
@@ -988,7 +1018,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,ssize_t ret;/* enforce forwards compatibility on users */ -if (unlikely (s->sqe->flags)) +if (unlikely (s->sqe->flags & ~IOSQE_FIXED_FILE ))return -EINVAL ; req =io_get_req (ctx, state);
@@ -1173,6 +1203,57 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,return ret; } +static int io_sqe_files_unregister (struct io_ring_ctx *ctx) +{ +int i; + +if (!ctx->user_files) +return -ENXIO ; + +for (i =0 ; i < ctx->nr_user_files; i++) +fput (ctx->user_files[i]); + +kfree (ctx->user_files); + ctx->user_files =NULL ; + ctx->nr_user_files =0 ; +return 0 ; +} + +static int io_sqe_files_register (struct io_ring_ctx *ctx,void __user *arg, +unsigned nr_args) +{ +__s32 __user *fds = (__s32 __user *) arg; +int fd, i, ret =0 ; + +if (ctx->user_files) +return -EBUSY ; +if (!nr_args) +return -EINVAL ; + + ctx->user_files =kcalloc (nr_args,sizeof (struct file *),GFP_KERNEL ); +if (!ctx->user_files) +return -ENOMEM ; + +for (i =0 ; i < nr_args; i++) { + ret = -EFAULT ; +if (copy_from_user (&fd, &fds[i],sizeof (fd))) +break ; + + ctx->user_files[i] =fget (fd); + + ret = -EBADF ; +if (!ctx->user_files[i]) +break ; + ctx->nr_user_files++; + ret =0 ; + } + +if (ret) +io_sqe_files_unregister (ctx); + +return ret; +} +static int io_sq_offload_start (struct io_ring_ctx *ctx) {int ret;
@@ -1468,6 +1549,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)io_sq_offload_stop (ctx);io_iopoll_reap_events (ctx);io_free_scq_urings (ctx); +io_sqe_files_unregister (ctx);io_sqe_buffer_unregister (ctx);percpu_ref_exit (&ctx->refs);io_unaccount_mem (ctx,ring_pages (ctx->sq_entries, ctx->cq_entries));
@@ -1780,6 +1862,15 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,break ; ret =io_sqe_buffer_unregister (ctx);break ; +case IORING_REGISTER_FILES : + ret =io_sqe_files_register (ctx, arg, nr_args); +break ; +case IORING_UNREGISTER_FILES : + ret = -EINVAL ; +if (arg || nr_args) +break ; + ret =io_sqe_files_unregister (ctx); +break ;default : ret = -EINVAL ;break ;
include/uapi/linux/io_uring.h +8 -1
@@ -18,7 +18,7 @@*/ struct io_uring_sqe {__u8 opcode;/* type of operation for this sqe */ -__u8 flags;/* as of now unused */ +__u8 flags;/* IOSQE_ flags */ __u16 ioprio;/* ioprio for the request */ __s32 fd;/* file descriptor to do IO on */ __u64 off;/* offset into file */
@@ -35,6 +35,11 @@ struct io_uring_sqe { }; }; +/* +* sqe->flags +*/ +#define IOSQE_FIXED_FILE (1 << 0)/* use fixed fileset */ +/* * io_uring_setup() flags */
@@ -114,5 +119,7 @@ struct io_uring_params {*/ #define IORING_REGISTER_BUFFERS 0#define IORING_UNREGISTER_BUFFERS 1 +#define IORING_REGISTER_FILES 2 +#define IORING_UNREGISTER_FILES 3#endif
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 9/13 io_uring: add submission polling
In reply to Jens AxboeThis enables an application to do IO, without ever entering the kernel. By using the SQ ring to fill in new sqes and watching for completions on the CQ ring, we can submit and reap IOs without doing a single system call. The kernel side thread will poll for new submissions, and in case of HIPRI/polled IO, it'll also poll for completions.
Proof of concept. If the thread has been idle for 1 second, it will set sq_ring->flags |= IORING_SQ_NEED_WAKEUP. The application will have to call io_uring_enter() to start things back up again. If IO is kept busy, that will never be needed. Basically an application that has this feature enabled will guard it's io_uring_enter(2) call with:
read_barrier();
if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP)
io_uring_enter(fd, to_submit, 0, 0);
instead of calling it unconditionally.
Improvements:
1) Maybe have smarter backoff. Busy loop for X time, then go to monitor/mwait, finally the schedule we have now after an idle second. Might not be worth the complexity.
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
--- fs/io_uring.c | 219 +++++++++++++++++++++++++++++++++- include/uapi/linux/io_uring.h | 10 +- 2 files changed, 222 insertions(+), 7 deletions(-)
fs/io_uring.c +213 -6
@@ -24,6 +24,7 @@#include <linux/percpu.h> #include <linux/slab.h> #include <linux/workqueue.h> +#include <linux/kthread.h> #include <linux/blkdev.h> #include <linux/bvec.h> #include <linux/anon_inodes.h>
@@ -87,8 +88,10 @@ struct io_ring_ctx {/* IO offload */ struct workqueue_struct *sqo_wq; +struct task_struct *sqo_thread;/* if using sq thread polling */ struct mm_struct *sqo_mm;struct files_struct *sqo_files; +wait_queue_head_t sqo_wait;struct {/* CQ ring */
@@ -264,6 +267,9 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait); +if ((ctx->flags &IORING_SETUP_SQPOLL ) && +waitqueue_active (&ctx->sqo_wait)) +wake_up (&ctx->sqo_wait); } static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
@@ -1106,6 +1112,168 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)return false; } +static int io_submit_sqes (struct io_ring_ctx *ctx,struct sqe_submit *sqes, +unsigned int nr,bool mm_fault) +{ +struct io_submit_state state, *statep =NULL ; +int ret, i, submitted =0 ; + +if (nr >IO_PLUG_THRESHOLD ) { +io_submit_state_start (&state, ctx, nr); + statep = &state; + } + +for (i =0 ; i < nr; i++) { +if (unlikely (mm_fault)) + ret = -EFAULT ; +else + ret =io_submit_sqe (ctx, &sqes[i], statep); +if (!ret) { + submitted++; +continue ; + } + +io_cqring_add_event (ctx, sqes[i].sqe->user_data, ret,0 ); + } + +if (statep) +io_submit_state_end (&state); + +return submitted; +} + +static int io_sq_thread (void *data) +{ +struct sqe_submit sqes[IO_IOPOLL_BATCH ]; +struct io_ring_ctx *ctx = data; +struct mm_struct *cur_mm =NULL ; +struct files_struct *old_files; +mm_segment_t old_fs; +DEFINE_WAIT (wait); +unsigned inflight; +unsigned long timeout; + + old_files = current->files; + current->files = ctx->sqo_files; + + old_fs =get_fs (); +set_fs (USER_DS ); + + timeout = inflight =0 ; +while (!kthread_should_stop ()) { +bool all_fixed, mm_fault = false; +int i; + +if (inflight) { +unsigned int nr_events =0 ; + +/* +* Normal IO, just pretend everything completed. +* We don't have to poll completions for that. +*/ +if (ctx->flags &IORING_SETUP_IOPOLL ) { +/* +* App should not use IORING_ENTER_GETEVENTS +* with thread polling, but if it does, then +* ensure we are mutually exclusive. +*/ +if (mutex_trylock (&ctx->uring_lock)) { +io_iopoll_check (ctx, &nr_events,0 ); +mutex_unlock (&ctx->uring_lock); + } + }else { + nr_events = inflight; + } + + inflight -= nr_events; +if (!inflight) + timeout = jiffies +HZ ; + } + +if (!io_get_sqring (ctx, &sqes[0 ])) { +/* +* We're polling, let us spin for a second without +* work before going to sleep. +*/ +if (inflight || !time_after (jiffies, timeout)) { +cpu_relax (); +continue ; + } + +/* +* Drop cur_mm before scheduling, we can't hold it for +* long periods (or over schedule()). Do this before +* adding ourselves to the waitqueue, as the unuse/drop +* may sleep. +*/ +if (cur_mm) { +unuse_mm (cur_mm); +mmput (cur_mm); + cur_mm =NULL ; + } + +prepare_to_wait (&ctx->sqo_wait, &wait, +TASK_INTERRUPTIBLE ); + +/* Tell userspace we may need a wakeup call */ + ctx->sq_ring->flags |=IORING_SQ_NEED_WAKEUP ; +smp_wmb (); + +if (!io_get_sqring (ctx, &sqes[0 ])) { +if (kthread_should_park ()) +kthread_parkme (); +if (kthread_should_stop ()) { +finish_wait (&ctx->sqo_wait, &wait); +break ; + } +if (signal_pending (current)) +flush_signals (current); +schedule (); +finish_wait (&ctx->sqo_wait, &wait); + + ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP ; +smp_wmb (); +continue ; + } +finish_wait (&ctx->sqo_wait, &wait); + + ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP ; +smp_wmb (); + } + + i =0 ; + all_fixed = true; +do { +if (all_fixed &&io_sqe_needs_user (sqes[i].sqe)) + all_fixed = false; + + i++; +if (i ==ARRAY_SIZE (sqes)) +break ; + }while (io_get_sqring (ctx, &sqes[i])); + +io_commit_sqring (ctx); + +/* Unless all new commands are FIXED regions, grab mm */ +if (!all_fixed && !cur_mm) { + mm_fault = !mmget_not_zero (ctx->sqo_mm); +if (!mm_fault) { +use_mm (ctx->sqo_mm); + cur_mm = ctx->sqo_mm; + } + } + + inflight +=io_submit_sqes (ctx, sqes, i, mm_fault); + } + current->files = old_files; +set_fs (old_fs); +if (cur_mm) { +unuse_mm (cur_mm); +mmput (cur_mm); + } +return 0 ; +} +static int io_ring_submit (struct io_ring_ctx *ctx,unsigned int to_submit) {struct io_submit_state state, *statep =NULL ;
@@ -1179,9 +1347,14 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,int ret =0 ;if (to_submit) { - ret =io_ring_submit (ctx, to_submit); -if (ret <0 ) -return ret; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +wake_up (&ctx->sqo_wait); + ret = to_submit; + }else { + ret =io_ring_submit (ctx, to_submit); +if (ret <0 ) +return ret; + } }if (flags &IORING_ENTER_GETEVENTS ) {unsigned nr_events =0 ;
@@ -1254,10 +1427,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,return ret; } -static int io_sq_offload_start (struct io_ring_ctx *ctx) +static int io_sq_offload_start (struct io_ring_ctx *ctx, +struct io_uring_params *p) {int ret; +init_waitqueue_head (&ctx->sqo_wait); ctx->sqo_mm = current->mm;/*
@@ -1270,6 +1445,27 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)if (!ctx->sqo_files) goto err; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (p->flags &IORING_SETUP_SQ_AFF ) { + ctx->sqo_thread =kthread_create_on_cpu (io_sq_thread, + ctx, p->sq_thread_cpu, +"io_uring-sq" ); + }else { + ctx->sqo_thread =kthread_create (io_sq_thread, ctx, +"io_uring-sq" ); + } +if (IS_ERR (ctx->sqo_thread)) { + ret =PTR_ERR (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + goto err; + } +wake_up_process (ctx->sqo_thread); + }else if (p->flags &IORING_SETUP_SQ_AFF ) { +/* Can't have SQ_AFF without SQPOLL */ + ret = -EINVAL ; + goto err; + } +/* Do QD, or 2 * CPUS, whatever is smallest */ ctx->sqo_wq =alloc_workqueue ("io_ring-wq" ,WQ_UNBOUND |WQ_FREEZABLE ,min (ctx->sq_entries -1 ,2 *num_online_cpus ()));
@@ -1280,6 +1476,11 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)return 0 ; err: +if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + }if (ctx->sqo_files) ctx->sqo_files =NULL ; ctx->sqo_mm =NULL ;
@@ -1288,6 +1489,11 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)static void io_sq_offload_stop (struct io_ring_ctx *ctx) { +if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + }if (ctx->sqo_wq) {destroy_workqueue (ctx->sqo_wq); ctx->sqo_wq =NULL ;
@@ -1780,7 +1986,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; - ret =io_sq_offload_start (ctx); + ret =io_sq_offload_start (ctx, p);if (ret) goto err;
@@ -1815,7 +2021,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params,return -EINVAL ; } -if (p.flags & ~IORING_SETUP_IOPOLL ) +if (p.flags & ~(IORING_SETUP_IOPOLL |IORING_SETUP_SQPOLL | +IORING_SETUP_SQ_AFF ))return -EINVAL ; ret =io_uring_create (entries, &p, compat);
include/uapi/linux/io_uring.h +9 -1
@@ -44,6 +44,8 @@ struct io_uring_sqe {* io_uring_setup() flags */ #define IORING_SETUP_IOPOLL (1 << 0)/* io_context is polled */ +#define IORING_SETUP_SQPOLL (1 << 1)/* SQ poll thread */ +#define IORING_SETUP_SQ_AFF (1 << 2)/* sq_thread_cpu is valid */ #define IORING_OP_NOP 0#define IORING_OP_READV 1
@@ -87,6 +89,11 @@ struct io_sqring_offsets {__u32 resv[3 ]; }; +/* +* sq_ring->flags +*/ +#define IORING_SQ_NEED_WAKEUP (1 << 0)/* needs io_uring_enter wakeup */ +struct io_cqring_offsets {__u32 head;__u32 tail;
@@ -109,7 +116,8 @@ struct io_uring_params {__u32 sq_entries;__u32 cq_entries;__u32 flags; -__u16 resv[10 ]; +__u16 sq_thread_cpu; +__u16 resv[9 ];struct io_sqring_offsets sq_off;struct io_cqring_offsets cq_off; };
Signature · 1 line
2.17.1
Re: [PATCH 09/13] io_uring: add submission polling
Quoted 1 line · Jens Axboe
Proof of concept.
Is that still true?
Quoted 6 lines
1) Maybe have smarter backoff. Busy loop for X time, then go to monitor/mwait, finally the schedule we have now after an idle second. Might not be worth the complexity.
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
2) actually sounds really useful. Should we look into it ASAP?
fs/io_uring.c static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, · line 271
Quoted 9 lines
fs/io_uring.c +2
-0struct {/* CQ ring */ @@ -264,6 +267,9 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait); +if ((ctx->flags &IORING_SETUP_SQPOLL ) && +waitqueue_active (&ctx->sqo_wait))
waitqueue_active is really cheap and sqo_wait should not otherwise by active. Do we really need the flags check here?
fs/io_uring.c static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s) · line 1178
Quoted 9 lines
fs/io_uring.c +9
-0+/* +* Normal IO, just pretend everything completed. +* We don't have to poll completions for that. +*/ +if (ctx->flags &IORING_SETUP_IOPOLL ) { +/* +* App should not use IORING_ENTER_GETEVENTS +* with thread polling, but if it does, then +* ensure we are mutually exclusive.
Should we just return an error early on in this case instead?
fs/io_uring.c static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit, · line 1352
Quoted 4 lines
fs/io_uring.c +3
-0if (to_submit) { +if (ctx->flags &IORING_SETUP_SQPOLL ) { +wake_up (&ctx->sqo_wait); + ret = to_submit;
Do these semantics really make sense? Maybe we should have an IORING_ENTER_WAKE_SQ instead of overloading the to_submit argument? Especially as we don't really care about returning the number passed in.
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (ctx->sqo_thread) { + kthread_park(ctx->sqo_thread);
Can you explain why we need the whole kthread_park game? It is only intended to deal with pausing a thread, and if need it to shut down a thread we have a bug somewhere.
fs/io_uring.c static int io_sq_offload_start(struct io_ring_ctx *ctx) · line 1482
Quoted 6 lines
static void
io_sq_offload_stop(struct io_ring_ctx *ctx){fs/io_uring.c +4
-0+if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ;
Also there isn't really much of a point in setting pointers to NULL just before freeing the containing structure. In the best case this now papers over bugs that poisoning or kasan would otherwise find.
Jens Axboe
maintainerRe: [PATCH 09/13] io_uring: add submission polling
Quoted 4 lines · Christoph Hellwig
Quoted 1 line · Jens Axboe
Proof of concept.
Is that still true?
I guess I can remove it now, dates back to when it was initially just a test. But it should be solid, I'll kill that part.
Quoted 8 lines
Quoted 6 lines
1) Maybe have smarter backoff. Busy loop for X time, then go to monitor/mwait, finally the schedule we have now after an idle second. Might not be worth the complexity.
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
2) actually sounds really useful. Should we look into it ASAP?
I think so. Question is what kind of granularity we need for this. I think we can go pretty coarse and keep it in msec, using a short to pass this in like we do for the thread CPU. That gives us 0..65535 msec, which should be plenty of range.
Quoted 11 lines
fs/io_uring.c static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data, · line 271
Quoted 8 lines
fs/io_uring.c +2
-0struct {/* CQ ring */ @@ -264,6 +267,9 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait); +if ((ctx->flags &IORING_SETUP_SQPOLL ) && +waitqueue_active (&ctx->sqo_wait))
waitqueue_activeis really cheap andsqo_waitshould not otherwise by active. Do we really need the flags check here?
Probably not, I'll kill it.
Quoted 11 lines
fs/io_uring.c static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s) · line 1178
Quoted 9 lines
fs/io_uring.c +9
-0+/* +* Normal IO, just pretend everything completed. +* We don't have to poll completions for that. +*/ +if (ctx->flags &IORING_SETUP_IOPOLL ) { +/* +* App should not use IORING_ENTER_GETEVENTS +* with thread polling, but if it does, then +* ensure we are mutually exclusive. Should we just return an error early on in this case instead?
I think that'd make it awkward, since it's out-of-line. If the app is doing things it shouldn't in this case, its own io_uring_enter() would most likely fail occasionally with -EBUSY anyway.
Quoted 8 lines
fs/io_uring.c static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit, · line 1352
Quoted 4 lines
fs/io_uring.c +3
-0if (to_submit) { +if (ctx->flags &IORING_SETUP_SQPOLL ) { +wake_up (&ctx->sqo_wait); + ret = to_submit;Do these semantics really make sense? Maybe we should have an
IORING_ENTER_WAKE_SQinstead of overloading theto_submitargument? Especially as we don't really care about returning the number passed in.
I like that change, I'll add IORING_ENTER_SQ_WAKEUP instead of using 'to_submit' for this. We can't validate the number anyway.
Quoted 6 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (ctx->sqo_thread) { + kthread_park(ctx->sqo_thread);Can you explain why we need the whole
kthread_parkgame? It is only intended to deal with pausing a thread, and if need it to shut down a thread we have a bug somewhere.
It is working around a bug in shutting down a thread that is affinitized to a single CPU, I just didn't want to deal with hunting that down right now.
Quoted 10 lines
fs/io_uring.c static int io_sq_offload_start(struct io_ring_ctx *ctx) · line 1482
Quoted 6 lines
static void
io_sq_offload_stop(struct io_ring_ctx *ctx){fs/io_uring.c +4
-0+if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ;Also there isn't really much of a point in setting pointers to NULL just before freeing the containing structure. In the best case this now papers over bugs that poisoning or kasan would otherwise find.
Removed.
Signature · 1 line
Jens Axboe
Re: [PATCH 09/13] io_uring: add submission polling
Quoted 9 lines · Jens Axboe
Quoted 4 lines
Quoted 2 lines
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
2) actually sounds really useful. Should we look into it ASAP?
I think so. Question is what kind of granularity we need for this. I think we can go pretty coarse and keep it in msec, using a short to pass this in like we do for the thread CPU. That gives us 0..65535 msec, which should be plenty of range.
msec granualarity sounds fine. But is it really worth using a short instead of a 32-bit value these days?
Also 16385 seems very limiting for a cpu value, especially as our normal cpu ids are ints.
Jens Axboe
maintainerRe: [PATCH 09/13] io_uring: add submission polling
Quoted 11 lines · Christoph Hellwig
Quoted 9 lines · Jens Axboe
Quoted 4 lines
Quoted 2 lines
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
2) actually sounds really useful. Should we look into it ASAP?
I think so. Question is what kind of granularity we need for this. I think we can go pretty coarse and keep it in msec, using a short to pass this in like we do for the thread CPU. That gives us 0..65535 msec, which should be plenty of range.
Quoted 5 lines
msec granualarity sounds fine. But is it really worth using a short instead of a 32-bit value these days?
Also 16385 seems very limiting for a cpu value, especially as our normal cpu ids are ints.
I'll bump them both to 4-bytes.
Signature · 1 line
Jens Axboe
Re: [PATCH 09/13] io_uring: add submission polling
In reply to Jens Axboefs/io_uring.c static int io_sq_offload_start(struct io_ring_ctx *ctx) · line 1452
Quoted 9 lines · Jens Axboe
fs/io_uring.c +5
-0@@ -1270,6 +1445,27 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)if (!ctx->sqo_files) goto err; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (p->flags &IORING_SETUP_SQ_AFF ) { + ctx->sqo_thread =kthread_create_on_cpu (io_sq_thread, + ctx, p->sq_thread_cpu, +"io_uring-sq" );
sq_thread_cpu looks like another candidate for array_index_nospec. Following the macros, kthread_create_on_cpu calls cpu_to_node, which does:
return per_cpu(x86_cpu_to_node_map, cpu);
#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
#define per_cpu_ptr(ptr, cpu) \
({ \
__verify_pcpu_ptr(ptr); \
SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
})
#define per_cpu_offset(x) (__per_cpu_offset[x])
^^^^^^^^^^^^^^^^^^^It also looks like there's no bounds checking there, so we probably want to make sure sq_thread_cpu can't overflow the __per_cpu_offset array (NR_CPUS).
-Jeff
Jens Axboe
maintainerRe: [PATCH 09/13] io_uring: add submission polling
Quoted 26 lines · Jeff Moyer
fs/io_uring.c static int io_sq_offload_start(struct io_ring_ctx *ctx) · line 1452
Quoted 9 lines · Jens Axboe
fs/io_uring.c +5
-0@@ -1270,6 +1445,27 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)if (!ctx->sqo_files) goto err; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (p->flags &IORING_SETUP_SQ_AFF ) { + ctx->sqo_thread =kthread_create_on_cpu (io_sq_thread, + ctx, p->sq_thread_cpu, +"io_uring-sq" );
sq_thread_cpulooks like another candidate forarray_index_nospec. Following the macros,kthread_create_on_cpucallscpu_to_node, which does:return per_cpu(x86_cpu_to_node_map, cpu);#define
per_cpu(var, cpu)(*per_cpu_ptr(&(var), cpu))#define per_cpu_ptr(ptr, cpu) \ ({ \ __verify_pcpu_ptr(ptr); \ SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \ }) #define per_cpu_offset(x) (__per_cpu_offset[x]) ^^^^^^^^^^^^^^^^^^^
Quoted 3 lines
It also looks like there's no bounds checking there, so we probably want to make sure
sq_thread_cpucan't overflow the__per_cpu_offsetarray (NR_CPUS).
Added, can't hurt in any case.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 9/18 io_uring: use fget/fput_many() for file references
In reply to Jens AxboeAdd a separate io_submit_state structure, to cache some of the things we need for IO submission.
One such example is file reference batching. io_submit_state. We get as many references as the number of sqes we are submitting, and drop unused ones if we end up switching files. The assumption here is that we're usually only dealing with one fd, and if there are multiple, hopefuly they are at least somewhat ordered. Could trivially be extended to cover multiple fds, if needed.
On the completion side we do the same thing, except this is trivially done just locally in io_iopoll_reap().
--- fs/io_uring.c | 139 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 118 insertions(+), 21 deletions(-)
fs/io_uring.c +118 -21
@@ -132,6 +132,19 @@ struct io_kiocb {#define IO_PLUG_THRESHOLD 2#define IO_IOPOLL_BATCH 8 +struct io_submit_state { +struct blk_plug plug; + +/* +* File reference cache +*/ +struct file *file; +unsigned int fd; +unsigned int has_refs; +unsigned int used_refs; +unsigned int ios_left; +}; +static struct kmem_cache *req_cachep;static const struct file_operations io_uring_fops;
@@ -285,9 +298,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, struct list_head *done) {void *reqs[IO_IOPOLL_BATCH ]; +int file_count, to_free; +struct file *file =NULL ;struct io_kiocb *req; -int to_free =0 ; + file_count = to_free =0 ;while (!list_empty (done)) { req = list_first_entry(done, struct io_kiocb, list);list_del (&req->list);
@@ -297,12 +312,28 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, reqs[to_free++] = req; (*nr_events)++; -fput (req->rw.ki_filp); +/* +* Batched puts of the same file, to avoid dirtying the +* file usage count multiple times, if avoidable. +*/ +if (!file) { + file = req->rw.ki_filp; + file_count =1 ; + }else if (file == req->rw.ki_filp) { + file_count++; + }else { +fput_many (file, file_count); + file = req->rw.ki_filp; + file_count =1 ; + } +if (to_free ==ARRAY_SIZE (reqs))io_free_req_many (ctx, reqs, &to_free); }io_commit_cqring (ctx); +if (file) +fput_many (file, file_count);if (to_free)io_free_req_many (ctx, reqs, &to_free); }
@@ -491,14 +522,56 @@ static void io_iopoll_req_issued(struct io_kiocb *req)list_add_tail (&req->list, &ctx->poll_list); } +static void io_file_put (struct io_submit_state *state,struct file *file) +{ +if (!state) { +fput (file); + }else if (state->file) { +int diff = state->has_refs - state->used_refs; + +if (diff) +fput_many (state->file, diff); + state->file =NULL ; + } +} + +/* +* Get as many references to a file as we have IOs left in this submission, +* assuming most submissions are for one file, or at least that each file +* has more than one submission. +*/ +static struct file *io_file_get (struct io_submit_state *state,int fd) +{ +if (!state) +return fget (fd); + +if (state->file) { +if (state->fd == fd) { + state->used_refs++; + state->ios_left--; +return state->file; + } +io_file_put (state,NULL ); + } + state->file =fget_many (fd, state->ios_left); +if (!state->file) +return NULL ; + + state->fd = fd; + state->has_refs = state->ios_left; + state->used_refs =1 ; + state->ios_left--; +return state->file; +} +static int io_prep_rw (struct io_kiocb *req,const struct io_uring_sqe *sqe, -bool force_nonblock) +bool force_nonblock,struct io_submit_state *state) {struct io_ring_ctx *ctx = req->ctx;struct kiocb *kiocb = &req->rw;int ret; - kiocb->ki_filp =fget (sqe->fd); + kiocb->ki_filp =io_file_get (state, sqe->fd);if (unlikely (!kiocb->ki_filp))return -EBADF ; kiocb->ki_pos = sqe->off;
@@ -537,7 +610,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, }return 0 ; out_fput: -fput (kiocb->ki_filp); +io_file_put (state, kiocb->ki_filp);return ret; }
@@ -577,7 +650,7 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw, }static ssize_t io_read (struct io_kiocb *req,const struct io_uring_sqe *sqe, -bool force_nonblock) +bool force_nonblock,struct io_submit_state *state) {struct iovec inline_vecs[UIO_FASTIOV ], *iovec = inline_vecs;struct kiocb *kiocb = &req->rw;
@@ -585,7 +658,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct file *file;ssize_t ret; - ret =io_prep_rw (req, sqe, force_nonblock); + ret =io_prep_rw (req, sqe, force_nonblock, state);if (ret)return ret; file = kiocb->ki_filp;
@@ -620,7 +693,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe, }static ssize_t io_write (struct io_kiocb *req,const struct io_uring_sqe *sqe, -bool force_nonblock) +bool force_nonblock,struct io_submit_state *state) {struct iovec inline_vecs[UIO_FASTIOV ], *iovec = inline_vecs;struct kiocb *kiocb = &req->rw;
@@ -628,7 +701,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct file *file;ssize_t ret; - ret =io_prep_rw (req, sqe, force_nonblock); + ret =io_prep_rw (req, sqe, force_nonblock, state);if (ret)return ret; file = kiocb->ki_filp;
@@ -722,7 +795,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe, }static int __io_submit_sqe (struct io_ring_ctx *ctx,struct io_kiocb *req, -struct sqe_submit *s,bool force_nonblock) +struct sqe_submit *s,bool force_nonblock, +struct io_submit_state *state) {const struct io_uring_sqe *sqe = s->sqe;ssize_t ret;
@@ -737,10 +811,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req, ret =io_nop (req, sqe);break ;case IORING_OP_READV : - ret =io_read (req, sqe, force_nonblock); + ret =io_read (req, sqe, force_nonblock, state);break ;case IORING_OP_WRITEV : - ret =io_write (req, sqe, force_nonblock); + ret =io_write (req, sqe, force_nonblock, state);break ;case IORING_OP_FSYNC : ret =io_fsync (req, sqe, force_nonblock);
@@ -786,7 +860,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)use_mm (ctx->sqo_mm);set_fs (USER_DS ); - ret =__io_submit_sqe (ctx, req, s, false); + ret =__io_submit_sqe (ctx, req, s, false,NULL );set_fs (old_fs);unuse_mm (ctx->sqo_mm);
@@ -799,7 +873,8 @@ static void io_sq_wq_submit_work(struct work_struct *work) current->files = old_files; } -static int io_submit_sqe (struct io_ring_ctx *ctx,struct sqe_submit *s) +static int io_submit_sqe (struct io_ring_ctx *ctx,struct sqe_submit *s, +struct io_submit_state *state) {struct io_kiocb *req;ssize_t ret;
@@ -812,7 +887,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s)if (unlikely (!req))return -EAGAIN ; - ret =__io_submit_sqe (ctx, req, s, true); + ret =__io_submit_sqe (ctx, req, s, true, state);if (ret == -EAGAIN ) {memcpy (&req->submit, s,sizeof (*s));INIT_WORK (&req->work, io_sq_wq_submit_work);
@@ -825,6 +900,26 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s)return ret; } +/* +* Batched submission is done, ensure local IO is flushed out. +*/ +static void io_submit_state_end (struct io_submit_state *state) +{ +blk_finish_plug (&state->plug); +io_file_put (state,NULL ); +} + +/* +* Start submission side cache. +*/ +static void io_submit_state_start (struct io_submit_state *state, +struct io_ring_ctx *ctx,unsigned max_ios) +{ +blk_start_plug (&state->plug); + state->file =NULL ; + state->ios_left = max_ios; +} +static void io_commit_sqring (struct io_ring_ctx *ctx) {struct io_sq_ring *ring = ctx->sq_ring;
@@ -871,11 +966,13 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)static int io_ring_submit (struct io_ring_ctx *ctx,unsigned int to_submit) { +struct io_submit_state state, *statep =NULL ;int i, ret =0 , submit =0 ; -struct blk_plug plug; -if (to_submit >IO_PLUG_THRESHOLD ) -blk_start_plug (&plug); +if (to_submit >IO_PLUG_THRESHOLD ) { +io_submit_state_start (&state, ctx, to_submit); + statep = &state; + }for (i =0 ; i < to_submit; i++) {struct sqe_submit s;
@@ -883,7 +980,7 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)if (!io_get_sqring (ctx, &s))break ; - ret =io_submit_sqe (ctx, &s); + ret =io_submit_sqe (ctx, &s, statep);if (ret) {io_drop_sqring (ctx);break ;
@@ -893,8 +990,8 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit) }io_commit_sqring (ctx); -if (to_submit >IO_PLUG_THRESHOLD ) -blk_finish_plug (&plug); +if (statep) +io_submit_state_end (statep);return submit ? submit : ret; }
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 10/13 io_uring: add io_kiocb ref count
In reply to Jens AxboeWe'll use this for the POLL implementation. Regular requests will NOT be using references, so initialize it to 0. Any real use of the io_kiocb ref will initialize it to at least 2.
--- fs/io_uring.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
fs/io_uring.c +6 -2
@@ -141,6 +141,7 @@ struct io_kiocb {struct io_ring_ctx *ctx;struct list_head list;unsigned int flags; +refcount_t refs;#define REQ_F_FORCE_NONBLOCK 1/* inline submission attempt */ #define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */
@@ -322,6 +323,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,if (req) { req->ctx = ctx; req->flags =0 ; +refcount_set (&req->refs,0 );return req; }
@@ -341,8 +343,10 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)static void io_free_req (struct io_kiocb *req) { -io_ring_drop_ctx_refs (req->ctx,1 ); -kmem_cache_free (req_cachep, req); +if (!refcount_read (&req->refs) ||refcount_dec_and_test (&req->refs)) { +io_ring_drop_ctx_refs (req->ctx,1 ); +kmem_cache_free (req_cachep, req); + } }/*
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 10/18 io_uring: batch io_kiocb allocation
In reply to Jens AxboeSimilarly to how we use the state->ios_left to know how many references to get to a file, we can use it to allocate the io_kiocb's we need in bulk.
--- fs/io_uring.c | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-)
fs/io_uring.c +36 -3
@@ -135,6 +135,13 @@ struct io_kiocb {struct io_submit_state {struct blk_plug plug; +/* +* io_kiocb alloc cache +*/ +void *reqs[IO_IOPOLL_BATCH ]; +unsigned int free_reqs; +unsigned int cur_req; +/* * File reference cache */
@@ -258,20 +265,42 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)wake_up (&ctx->wait); } -static struct io_kiocb *io_get_req (struct io_ring_ctx *ctx) +static struct io_kiocb *io_get_req (struct io_ring_ctx *ctx, +struct io_submit_state *state) { +gfp_t gfp =GFP_ATOMIC | __GFP_NOWARN;struct io_kiocb *req;/* safe to use the non tryget, as we're inside ring ref already */ percpu_ref_get (&ctx->refs); - req =kmem_cache_alloc (req_cachep,GFP_ATOMIC | __GFP_NOWARN); +if (!state) + req =kmem_cache_alloc (req_cachep, gfp); +else if (!state->free_reqs) { +size_t sz; +int ret; + + sz =min_t (size_t, state->ios_left,ARRAY_SIZE (state->reqs)); + ret =kmem_cache_alloc_bulk (req_cachep, gfp, sz, + state->reqs); +if (ret <=0 ) + goto out; + state->free_reqs = ret -1 ; + state->cur_req =1 ; + req = state->reqs[0 ]; + }else { + req = state->reqs[state->cur_req]; + state->free_reqs--; + state->cur_req++; + } +if (req) { req->ctx = ctx; req->flags =0 ;return req; } +out:io_ring_drop_ctx_refs (ctx,1 );return NULL ; }
@@ -883,7 +912,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,if (unlikely (s->sqe->flags))return -EINVAL ; - req =io_get_req (ctx); + req =io_get_req (ctx, state);if (unlikely (!req))return -EAGAIN ;
@@ -907,6 +936,9 @@ static void io_submit_state_end(struct io_submit_state *state) {blk_finish_plug (&state->plug);io_file_put (state,NULL ); +if (state->free_reqs) +kmem_cache_free_bulk (req_cachep, state->free_reqs, + &state->reqs[state->cur_req]); }/*
@@ -916,6 +948,7 @@ static void io_submit_state_start(struct io_submit_state *state, struct io_ring_ctx *ctx, unsigned max_ios) {blk_start_plug (&state->plug); + state->free_reqs =0 ; state->file =NULL ; state->ios_left = max_ios; }
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 11/18 block: implement bio helper to add iter bvec pages to bio
In reply to Jens AxboeFor an ITER_BVEC, we can just iterate the iov and add the pages to the bio directly. This requires that the caller doesn't releases the pages on IO completion, we add a BIO_HOLD_PAGES flag for that.
The current two callers of bio_iov_iter_get_pages() are updated to check if they need to release pages on completion. This makes them work with bvecs that contain kernel mapped pages already.
--- block/bio.c | 59 ++++++++++++++++++++++++++++++++------- fs/block_dev.c | 5 ++-- fs/iomap.c | 5 ++-- include/linux/blk_types.h | 1 + 4 files changed, 56 insertions(+), 14 deletions(-)
block/bio.c +49 -10
@@ -828,6 +828,23 @@ int bio_add_page(struct bio *bio, struct page *page, }EXPORT_SYMBOL (bio_add_page); +static int __bio_iov_bvec_add_pages (struct bio *bio,struct iov_iter *iter) +{ +const struct bio_vec *bv = iter->bvec; +unsigned int len; +size_t size; + + len =min_t (size_t, bv->bv_len, iter->count); + size =bio_add_page (bio, bv->bv_page, len, + bv->bv_offset + iter->iov_offset); +if (size == len) { +iov_iter_advance (iter, size); +return 0 ; + } + +return -EINVAL ; +} +#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))/**
@@ -876,23 +893,43 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) }/** -* bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio +* bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to -* @iter: iov iterator describing the region to be mapped +* @iter: iov iterator describing the region to be added +* +* This takes either an iterator pointing to user memory, or one pointing to +* kernel pages (BVEC iterator). If we're adding user pages, we pin them and +* map them into the kernel. On IO completion, the caller should put those +* pages. If we're adding kernel pages, we just have to add the pages to the +* bio directly. We don't grab an extra reference to those pages (the user +* should already have that), and we don't put the page on IO completion. +* The caller needs to check if the bio is flagged BIO_HOLD_PAGES on IO +* completion. If it isn't, then pages should be released. * -* Pins pages from *iter and appends them to @bio's bvec array. The -* pages will have to be released using put_page() when done. * The function tries, but does not guarantee, to pin as many pages as -* fit into the bio, or are requested in *iter, whatever is smaller. -* If MM encounters an error pinning the requested pages, it stops. -* Error is returned only if 0 pages could be pinned. +* fit into the bio, or are requested in *iter, whatever is smaller. If +* MM encounters an error pinning the requested pages, it stops. Error +* is returned only if 0 pages could be pinned. */ int bio_iov_iter_get_pages (struct bio *bio,struct iov_iter *iter) { +const bool is_bvec =iov_iter_is_bvec (iter);unsigned short orig_vcnt = bio->bi_vcnt; +/* +* If this is a BVEC iter, then the pages are kernel pages. Don't +* release them on IO completion. +*/ +if (is_bvec) +bio_set_flag (bio,BIO_HOLD_PAGES ); + do { - int ret = __bio_iov_iter_get_pages(bio, iter); + int ret; + + if (is_bvec) + ret = __bio_iov_bvec_add_pages(bio, iter); + else + ret = __bio_iov_iter_get_pages(bio, iter); if (unlikely(ret)) return bio->bi_vcnt > orig_vcnt ? 0 : ret;
@@ -1634,7 +1671,8 @@ static void bio_dirty_fn(struct work_struct *work) next = bio->bi_private;bio_set_pages_dirty (bio); -bio_release_pages (bio); +if (!bio_flagged (bio,BIO_HOLD_PAGES )) +bio_release_pages (bio);bio_put (bio); } }
@@ -1650,7 +1688,8 @@ void bio_check_pages_dirty(struct bio *bio) goto defer; } -bio_release_pages (bio); +if (!bio_flagged (bio,BIO_HOLD_PAGES )) +bio_release_pages (bio);bio_put (bio);return ; defer:
fs/block_dev.c +3 -2
@@ -338,8 +338,9 @@ static void blkdev_bio_end_io(struct bio *bio)struct bio_vec *bvec;int i; -bio_for_each_segment_all (bvec, bio, i) -put_page (bvec->bv_page); +if (!bio_flagged (bio,BIO_HOLD_PAGES )) +bio_for_each_segment_all (bvec, bio, i) +put_page (bvec->bv_page);bio_put (bio); } }
fs/iomap.c +3 -2
@@ -1582,8 +1582,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)struct bio_vec *bvec;int i; -bio_for_each_segment_all (bvec, bio, i) -put_page (bvec->bv_page); +if (!bio_flagged (bio,BIO_HOLD_PAGES )) +bio_for_each_segment_all (bvec, bio, i) +put_page (bvec->bv_page);bio_put (bio); } }
include/linux/blk_types.h +1 -0
@@ -215,6 +215,7 @@ struct bio {/* * bio flags */ +#define BIO_HOLD_PAGES 0/* don't put O_DIRECT pages */ #define BIO_SEG_VALID 1/* bi_phys_segments valid */ #define BIO_CLONED 2/* doesn't own data */ #define BIO_BOUNCED 3/* bio is a bounce bio */
Signature · 1 line
2.17.1
Re: [PATCH 11/18] block: implement bio helper to add iter bvec pages to bio
Quoted 1 line
+#define
BIO_HOLD_PAGES0 /* don't putO_DIRECTpages */
They don't really have to be O_DIRECT pages, just something with a permanent reference. Maybe rename this to something BIO_NO_PAGE_REF?
Otherwise looks fine:
Jens Axboe
maintainerRe: [PATCH 11/18] block: implement bio helper to add iter bvec pages to bio
Quoted 4 lines · Christoph Hellwig
Quoted 1 line
+#define
BIO_HOLD_PAGES0 /* don't putO_DIRECTpages */They don't really have to be
O_DIRECTpages, just something with a permanent reference. Maybe rename this to somethingBIO_NO_PAGE_REF?
That is a better name, thanks. I'll make the change.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 11/13 io_uring: add support for IORING_OP_POLL
In reply to Jens AxboeThis is basically a direct port of bfe4037e722e, which implements a one-shot poll command through aio. Description below is based on that commit as well. However, instead of adding a POLL command and relying on io_cancel(2) to remove it, we mimic the epoll(2) interface of having a command to add a poll notification, IORING_OP_POLL_ADD, and one to remove it again, IORING_OP_POLL_REMOVE.
To poll for a file descriptor the application should submit an sqe of type IORING_OP_POLL. It will poll the fd for the events specified in the poll_events field.
Unlike poll or epoll without EPOLLONESHOT this interface always works in one shot mode, that is once the sqe is completed, it will have to be resubmitted.
Based-on-code-from: Christoph Hellwig <[email protected]>
--- fs/io_uring.c | 245 ++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 3 + 2 files changed, 248 insertions(+)
fs/io_uring.c +245 -0
@@ -124,6 +124,7 @@ struct io_ring_ctx {spinlock_t completion_lock;unsigned poll_multi_file;struct list_head poll_list; +struct list_head cancel_list; }____cacheline_aligned_in_smp ; };
@@ -132,9 +133,19 @@ struct sqe_submit {unsigned index; }; +struct io_poll_iocb { +struct file *file; +struct wait_queue_head *head; +__poll_t events; +bool woken; +bool canceled; +struct wait_queue_entry wait; +}; +struct io_kiocb {union {struct kiocb rw; +struct io_poll_iocb poll;struct sqe_submit submit; };
@@ -206,6 +217,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)init_waitqueue_head (&ctx->wait);spin_lock_init (&ctx->completion_lock);INIT_LIST_HEAD (&ctx->poll_list); +INIT_LIST_HEAD (&ctx->cancel_list);return ctx; }
@@ -916,6 +928,232 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,return 0 ; } +static void io_poll_remove_one (struct io_kiocb *req) +{ +struct io_poll_iocb *poll = &req->poll; + +spin_lock (&poll->head->lock); +WRITE_ONCE (poll->canceled, true); +if (!list_empty (&poll->wait.entry)) { +list_del_init (&poll->wait.entry); +queue_work (req->ctx->sqo_wq, &req->work); + } +spin_unlock (&poll->head->lock); + +list_del_init (&req->list); +} + +static void io_poll_remove_all (struct io_ring_ctx *ctx) +{ +struct io_kiocb *req; + +spin_lock_irq (&ctx->completion_lock); +while (!list_empty (&ctx->cancel_list)) { + req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list); +io_poll_remove_one (req); + } +spin_unlock_irq (&ctx->completion_lock); +} + +/* +* Find a running poll command that matches one specified in sqe->addr, +* and remove it if found. +*/ +static int io_poll_remove (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_ring_ctx *ctx = req->ctx; +struct io_kiocb *poll_req, *next; +int ret = -ENOENT ; + +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; +if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || + sqe->poll_events) +return -EINVAL ; + +spin_lock_irq (&ctx->completion_lock); +list_for_each_entry_safe (poll_req, next, &ctx->cancel_list, list) { +if (sqe->addr == poll_req->user_data) { +io_poll_remove_one (poll_req); + ret =0 ; +break ; + } + } +spin_unlock_irq (&ctx->completion_lock); + +io_cqring_add_event (req->ctx, sqe->user_data, ret,0 ); +io_free_req (req); +return 0 ; +} + +static void io_poll_complete (struct io_kiocb *req,__poll_t mask) +{ +io_cqring_add_event (req->ctx, req->user_data,mangle_poll (mask),0 ); +io_fput (req); +io_free_req (req); +} + +static void io_poll_complete_work (struct work_struct *work) +{ + struct io_kiocb *req = container_of(work, struct io_kiocb, work); +struct io_poll_iocb *poll = &req->poll; +struct poll_table_struct pt = { ._key = poll->events }; +struct io_ring_ctx *ctx = req->ctx; +__poll_t mask =0 ; + +if (!READ_ONCE (poll->canceled)) + mask =vfs_poll (poll->file, &pt) & poll->events; + +/* +* Note that ->ki_cancel callers also delete iocb from active_reqs after +* calling ->ki_cancel. We need the ctx_lock roundtrip here to +* synchronize with them. In the cancellation case the list_del_init +* itself is not actually needed, but harmless so we keep it in to +* avoid further branches in the fast path. +*/ +spin_lock_irq (&ctx->completion_lock); +if (!mask && !READ_ONCE (poll->canceled)) { +add_wait_queue (poll->head, &poll->wait); +spin_unlock_irq (&ctx->completion_lock); +return ; + } +list_del_init (&req->list); +spin_unlock_irq (&ctx->completion_lock); + +io_poll_complete (req, mask); +} + +static int io_poll_wake (struct wait_queue_entry *wait,unsigned mode,int sync, +void *key) +{ + struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, + wait); + struct io_kiocb *req = container_of(poll, struct io_kiocb, poll); +struct io_ring_ctx *ctx = req->ctx; +__poll_t mask =key_to_poll (key); + + poll->woken = true; + +/* for instances that support it check for an event match first: */ +if (mask) { +if (!(mask & poll->events)) +return 0 ; + +/* try to complete the iocb inline if we can: */ +if (spin_trylock (&ctx->completion_lock)) { +list_del (&req->list); +spin_unlock (&ctx->completion_lock); + +list_del_init (&poll->wait.entry); +io_poll_complete (req, mask); +return 1 ; + } + } + +list_del_init (&poll->wait.entry); +queue_work (ctx->sqo_wq, &req->work); +return 1 ; +} + +struct io_poll_table { +struct poll_table_struct pt; +struct io_kiocb *req; +int error; +}; + +static void io_poll_queue_proc (struct file *file,struct wait_queue_head *head, +struct poll_table_struct *p) +{ + struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); + +if (unlikely (pt->req->poll.head)) { + pt->error = -EINVAL ; +return ; + } + + pt->error =0 ; + pt->req->poll.head = head; +add_wait_queue (head, &pt->req->poll.wait); +} + +static int io_poll_add (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_poll_iocb *poll = &req->poll; +struct io_ring_ctx *ctx = req->ctx; +struct io_poll_table ipt; +__poll_t mask; + +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; +if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index) +return -EINVAL ; + +INIT_WORK (&req->work, io_poll_complete_work); + poll->events =demangle_poll (sqe->poll_events) |EPOLLERR |EPOLLHUP ; + +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + poll->file = ctx->user_files[sqe->fd]; + req->flags |=REQ_F_FIXED_FILE ; + }else { + poll->file =fget (sqe->fd); + } +if (unlikely (!poll->file)) +return -EBADF ; + + poll->head =NULL ; + poll->woken = false; + poll->canceled = false; + + ipt.pt._qproc = io_poll_queue_proc; + ipt.pt._key = poll->events; + ipt.req = req; + ipt.error = -EINVAL ;/* same as no support for IOCB_CMD_POLL */ + +/* initialized the list so that we can do list_empty checks */ +INIT_LIST_HEAD (&poll->wait.entry); +init_waitqueue_func_entry (&poll->wait, io_poll_wake); + +/* one for removal from waitqueue, one for this function */ +refcount_set (&req->refs,2 ); + + mask =vfs_poll (poll->file, &ipt.pt) & poll->events; +if (unlikely (!poll->head)) { +/* we did not manage to set up a waitqueue, done */ + goto out; + } + +spin_lock_irq (&ctx->completion_lock); +spin_lock (&poll->head->lock); +if (poll->woken) { +/* wake_up context handles the rest */ + mask =0 ; + ipt.error =0 ; + }else if (mask || ipt.error) { +/* if we get an error or a mask we are done */ +WARN_ON_ONCE (list_empty (&poll->wait.entry)); +list_del_init (&poll->wait.entry); + }else { +/* actually waiting for an event */ +list_add_tail (&req->list, &ctx->cancel_list); + } +spin_unlock (&poll->head->lock); +spin_unlock_irq (&ctx->completion_lock); + +out: +if (unlikely (ipt.error)) { +if (!(sqe->flags &IOSQE_FIXED_FILE )) +fput (poll->file); +return ipt.error; + } + +if (mask) +io_poll_complete (req, mask); +io_free_req (req); +return 0 ; +} +static int __io_submit_sqe (struct io_ring_ctx *ctx,struct io_kiocb *req,struct sqe_submit *s,bool force_nonblock,struct io_submit_state *state)
@@ -951,6 +1189,12 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,case IORING_OP_FSYNC : ret =io_fsync (req, sqe, force_nonblock);break ; +case IORING_OP_POLL_ADD : + ret =io_poll_add (req, sqe); +break ; +case IORING_OP_POLL_REMOVE : + ret =io_poll_remove (req, sqe); +break ;default : ret = -EINVAL ;break ;
@@ -1794,6 +2038,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)percpu_ref_kill (&ctx->refs);mutex_unlock (&ctx->uring_lock); +io_poll_remove_all (ctx);io_iopoll_reap_events (ctx);wait_for_completion (&ctx->ctx_done);io_ring_ctx_free (ctx);
include/uapi/linux/io_uring.h +3 -0
@@ -27,6 +27,7 @@ struct io_uring_sqe {union {__kernel_rwf_t rw_flags;__u32 fsync_flags; +__u16 poll_events; };__u64 user_data;/* data to be passed back at completion time */ union {
@@ -53,6 +54,8 @@ struct io_uring_sqe {#define IORING_OP_FSYNC 3#define IORING_OP_READ_FIXED 4#define IORING_OP_WRITE_FIXED 5 +#define IORING_OP_POLL_ADD 6 +#define IORING_OP_POLL_REMOVE 7/* * sqe->fsync_flags
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 12/18 io_uring: add support for pre-mapped user IO buffers
In reply to Jens AxboeIf we have fixed user buffers, we can map them into the kernel when we setup the io_context. That avoids the need to do get_user_pages() for each and every IO.
To utilize this feature, the application must call io_uring_register() after having setup an io_uring context, passing in IORING_REGISTER_BUFFERS as the opcode. The argument must be a pointer to an iovec array, and the nr_args should contain how many iovecs the application wishes to map.
If successful, these buffers are now mapped into the kernel, eligible for IO. To use these fixed buffers, the application must use the IORING_OP_READ_FIXED and IORING_OP_WRITE_FIXED opcodes, and then set sqe->index to the desired buffer index. sqe->addr..sqe->addr+seq->len must point to somewhere inside the indexed buffer.
The application may register buffers throughout the lifetime of the io_uring context. It can call io_uring_register() with IORING_UNREGISTER_BUFFERS as the opcode to unregister the current set of buffers, and then register a new set. The application need not unregister buffers explicitly before shutting down the io_uring context.
It's perfectly valid to setup a larger buffer, and then sometimes only use parts of it for an IO. As long as the range is within the originally mapped region, it will work just fine.
For now, buffers must not be file backed. If file backed buffers are passed in, the registration will fail with -1/EOPNOTSUPP. This restriction may be relaxed in the future.
RLIMIT_MEMLOCK is used to check how much memory we can pin. A somewhat arbitrary 1G per buffer size is also imposed.
--- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + fs/io_uring.c | 357 ++++++++++++++++++++++++- include/linux/sched/user.h | 2 +- include/linux/syscalls.h | 2 + include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/io_uring.h | 13 +- kernel/sys_ni.c | 1 + 8 files changed, 367 insertions(+), 14 deletions(-)
arch/x86/entry/syscalls/syscall_32.tbl +1 -0
@@ -400,3 +400,4 @@ 386 i386 rseq sys_rseq __ia32_sys_rseq 425 i386 io_uring_setup sys_io_uring_setup __ia32_compat_sys_io_uring_setup 426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter +427 i386 io_uring_register sys_io_uring_register __ia32_sys_io_uring_register
arch/x86/entry/syscalls/syscall_64.tbl +1 -0
@@ -345,6 +345,7 @@ 334 common rseq __x64_sys_rseq 425 common io_uring_setup __x64_sys_io_uring_setup 426 common io_uring_enter __x64_sys_io_uring_enter +427 common io_uring_register __x64_sys_io_uring_register # # x32-specific system call numbers start at 512 to avoid cache impact
fs/io_uring.c +346 -11
451 lines not on this page · raw message
include/linux/sched/user.h +1 -1
@@ -40,7 +40,7 @@ struct user_struct {kuid_t uid;#if defined(CONFIG_PERF_EVENTS ) || defined(CONFIG_BPF_SYSCALL ) || \ - defined(CONFIG_NET ) + defined(CONFIG_NET ) || defined(CONFIG_IO_URING )atomic_long_t locked_vm;#endif
include/linux/syscalls.h +2 -0
@@ -314,6 +314,8 @@ asmlinkage long sys_io_uring_setup(u32 entries, struct io_uring_params __user *p);asmlinkage long sys_io_uring_enter (unsigned int fd,u32 to_submit,u32 min_complete,u32 flags); +asmlinkage long sys_io_uring_register (unsigned int fd,unsigned int op, +void __user *arg,unsigned int nr_args);/* fs/xattr.c */ asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
include/uapi/asm-generic/unistd.h +3 -1
@@ -744,9 +744,11 @@ __SYSCALL(__NR_kexec_file_load, sys_kexec_file_load) __SYSCALL(__NR_io_uring_setup, sys_io_uring_setup)#define __NR_io_uring_enter 426 __SYSCALL(__NR_io_uring_enter, sys_io_uring_enter) +#define __NR_io_uring_register 427 +__SYSCALL(__NR_io_uring_register, sys_io_uring_register)#undef __NR_syscalls -#define __NR_syscalls 427 +#define __NR_syscalls 428/* * 32 bit systems traditionally used different
include/uapi/linux/io_uring.h +12 -1
@@ -29,7 +29,10 @@ struct io_uring_sqe {__u32 fsync_flags; };__u64 user_data;/* data to be passed back at completion time */ -__u64 __pad2[3 ]; +union { +__u16 buf_index;/* index into fixed buffers, if used */ +__u64 __pad2[3 ]; + }; };/*
@@ -41,6 +44,8 @@ struct io_uring_sqe {#define IORING_OP_READV 1#define IORING_OP_WRITEV 2#define IORING_OP_FSYNC 3 +#define IORING_OP_READ_FIXED 4 +#define IORING_OP_WRITE_FIXED 5/* * sqe->fsync_flags
@@ -104,4 +109,10 @@ struct io_uring_params { struct io_cqring_offsets cq_off; }; +/* +* io_uring_register(2) opcodes and arguments +*/ +#define IORING_REGISTER_BUFFERS 0 +#define IORING_UNREGISTER_BUFFERS 1 +#endif
kernel/sys_ni.c +1 -0
@@ -49,6 +49,7 @@ COND_SYSCALL_COMPAT(io_pgetevents);COND_SYSCALL (io_uring_setup);COND_SYSCALL_COMPAT (io_uring_setup);COND_SYSCALL (io_uring_enter); +COND_SYSCALL (io_uring_register);/* fs/xattr.c */
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 12/13 io_uring: allow workqueue item to handle multiple buffered requests
In reply to Jens AxboeRight now we punt any buffered request that ends up triggering an -EAGAIN to an async workqueue. This works fine in terms of providing async execution of them, but it also can create quite a lot of work queue items. For sequentially buffered IO, it's advantageous to serialize the issue of them. For reads, the first one will trigger a read-ahead, and subsequent request merely end up waiting on later pages to complete. For writes, devices usually respond better to streamed sequential writes.
Add state to track the last buffered request we punted to a work queue, and if the next one is sequential to the previous, attempt to get the previous work item to handle it. We limit the number of sequential add-ons to the a multiple (8) of the max read-ahead size of the file. This should be a good number for both reads and wries, as it defines the max IO size the device can do directly.
This drastically cuts down on the number of context switches we need to handle buffered sequential IO, and a basic test case of copying a big file with io_uring sees a 5x speedup.
--- fs/io_uring.c | 231 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 194 insertions(+), 37 deletions(-)
fs/io_uring.c +194 -37
@@ -68,6 +68,16 @@ struct io_mapped_ubuf {unsigned int nr_bvecs; }; +struct async_list { +spinlock_t lock; +atomic_t cnt; +struct list_head list; + +struct file *file; +off_t io_end; +size_t io_pages; +}; + struct io_ring_ctx { struct { struct percpu_ref refs;
@@ -126,6 +136,8 @@ struct io_ring_ctx {struct list_head poll_list;struct list_head cancel_list; }____cacheline_aligned_in_smp ; + +struct async_list pending_async[2 ]; };struct sqe_submit {
@@ -157,6 +169,7 @@ struct io_kiocb {#define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */ #define REQ_F_FIXED_FILE 8/* ctx owns file */ +#define REQ_F_SEQ_PREV 16/* sequential with previous */ u64 user_data;u64 res;
@@ -200,6 +213,7 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref)static struct io_ring_ctx *io_ring_ctx_alloc (struct io_uring_params *p) {struct io_ring_ctx *ctx; +int i; ctx =kzalloc (sizeof (*ctx),GFP_KERNEL );if (!ctx)
@@ -215,6 +229,11 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)init_completion (&ctx->ctx_done);mutex_init (&ctx->uring_lock);init_waitqueue_head (&ctx->wait); +for (i =0 ; i <ARRAY_SIZE (ctx->pending_async); i++) { +spin_lock_init (&ctx->pending_async[i].lock); +INIT_LIST_HEAD (&ctx->pending_async[i].list); +atomic_set (&ctx->pending_async[i].cnt,0 ); + }spin_lock_init (&ctx->completion_lock);INIT_LIST_HEAD (&ctx->poll_list);INIT_LIST_HEAD (&ctx->cancel_list);
@@ -774,6 +793,39 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,return import_iovec (rw, buf, sqe->len,UIO_FASTIOV , iovec, iter); } +static void io_async_list_note (int rw,struct io_kiocb *req,size_t len) +{ +struct async_list *async_list = &req->ctx->pending_async[rw]; +struct kiocb *kiocb = &req->rw; +struct file *filp = kiocb->ki_filp; +off_t io_end = kiocb->ki_pos + len; + +if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) { +unsigned long max_pages; + +/* Use 8x RA size as a decent limiter for both reads/writes */ + max_pages = filp->f_ra.ra_pages; +if (!max_pages) + max_pages =VM_MAX_READAHEAD >> (PAGE_SHIFT -10 ); + max_pages *=8 ; + + len >>=PAGE_SHIFT ; +if (async_list->io_pages + len <= max_pages) { + req->flags |=REQ_F_SEQ_PREV ; + async_list->io_pages += len; + }else { + io_end =0 ; + async_list->io_pages =0 ; + } + } + +if (async_list->file != filp) { + async_list->io_pages =0 ; + async_list->file = filp; + } + async_list->io_end = io_end; +} +static ssize_t io_read (struct io_kiocb *req,const struct io_uring_sqe *sqe,bool force_nonblock,struct io_submit_state *state) {
@@ -781,6 +833,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;struct iov_iter iter;struct file *file; +size_t iov_count;ssize_t ret; ret =io_prep_rw (req, sqe, force_nonblock, state);
@@ -799,16 +852,19 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (ret) goto out_fput; - ret =rw_verify_area (READ , file, &kiocb->ki_pos,iov_iter_count (&iter)); + iov_count =iov_iter_count (&iter); + ret =rw_verify_area (READ , file, &kiocb->ki_pos, iov_count);if (!ret) {ssize_t ret2;/* Catch -EAGAIN return for forced non-blocking submission */ ret2 =call_read_iter (file, kiocb, &iter); -if (!force_nonblock || ret2 != -EAGAIN ) +if (!force_nonblock || ret2 != -EAGAIN ) {io_rw_done (kiocb, ret2); -else + }else { +io_async_list_note (READ , req, iov_count); ret = -EAGAIN ; + } }kfree (iovec); out_fput:
@@ -824,6 +880,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;struct iov_iter iter;struct file *file; +size_t iov_count;ssize_t ret; ret =io_prep_rw (req, sqe, force_nonblock, state);
@@ -831,10 +888,6 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,return ret; file = kiocb->ki_filp; - ret = -EAGAIN ; -if (force_nonblock && !(kiocb->ki_flags &IOCB_DIRECT )) - goto out_fput; - ret = -EBADF ;if (unlikely (!(file->f_mode &FMODE_WRITE ))) goto out_fput;
@@ -846,8 +899,15 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (ret) goto out_fput; - ret =rw_verify_area (WRITE , file, &kiocb->ki_pos, -iov_iter_count (&iter)); + iov_count =iov_iter_count (&iter); + + ret = -EAGAIN ; +if (force_nonblock && !(kiocb->ki_flags &IOCB_DIRECT )) { +io_async_list_note (WRITE , req, iov_count); + goto out_free; + } + + ret =rw_verify_area (WRITE , file, &kiocb->ki_pos, iov_count);if (!ret) {/* * Open-code file_start_write here to grab freeze protection,
@@ -865,6 +925,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_flags |=IOCB_WRITE ;io_rw_done (kiocb,call_write_iter (file, kiocb, &iter)); } +out_free:kfree (iovec); out_fput:if (unlikely (ret))
@@ -1212,6 +1273,21 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,return 0 ; } +static struct async_list *io_async_list_from_sqe (struct io_ring_ctx *ctx, +const struct io_uring_sqe *sqe) +{ +switch (sqe->opcode) { +case IORING_OP_READV : +case IORING_OP_READ_FIXED : +return &ctx->pending_async[READ ]; +case IORING_OP_WRITEV : +case IORING_OP_WRITE_FIXED : +return &ctx->pending_async[WRITE ]; +default : +return NULL ; + } +} +static inline bool io_sqe_needs_user (const struct io_uring_sqe *sqe) { return !(sqe->opcode == IORING_OP_READ_FIXED ||
@@ -1221,50 +1297,124 @@ static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)static void io_sq_wq_submit_work (struct work_struct *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); -struct sqe_submit *s = &req->submit; -u64 user_data = s->sqe->user_data;struct io_ring_ctx *ctx = req->ctx; +struct mm_struct *cur_mm =NULL ;struct files_struct *old_files; +struct async_list *async_list; +LIST_HEAD (req_list);mm_segment_t old_fs; -bool needs_user;int ret; -/* Ensure we clear previously set forced non-block flag */ - req->flags &= ~REQ_F_FORCE_NONBLOCK ; - old_files = current->files; current->files = ctx->sqo_files; + async_list =io_async_list_from_sqe (ctx, req->submit.sqe); +restart: +do { +struct sqe_submit *s = &req->submit; +u64 user_data = s->sqe->user_data; + +/* Ensure we clear previously set forced non-block flag */ + req->flags &= ~REQ_F_FORCE_NONBLOCK ; + + ret =0 ; +if (io_sqe_needs_user (s->sqe) && !cur_mm) { +if (!mmget_not_zero (ctx->sqo_mm)) { + ret = -EFAULT ; + }else { + cur_mm = ctx->sqo_mm;; +use_mm (ctx->sqo_mm); + old_fs =get_fs (); +set_fs (USER_DS ); + } + } + +if (!ret) + ret =__io_submit_sqe (ctx, req, s, false,NULL ); +if (ret) { +io_cqring_add_event (ctx, user_data, ret,0 ); +io_free_req (req); + } +if (!async_list) +break ; +if (!list_empty (&req_list)) { + req = list_first_entry(&req_list, struct io_kiocb, + list); +list_del (&req->list); +continue ; + } +if (list_empty (&async_list->list)) +break ; + + req =NULL ; +spin_lock (&async_list->lock); +if (list_empty (&async_list->list)) { +spin_unlock (&async_list->lock); +break ; + } +list_splice_init (&async_list->list, &req_list); +spin_unlock (&async_list->lock); + + req = list_first_entry(&req_list, struct io_kiocb, list); +list_del (&req->list); + }while (req); +/* -* If we're doing IO to fixed buffers, we don't need to get/set -* user context +* Rare case of racing with a submitter. If we find the count has +* dropped to zero AND we have pending work items, then restart +* the processing. This is a tiny race window. */ - needs_user =io_sqe_needs_user (s->sqe); -if (needs_user) { -if (!mmget_not_zero (ctx->sqo_mm)) { - ret = -EFAULT ; - goto err; + ret =atomic_dec_return (&async_list->cnt); +while (!ret && !list_empty (&async_list->list)) { +spin_lock (&async_list->lock); +atomic_inc (&async_list->cnt); +list_splice_init (&async_list->list, &req_list); +spin_unlock (&async_list->lock); + +if (!list_empty (&req_list)) { + req = list_first_entry(&req_list, struct io_kiocb, + list); +list_del (&req->list); + goto restart; } -use_mm (ctx->sqo_mm); - old_fs =get_fs (); -set_fs (USER_DS ); + ret =atomic_dec_return (&async_list->cnt); } - ret =__io_submit_sqe (ctx, req, s, false,NULL ); - -if (needs_user) { +if (cur_mm) {set_fs (old_fs); -unuse_mm (ctx->sqo_mm); -mmput (ctx->sqo_mm); - } -err: -if (ret) { -io_cqring_add_event (ctx, user_data, ret,0 ); -io_free_req (req); +unuse_mm (cur_mm); +mmput (cur_mm); } current->files = old_files; } +/* +* See if we can piggy back onto previously submitted work, that is still +* running. We currently only allow this if the new request is sequential +* to the previous one we punted. +*/ +static bool io_add_to_prev_work (struct async_list *list,struct io_kiocb *req) +{ +bool ret = false; + +if (!list) +return false; +if (!(req->flags &REQ_F_SEQ_PREV )) +return false; +if (!atomic_read (&list->cnt)) +return false; + + ret = true; +spin_lock (&list->lock); +list_add_tail (&req->list, &list->list); +if (!atomic_read (&list->cnt)) { +list_del_init (&req->list); + ret = false; + } +spin_unlock (&list->lock); +return ret; +} +static int io_submit_sqe (struct io_ring_ctx *ctx,struct sqe_submit *s,struct io_submit_state *state) {
@@ -1281,9 +1431,16 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, ret =__io_submit_sqe (ctx, req, s, true, state);if (ret == -EAGAIN ) { +struct async_list *list; + + list =io_async_list_from_sqe (ctx, s->sqe);memcpy (&req->submit, s,sizeof (*s)); -INIT_WORK (&req->work, io_sq_wq_submit_work); -queue_work (ctx->sqo_wq, &req->work); +if (!io_add_to_prev_work (list, req)) { +if (list) +atomic_inc (&list->cnt); +INIT_WORK (&req->work, io_sq_wq_submit_work); +queue_work (ctx->sqo_wq, &req->work); + } ret =0 ; }if (ret)
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 13/18 io_uring: add file set registration
In reply to Jens AxboeWe normally have to fget/fput for each IO we do on a file. Even with the batching we do, the cost of the atomic inc/dec of the file usage count adds up.
This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes for the io_uring_register(2) system call. The arguments passed in must be an array of __s32 holding file descriptors, and nr_args should hold the number of file descriptors the application wishes to pin for the duration of the io_uring context (or until IORING_UNREGISTER_FILES is called).
When used, the application must set IOSQE_FIXED_FILE in the sqe->flags member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd to the index in the array passed in to IORING_REGISTER_FILES.
Files are automatically unregistered when the io_uring context is torn down. An application need only unregister if it wishes to register a few set of fds.
--- fs/io_uring.c | 125 +++++++++++++++++++++++++++++----- include/uapi/linux/io_uring.h | 9 ++- 2 files changed, 116 insertions(+), 18 deletions(-)
fs/io_uring.c +108 -17
@@ -100,6 +100,10 @@ struct io_ring_ctx {struct fasync_struct *cq_fasync; }____cacheline_aligned_in_smp ; +/* if used, fixed file set */ +struct file **user_files; +unsigned nr_user_files; +/* if used, fixed mapped user buffers */ unsigned nr_user_bufs;struct io_mapped_ubuf *user_bufs;
@@ -137,6 +141,7 @@ struct io_kiocb {#define REQ_F_FORCE_NONBLOCK 1/* inline submission attempt */ #define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */ +#define REQ_F_FIXED_FILE 8/* ctx owns file */ u64 user_data;u64 res;
@@ -359,15 +364,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,* Batched puts of the same file, to avoid dirtying the * file usage count multiple times, if avoidable. */ -if (!file) { - file = req->rw.ki_filp; - file_count =1 ; - }else if (file == req->rw.ki_filp) { - file_count++; - }else { -fput_many (file, file_count); - file = req->rw.ki_filp; - file_count =1 ; +if (!(req->flags &REQ_F_FIXED_FILE )) { +if (!file) { + file = req->rw.ki_filp; + file_count =1 ; + }else if (file == req->rw.ki_filp) { + file_count++; + }else { +fput_many (file, file_count); + file = req->rw.ki_filp; + file_count =1 ; + } }if (to_free ==ARRAY_SIZE (reqs))
@@ -504,13 +511,19 @@ static void kiocb_end_write(struct kiocb *kiocb) } } +static void io_fput (struct io_kiocb *req) +{ +if (!(req->flags &REQ_F_FIXED_FILE )) +fput (req->rw.ki_filp); +} +static void io_complete_rw (struct kiocb *kiocb,long res,long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);kiocb_end_write (kiocb); -fput (kiocb->ki_filp); +io_fput (req);io_cqring_add_event (req->ctx, req->user_data, res,0 );io_free_req (req); }
@@ -614,7 +627,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;int ret; - kiocb->ki_filp =io_file_get (state, sqe->fd); +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + kiocb->ki_filp = ctx->user_files[sqe->fd]; + req->flags |=REQ_F_FIXED_FILE ; + }else { + kiocb->ki_filp =io_file_get (state, sqe->fd); + }if (unlikely (!kiocb->ki_filp))return -EBADF ; kiocb->ki_pos = sqe->off;
@@ -653,7 +673,8 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, }return 0 ; out_fput: -io_file_put (state, kiocb->ki_filp); +if (!(sqe->flags &IOSQE_FIXED_FILE )) +io_file_put (state, kiocb->ki_filp);return ret; }
@@ -770,7 +791,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,kfree (iovec); out_fput:if (unlikely (ret)) -fput (file); +io_fput (req);return ret; }
@@ -825,7 +846,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,kfree (iovec); out_fput:if (unlikely (ret)) -fput (file); +io_fput (req);return ret; }
@@ -863,14 +884,23 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (unlikely (sqe->fsync_flags & ~IORING_FSYNC_DATASYNC ))return -EINVAL ; - file =fget (sqe->fd); +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + file = ctx->user_files[sqe->fd]; + }else { + file =fget (sqe->fd); + } +if (unlikely (!file))return -EBADF ; ret =vfs_fsync_range (file, sqe->off, end >0 ? end :LLONG_MAX , sqe->fsync_flags &IORING_FSYNC_DATASYNC ); -fput (file); +if (!(sqe->flags &IOSQE_FIXED_FILE )) +fput (file); +io_cqring_add_event (ctx, sqe->user_data, ret,0 );io_free_req (req);return 0 ;
@@ -988,7 +1018,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,ssize_t ret;/* enforce forwards compatibility on users */ -if (unlikely (s->sqe->flags)) +if (unlikely (s->sqe->flags & ~IOSQE_FIXED_FILE ))return -EINVAL ; req =io_get_req (ctx, state);
@@ -1173,6 +1203,57 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,return ret; } +static int io_sqe_files_unregister (struct io_ring_ctx *ctx) +{ +int i; + +if (!ctx->user_files) +return -ENXIO ; + +for (i =0 ; i < ctx->nr_user_files; i++) +fput (ctx->user_files[i]); + +kfree (ctx->user_files); + ctx->user_files =NULL ; + ctx->nr_user_files =0 ; +return 0 ; +} + +static int io_sqe_files_register (struct io_ring_ctx *ctx,void __user *arg, +unsigned nr_args) +{ +__s32 __user *fds = (__s32 __user *) arg; +int fd, i, ret =0 ; + +if (ctx->user_files) +return -EBUSY ; +if (!nr_args) +return -EINVAL ; + + ctx->user_files =kcalloc (nr_args,sizeof (struct file *),GFP_KERNEL ); +if (!ctx->user_files) +return -ENOMEM ; + +for (i =0 ; i < nr_args; i++) { + ret = -EFAULT ; +if (copy_from_user (&fd, &fds[i],sizeof (fd))) +break ; + + ctx->user_files[i] =fget (fd); + + ret = -EBADF ; +if (!ctx->user_files[i]) +break ; + ctx->nr_user_files++; + ret =0 ; + } + +if (ret) +io_sqe_files_unregister (ctx); + +return ret; +} +static int io_sq_offload_start (struct io_ring_ctx *ctx) {int ret;
@@ -1468,6 +1549,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)io_sq_offload_stop (ctx);io_iopoll_reap_events (ctx);io_free_scq_urings (ctx); +io_sqe_files_unregister (ctx);io_sqe_buffer_unregister (ctx);percpu_ref_exit (&ctx->refs);io_unaccount_mem (ctx,ring_pages (ctx->sq_entries, ctx->cq_entries));
@@ -1780,6 +1862,15 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,break ; ret =io_sqe_buffer_unregister (ctx);break ; +case IORING_REGISTER_FILES : + ret =io_sqe_files_register (ctx, arg, nr_args); +break ; +case IORING_UNREGISTER_FILES : + ret = -EINVAL ; +if (arg || nr_args) +break ; + ret =io_sqe_files_unregister (ctx); +break ;default : ret = -EINVAL ;break ;
include/uapi/linux/io_uring.h +8 -1
@@ -18,7 +18,7 @@*/ struct io_uring_sqe {__u8 opcode;/* type of operation for this sqe */ -__u8 flags;/* as of now unused */ +__u8 flags;/* IOSQE_ flags */ __u16 ioprio;/* ioprio for the request */ __s32 fd;/* file descriptor to do IO on */ __u64 off;/* offset into file */
@@ -35,6 +35,11 @@ struct io_uring_sqe { }; }; +/* +* sqe->flags +*/ +#define IOSQE_FIXED_FILE (1 << 0)/* use fixed fileset */ +/* * io_uring_setup() flags */
@@ -114,5 +119,7 @@ struct io_uring_params {*/ #define IORING_REGISTER_BUFFERS 0#define IORING_UNREGISTER_BUFFERS 1 +#define IORING_REGISTER_FILES 2 +#define IORING_UNREGISTER_FILES 3#endif
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 13/13 io_uring: add io_uring_event cache hit information
In reply to Jens AxboeAdd hint on whether a read was served out of the page cache, or if it hit media. This is useful for buffered async IO, O_DIRECT reads would never have this set (for obvious reasons).
If the read hit page cache, cqe->flags will have IOCQE_FLAG_CACHEHIT set.
--- fs/io_uring.c | 7 ++++++- include/uapi/linux/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-)
fs/io_uring.c +6 -1
@@ -561,11 +561,16 @@ static void io_fput(struct io_kiocb *req)static void io_complete_rw (struct kiocb *kiocb,long res,long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw); +unsigned ev_flags =0 ;kiocb_end_write (kiocb);io_fput (req); -io_cqring_add_event (req->ctx, req->user_data, res,0 ); + +if (res >0 && (req->flags &REQ_F_FORCE_NONBLOCK )) + ev_flags =IOCQE_FLAG_CACHEHIT ; + +io_cqring_add_event (req->ctx, req->user_data, res, ev_flags);io_free_req (req); }
include/uapi/linux/io_uring.h +5 -0
@@ -71,6 +71,11 @@ struct io_uring_cqe {__u32 flags; }; +/* +* io_uring_event->flags +*/ +#define IOCQE_FLAG_CACHEHIT (1 << 0)/* IO did not hit media */ +/* * Magic offsets for the application to mmap the data it needs */
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 14/18 io_uring: add submission polling
In reply to Jens AxboeThis enables an application to do IO, without ever entering the kernel. By using the SQ ring to fill in new sqes and watching for completions on the CQ ring, we can submit and reap IOs without doing a single system call. The kernel side thread will poll for new submissions, and in case of HIPRI/polled IO, it'll also poll for completions.
Proof of concept. If the thread has been idle for 1 second, it will set sq_ring->flags |= IORING_SQ_NEED_WAKEUP. The application will have to call io_uring_enter() to start things back up again. If IO is kept busy, that will never be needed. Basically an application that has this feature enabled will guard it's io_uring_enter(2) call with:
read_barrier();
if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP)
io_uring_enter(fd, to_submit, 0, 0);
instead of calling it unconditionally.
Improvements:
1) Maybe have smarter backoff. Busy loop for X time, then go to monitor/mwait, finally the schedule we have now after an idle second. Might not be worth the complexity.
2) Probably want the application to pass in the appropriate grace period, not hard code it at 1 second.
--- fs/io_uring.c | 219 +++++++++++++++++++++++++++++++++- include/uapi/linux/io_uring.h | 10 +- 2 files changed, 222 insertions(+), 7 deletions(-)
fs/io_uring.c +213 -6
@@ -24,6 +24,7 @@#include <linux/percpu.h> #include <linux/slab.h> #include <linux/workqueue.h> +#include <linux/kthread.h> #include <linux/blkdev.h> #include <linux/bvec.h> #include <linux/anon_inodes.h>
@@ -87,8 +88,10 @@ struct io_ring_ctx {/* IO offload */ struct workqueue_struct *sqo_wq; +struct task_struct *sqo_thread;/* if using sq thread polling */ struct mm_struct *sqo_mm;struct files_struct *sqo_files; +wait_queue_head_t sqo_wait;struct {/* CQ ring */
@@ -264,6 +267,9 @@ static void __io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,if (waitqueue_active (&ctx->wait))wake_up (&ctx->wait); +if ((ctx->flags &IORING_SETUP_SQPOLL ) && +waitqueue_active (&ctx->sqo_wait)) +wake_up (&ctx->sqo_wait); } static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
@@ -1106,6 +1112,168 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)return false; } +static int io_submit_sqes (struct io_ring_ctx *ctx,struct sqe_submit *sqes, +unsigned int nr,bool mm_fault) +{ +struct io_submit_state state, *statep =NULL ; +int ret, i, submitted =0 ; + +if (nr >IO_PLUG_THRESHOLD ) { +io_submit_state_start (&state, ctx, nr); + statep = &state; + } + +for (i =0 ; i < nr; i++) { +if (unlikely (mm_fault)) + ret = -EFAULT ; +else + ret =io_submit_sqe (ctx, &sqes[i], statep); +if (!ret) { + submitted++; +continue ; + } + +io_cqring_add_event (ctx, sqes[i].sqe->user_data, ret,0 ); + } + +if (statep) +io_submit_state_end (&state); + +return submitted; +} + +static int io_sq_thread (void *data) +{ +struct sqe_submit sqes[IO_IOPOLL_BATCH ]; +struct io_ring_ctx *ctx = data; +struct mm_struct *cur_mm =NULL ; +struct files_struct *old_files; +mm_segment_t old_fs; +DEFINE_WAIT (wait); +unsigned inflight; +unsigned long timeout; + + old_files = current->files; + current->files = ctx->sqo_files; + + old_fs =get_fs (); +set_fs (USER_DS ); + + timeout = inflight =0 ; +while (!kthread_should_stop ()) { +bool all_fixed, mm_fault = false; +int i; + +if (inflight) { +unsigned int nr_events =0 ; + +/* +* Normal IO, just pretend everything completed. +* We don't have to poll completions for that. +*/ +if (ctx->flags &IORING_SETUP_IOPOLL ) { +/* +* App should not use IORING_ENTER_GETEVENTS +* with thread polling, but if it does, then +* ensure we are mutually exclusive. +*/ +if (mutex_trylock (&ctx->uring_lock)) { +io_iopoll_check (ctx, &nr_events,0 ); +mutex_unlock (&ctx->uring_lock); + } + }else { + nr_events = inflight; + } + + inflight -= nr_events; +if (!inflight) + timeout = jiffies +HZ ; + } + +if (!io_get_sqring (ctx, &sqes[0 ])) { +/* +* We're polling, let us spin for a second without +* work before going to sleep. +*/ +if (inflight || !time_after (jiffies, timeout)) { +cpu_relax (); +continue ; + } + +/* +* Drop cur_mm before scheduling, we can't hold it for +* long periods (or over schedule()). Do this before +* adding ourselves to the waitqueue, as the unuse/drop +* may sleep. +*/ +if (cur_mm) { +unuse_mm (cur_mm); +mmput (cur_mm); + cur_mm =NULL ; + } + +prepare_to_wait (&ctx->sqo_wait, &wait, +TASK_INTERRUPTIBLE ); + +/* Tell userspace we may need a wakeup call */ + ctx->sq_ring->flags |=IORING_SQ_NEED_WAKEUP ; +smp_wmb (); + +if (!io_get_sqring (ctx, &sqes[0 ])) { +if (kthread_should_park ()) +kthread_parkme (); +if (kthread_should_stop ()) { +finish_wait (&ctx->sqo_wait, &wait); +break ; + } +if (signal_pending (current)) +flush_signals (current); +schedule (); +finish_wait (&ctx->sqo_wait, &wait); + + ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP ; +smp_wmb (); +continue ; + } +finish_wait (&ctx->sqo_wait, &wait); + + ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP ; +smp_wmb (); + } + + i =0 ; + all_fixed = true; +do { +if (all_fixed &&io_sqe_needs_user (sqes[i].sqe)) + all_fixed = false; + + i++; +if (i ==ARRAY_SIZE (sqes)) +break ; + }while (io_get_sqring (ctx, &sqes[i])); + +io_commit_sqring (ctx); + +/* Unless all new commands are FIXED regions, grab mm */ +if (!all_fixed && !cur_mm) { + mm_fault = !mmget_not_zero (ctx->sqo_mm); +if (!mm_fault) { +use_mm (ctx->sqo_mm); + cur_mm = ctx->sqo_mm; + } + } + + inflight +=io_submit_sqes (ctx, sqes, i, mm_fault); + } + current->files = old_files; +set_fs (old_fs); +if (cur_mm) { +unuse_mm (cur_mm); +mmput (cur_mm); + } +return 0 ; +} +static int io_ring_submit (struct io_ring_ctx *ctx,unsigned int to_submit) {struct io_submit_state state, *statep =NULL ;
@@ -1179,9 +1347,14 @@ static int __io_uring_enter(struct io_ring_ctx *ctx, unsigned to_submit,int ret =0 ;if (to_submit) { - ret =io_ring_submit (ctx, to_submit); -if (ret <0 ) -return ret; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +wake_up (&ctx->sqo_wait); + ret = to_submit; + }else { + ret =io_ring_submit (ctx, to_submit); +if (ret <0 ) +return ret; + } }if (flags &IORING_ENTER_GETEVENTS ) {unsigned nr_events =0 ;
@@ -1254,10 +1427,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,return ret; } -static int io_sq_offload_start (struct io_ring_ctx *ctx) +static int io_sq_offload_start (struct io_ring_ctx *ctx, +struct io_uring_params *p) {int ret; +init_waitqueue_head (&ctx->sqo_wait); ctx->sqo_mm = current->mm;/*
@@ -1270,6 +1445,27 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)if (!ctx->sqo_files) goto err; +if (ctx->flags &IORING_SETUP_SQPOLL ) { +if (p->flags &IORING_SETUP_SQ_AFF ) { + ctx->sqo_thread =kthread_create_on_cpu (io_sq_thread, + ctx, p->sq_thread_cpu, +"io_uring-sq" ); + }else { + ctx->sqo_thread =kthread_create (io_sq_thread, ctx, +"io_uring-sq" ); + } +if (IS_ERR (ctx->sqo_thread)) { + ret =PTR_ERR (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + goto err; + } +wake_up_process (ctx->sqo_thread); + }else if (p->flags &IORING_SETUP_SQ_AFF ) { +/* Can't have SQ_AFF without SQPOLL */ + ret = -EINVAL ; + goto err; + } +/* Do QD, or 2 * CPUS, whatever is smallest */ ctx->sqo_wq =alloc_workqueue ("io_ring-wq" ,WQ_UNBOUND |WQ_FREEZABLE ,min (ctx->sq_entries -1 ,2 *num_online_cpus ()));
@@ -1280,6 +1476,11 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)return 0 ; err: +if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + }if (ctx->sqo_files) ctx->sqo_files =NULL ; ctx->sqo_mm =NULL ;
@@ -1288,6 +1489,11 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)static void io_sq_offload_stop (struct io_ring_ctx *ctx) { +if (ctx->sqo_thread) { +kthread_park (ctx->sqo_thread); +kthread_stop (ctx->sqo_thread); + ctx->sqo_thread =NULL ; + }if (ctx->sqo_wq) {destroy_workqueue (ctx->sqo_wq); ctx->sqo_wq =NULL ;
@@ -1780,7 +1986,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p,if (ret) goto err; - ret =io_sq_offload_start (ctx); + ret =io_sq_offload_start (ctx, p);if (ret) goto err;
@@ -1815,7 +2021,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params,return -EINVAL ; } -if (p.flags & ~IORING_SETUP_IOPOLL ) +if (p.flags & ~(IORING_SETUP_IOPOLL |IORING_SETUP_SQPOLL | +IORING_SETUP_SQ_AFF ))return -EINVAL ; ret =io_uring_create (entries, &p, compat);
include/uapi/linux/io_uring.h +9 -1
@@ -44,6 +44,8 @@ struct io_uring_sqe {* io_uring_setup() flags */ #define IORING_SETUP_IOPOLL (1 << 0)/* io_context is polled */ +#define IORING_SETUP_SQPOLL (1 << 1)/* SQ poll thread */ +#define IORING_SETUP_SQ_AFF (1 << 2)/* sq_thread_cpu is valid */ #define IORING_OP_NOP 0#define IORING_OP_READV 1
@@ -87,6 +89,11 @@ struct io_sqring_offsets {__u32 resv[3 ]; }; +/* +* sq_ring->flags +*/ +#define IORING_SQ_NEED_WAKEUP (1 << 0)/* needs io_uring_enter wakeup */ +struct io_cqring_offsets {__u32 head;__u32 tail;
@@ -109,7 +116,8 @@ struct io_uring_params {__u32 sq_entries;__u32 cq_entries;__u32 flags; -__u16 resv[10 ]; +__u16 sq_thread_cpu; +__u16 resv[9 ];struct io_sqring_offsets sq_off;struct io_cqring_offsets cq_off; };
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 15/18 io_uring: add io_kiocb ref count
In reply to Jens AxboeWe'll use this for the POLL implementation. Regular requests will NOT be using references, so initialize it to 0. Any real use of the io_kiocb ref will initialize it to at least 2.
--- fs/io_uring.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
fs/io_uring.c +6 -2
@@ -141,6 +141,7 @@ struct io_kiocb {struct io_ring_ctx *ctx;struct list_head list;unsigned int flags; +refcount_t refs;#define REQ_F_FORCE_NONBLOCK 1/* inline submission attempt */ #define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */
@@ -322,6 +323,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,if (req) { req->ctx = ctx; req->flags =0 ; +refcount_set (&req->refs,0 );return req; }
@@ -341,8 +343,10 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)static void io_free_req (struct io_kiocb *req) { -io_ring_drop_ctx_refs (req->ctx,1 ); -kmem_cache_free (req_cachep, req); +if (!refcount_read (&req->refs) ||refcount_dec_and_test (&req->refs)) { +io_ring_drop_ctx_refs (req->ctx,1 ); +kmem_cache_free (req_cachep, req); + } }/*
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 16/18 io_uring: add support for IORING_OP_POLL
In reply to Jens AxboeThis is basically a direct port of bfe4037e722e, which implements a one-shot poll command through aio. Description below is based on that commit as well. However, instead of adding a POLL command and relying on io_cancel(2) to remove it, we mimic the epoll(2) interface of having a command to add a poll notification, IORING_OP_POLL_ADD, and one to remove it again, IORING_OP_POLL_REMOVE.
To poll for a file descriptor the application should submit an sqe of type IORING_OP_POLL. It will poll the fd for the events specified in the poll_events field.
Unlike poll or epoll without EPOLLONESHOT this interface always works in one shot mode, that is once the sqe is completed, it will have to be resubmitted.
Based-on-code-from: Christoph Hellwig <[email protected]>
--- fs/io_uring.c | 245 ++++++++++++++++++++++++++++++++++ include/uapi/linux/io_uring.h | 3 + 2 files changed, 248 insertions(+)
fs/io_uring.c +245 -0
@@ -124,6 +124,7 @@ struct io_ring_ctx {spinlock_t completion_lock;unsigned poll_multi_file;struct list_head poll_list; +struct list_head cancel_list; }____cacheline_aligned_in_smp ; };
@@ -132,9 +133,19 @@ struct sqe_submit {unsigned index; }; +struct io_poll_iocb { +struct file *file; +struct wait_queue_head *head; +__poll_t events; +bool woken; +bool canceled; +struct wait_queue_entry wait; +}; +struct io_kiocb {union {struct kiocb rw; +struct io_poll_iocb poll;struct sqe_submit submit; };
@@ -206,6 +217,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)init_waitqueue_head (&ctx->wait);spin_lock_init (&ctx->completion_lock);INIT_LIST_HEAD (&ctx->poll_list); +INIT_LIST_HEAD (&ctx->cancel_list);return ctx; }
@@ -916,6 +928,232 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,return 0 ; } +static void io_poll_remove_one (struct io_kiocb *req) +{ +struct io_poll_iocb *poll = &req->poll; + +spin_lock (&poll->head->lock); +WRITE_ONCE (poll->canceled, true); +if (!list_empty (&poll->wait.entry)) { +list_del_init (&poll->wait.entry); +queue_work (req->ctx->sqo_wq, &req->work); + } +spin_unlock (&poll->head->lock); + +list_del_init (&req->list); +} + +static void io_poll_remove_all (struct io_ring_ctx *ctx) +{ +struct io_kiocb *req; + +spin_lock_irq (&ctx->completion_lock); +while (!list_empty (&ctx->cancel_list)) { + req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list); +io_poll_remove_one (req); + } +spin_unlock_irq (&ctx->completion_lock); +} + +/* +* Find a running poll command that matches one specified in sqe->addr, +* and remove it if found. +*/ +static int io_poll_remove (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_ring_ctx *ctx = req->ctx; +struct io_kiocb *poll_req, *next; +int ret = -ENOENT ; + +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; +if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index || + sqe->poll_events) +return -EINVAL ; + +spin_lock_irq (&ctx->completion_lock); +list_for_each_entry_safe (poll_req, next, &ctx->cancel_list, list) { +if (sqe->addr == poll_req->user_data) { +io_poll_remove_one (poll_req); + ret =0 ; +break ; + } + } +spin_unlock_irq (&ctx->completion_lock); + +io_cqring_add_event (req->ctx, sqe->user_data, ret,0 ); +io_free_req (req); +return 0 ; +} + +static void io_poll_complete (struct io_kiocb *req,__poll_t mask) +{ +io_cqring_add_event (req->ctx, req->user_data,mangle_poll (mask),0 ); +io_fput (req); +io_free_req (req); +} + +static void io_poll_complete_work (struct work_struct *work) +{ + struct io_kiocb *req = container_of(work, struct io_kiocb, work); +struct io_poll_iocb *poll = &req->poll; +struct poll_table_struct pt = { ._key = poll->events }; +struct io_ring_ctx *ctx = req->ctx; +__poll_t mask =0 ; + +if (!READ_ONCE (poll->canceled)) + mask =vfs_poll (poll->file, &pt) & poll->events; + +/* +* Note that ->ki_cancel callers also delete iocb from active_reqs after +* calling ->ki_cancel. We need the ctx_lock roundtrip here to +* synchronize with them. In the cancellation case the list_del_init +* itself is not actually needed, but harmless so we keep it in to +* avoid further branches in the fast path. +*/ +spin_lock_irq (&ctx->completion_lock); +if (!mask && !READ_ONCE (poll->canceled)) { +add_wait_queue (poll->head, &poll->wait); +spin_unlock_irq (&ctx->completion_lock); +return ; + } +list_del_init (&req->list); +spin_unlock_irq (&ctx->completion_lock); + +io_poll_complete (req, mask); +} + +static int io_poll_wake (struct wait_queue_entry *wait,unsigned mode,int sync, +void *key) +{ + struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb, + wait); + struct io_kiocb *req = container_of(poll, struct io_kiocb, poll); +struct io_ring_ctx *ctx = req->ctx; +__poll_t mask =key_to_poll (key); + + poll->woken = true; + +/* for instances that support it check for an event match first: */ +if (mask) { +if (!(mask & poll->events)) +return 0 ; + +/* try to complete the iocb inline if we can: */ +if (spin_trylock (&ctx->completion_lock)) { +list_del (&req->list); +spin_unlock (&ctx->completion_lock); + +list_del_init (&poll->wait.entry); +io_poll_complete (req, mask); +return 1 ; + } + } + +list_del_init (&poll->wait.entry); +queue_work (ctx->sqo_wq, &req->work); +return 1 ; +} + +struct io_poll_table { +struct poll_table_struct pt; +struct io_kiocb *req; +int error; +}; + +static void io_poll_queue_proc (struct file *file,struct wait_queue_head *head, +struct poll_table_struct *p) +{ + struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); + +if (unlikely (pt->req->poll.head)) { + pt->error = -EINVAL ; +return ; + } + + pt->error =0 ; + pt->req->poll.head = head; +add_wait_queue (head, &pt->req->poll.wait); +} + +static int io_poll_add (struct io_kiocb *req,const struct io_uring_sqe *sqe) +{ +struct io_poll_iocb *poll = &req->poll; +struct io_ring_ctx *ctx = req->ctx; +struct io_poll_table ipt; +__poll_t mask; + +if (unlikely (req->ctx->flags &IORING_SETUP_IOPOLL )) +return -EINVAL ; +if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index) +return -EINVAL ; + +INIT_WORK (&req->work, io_poll_complete_work); + poll->events =demangle_poll (sqe->poll_events) |EPOLLERR |EPOLLHUP ; + +if (sqe->flags &IOSQE_FIXED_FILE ) { +if (unlikely (!ctx->user_files || sqe->fd >= ctx->nr_user_files)) +return -EBADF ; + poll->file = ctx->user_files[sqe->fd]; + req->flags |=REQ_F_FIXED_FILE ; + }else { + poll->file =fget (sqe->fd); + } +if (unlikely (!poll->file)) +return -EBADF ; + + poll->head =NULL ; + poll->woken = false; + poll->canceled = false; + + ipt.pt._qproc = io_poll_queue_proc; + ipt.pt._key = poll->events; + ipt.req = req; + ipt.error = -EINVAL ;/* same as no support for IOCB_CMD_POLL */ + +/* initialized the list so that we can do list_empty checks */ +INIT_LIST_HEAD (&poll->wait.entry); +init_waitqueue_func_entry (&poll->wait, io_poll_wake); + +/* one for removal from waitqueue, one for this function */ +refcount_set (&req->refs,2 ); + + mask =vfs_poll (poll->file, &ipt.pt) & poll->events; +if (unlikely (!poll->head)) { +/* we did not manage to set up a waitqueue, done */ + goto out; + } + +spin_lock_irq (&ctx->completion_lock); +spin_lock (&poll->head->lock); +if (poll->woken) { +/* wake_up context handles the rest */ + mask =0 ; + ipt.error =0 ; + }else if (mask || ipt.error) { +/* if we get an error or a mask we are done */ +WARN_ON_ONCE (list_empty (&poll->wait.entry)); +list_del_init (&poll->wait.entry); + }else { +/* actually waiting for an event */ +list_add_tail (&req->list, &ctx->cancel_list); + } +spin_unlock (&poll->head->lock); +spin_unlock_irq (&ctx->completion_lock); + +out: +if (unlikely (ipt.error)) { +if (!(sqe->flags &IOSQE_FIXED_FILE )) +fput (poll->file); +return ipt.error; + } + +if (mask) +io_poll_complete (req, mask); +io_free_req (req); +return 0 ; +} +static int __io_submit_sqe (struct io_ring_ctx *ctx,struct io_kiocb *req,struct sqe_submit *s,bool force_nonblock,struct io_submit_state *state)
@@ -951,6 +1189,12 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,case IORING_OP_FSYNC : ret =io_fsync (req, sqe, force_nonblock);break ; +case IORING_OP_POLL_ADD : + ret =io_poll_add (req, sqe); +break ; +case IORING_OP_POLL_REMOVE : + ret =io_poll_remove (req, sqe); +break ;default : ret = -EINVAL ;break ;
@@ -1794,6 +2038,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)percpu_ref_kill (&ctx->refs);mutex_unlock (&ctx->uring_lock); +io_poll_remove_all (ctx);io_iopoll_reap_events (ctx);wait_for_completion (&ctx->ctx_done);io_ring_ctx_free (ctx);
include/uapi/linux/io_uring.h +3 -0
@@ -27,6 +27,7 @@ struct io_uring_sqe {union {__kernel_rwf_t rw_flags;__u32 fsync_flags; +__u16 poll_events; };__u64 user_data;/* data to be passed back at completion time */ union {
@@ -53,6 +54,8 @@ struct io_uring_sqe {#define IORING_OP_FSYNC 3#define IORING_OP_READ_FIXED 4#define IORING_OP_WRITE_FIXED 5 +#define IORING_OP_POLL_ADD 6 +#define IORING_OP_POLL_REMOVE 7/* * sqe->fsync_flags
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 17/18 io_uring: allow workqueue item to handle multiple buffered requests
In reply to Jens AxboeRight now we punt any buffered request that ends up triggering an -EAGAIN to an async workqueue. This works fine in terms of providing async execution of them, but it also can create quite a lot of work queue items. For sequentially buffered IO, it's advantageous to serialize the issue of them. For reads, the first one will trigger a read-ahead, and subsequent request merely end up waiting on later pages to complete. For writes, devices usually respond better to streamed sequential writes.
Add state to track the last buffered request we punted to a work queue, and if the next one is sequential to the previous, attempt to get the previous work item to handle it. We limit the number of sequential add-ons to the a multiple (8) of the max read-ahead size of the file. This should be a good number for both reads and wries, as it defines the max IO size the device can do directly.
This drastically cuts down on the number of context switches we need to handle buffered sequential IO, and a basic test case of copying a big file with io_uring sees a 5x speedup.
--- fs/io_uring.c | 231 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 194 insertions(+), 37 deletions(-)
fs/io_uring.c +194 -37
@@ -68,6 +68,16 @@ struct io_mapped_ubuf {unsigned int nr_bvecs; }; +struct async_list { +spinlock_t lock; +atomic_t cnt; +struct list_head list; + +struct file *file; +off_t io_end; +size_t io_pages; +}; + struct io_ring_ctx { struct { struct percpu_ref refs;
@@ -126,6 +136,8 @@ struct io_ring_ctx {struct list_head poll_list;struct list_head cancel_list; }____cacheline_aligned_in_smp ; + +struct async_list pending_async[2 ]; };struct sqe_submit {
@@ -157,6 +169,7 @@ struct io_kiocb {#define REQ_F_IOPOLL_COMPLETED 2/* polled IO has completed */ #define REQ_F_IOPOLL_EAGAIN 4/* submission got EAGAIN */ #define REQ_F_FIXED_FILE 8/* ctx owns file */ +#define REQ_F_SEQ_PREV 16/* sequential with previous */ u64 user_data;u64 res;
@@ -200,6 +213,7 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref)static struct io_ring_ctx *io_ring_ctx_alloc (struct io_uring_params *p) {struct io_ring_ctx *ctx; +int i; ctx =kzalloc (sizeof (*ctx),GFP_KERNEL );if (!ctx)
@@ -215,6 +229,11 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)init_completion (&ctx->ctx_done);mutex_init (&ctx->uring_lock);init_waitqueue_head (&ctx->wait); +for (i =0 ; i <ARRAY_SIZE (ctx->pending_async); i++) { +spin_lock_init (&ctx->pending_async[i].lock); +INIT_LIST_HEAD (&ctx->pending_async[i].list); +atomic_set (&ctx->pending_async[i].cnt,0 ); + }spin_lock_init (&ctx->completion_lock);INIT_LIST_HEAD (&ctx->poll_list);INIT_LIST_HEAD (&ctx->cancel_list);
@@ -774,6 +793,39 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,return import_iovec (rw, buf, sqe->len,UIO_FASTIOV , iovec, iter); } +static void io_async_list_note (int rw,struct io_kiocb *req,size_t len) +{ +struct async_list *async_list = &req->ctx->pending_async[rw]; +struct kiocb *kiocb = &req->rw; +struct file *filp = kiocb->ki_filp; +off_t io_end = kiocb->ki_pos + len; + +if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) { +unsigned long max_pages; + +/* Use 8x RA size as a decent limiter for both reads/writes */ + max_pages = filp->f_ra.ra_pages; +if (!max_pages) + max_pages =VM_MAX_READAHEAD >> (PAGE_SHIFT -10 ); + max_pages *=8 ; + + len >>=PAGE_SHIFT ; +if (async_list->io_pages + len <= max_pages) { + req->flags |=REQ_F_SEQ_PREV ; + async_list->io_pages += len; + }else { + io_end =0 ; + async_list->io_pages =0 ; + } + } + +if (async_list->file != filp) { + async_list->io_pages =0 ; + async_list->file = filp; + } + async_list->io_end = io_end; +} +static ssize_t io_read (struct io_kiocb *req,const struct io_uring_sqe *sqe,bool force_nonblock,struct io_submit_state *state) {
@@ -781,6 +833,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;struct iov_iter iter;struct file *file; +size_t iov_count;ssize_t ret; ret =io_prep_rw (req, sqe, force_nonblock, state);
@@ -799,16 +852,19 @@ static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (ret) goto out_fput; - ret =rw_verify_area (READ , file, &kiocb->ki_pos,iov_iter_count (&iter)); + iov_count =iov_iter_count (&iter); + ret =rw_verify_area (READ , file, &kiocb->ki_pos, iov_count);if (!ret) {ssize_t ret2;/* Catch -EAGAIN return for forced non-blocking submission */ ret2 =call_read_iter (file, kiocb, &iter); -if (!force_nonblock || ret2 != -EAGAIN ) +if (!force_nonblock || ret2 != -EAGAIN ) {io_rw_done (kiocb, ret2); -else + }else { +io_async_list_note (READ , req, iov_count); ret = -EAGAIN ; + } }kfree (iovec); out_fput:
@@ -824,6 +880,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,struct kiocb *kiocb = &req->rw;struct iov_iter iter;struct file *file; +size_t iov_count;ssize_t ret; ret =io_prep_rw (req, sqe, force_nonblock, state);
@@ -831,10 +888,6 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,return ret; file = kiocb->ki_filp; - ret = -EAGAIN ; -if (force_nonblock && !(kiocb->ki_flags &IOCB_DIRECT )) - goto out_fput; - ret = -EBADF ;if (unlikely (!(file->f_mode &FMODE_WRITE ))) goto out_fput;
@@ -846,8 +899,15 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe,if (ret) goto out_fput; - ret =rw_verify_area (WRITE , file, &kiocb->ki_pos, -iov_iter_count (&iter)); + iov_count =iov_iter_count (&iter); + + ret = -EAGAIN ; +if (force_nonblock && !(kiocb->ki_flags &IOCB_DIRECT )) { +io_async_list_note (WRITE , req, iov_count); + goto out_free; + } + + ret =rw_verify_area (WRITE , file, &kiocb->ki_pos, iov_count);if (!ret) {/* * Open-code file_start_write here to grab freeze protection,
@@ -865,6 +925,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_flags |=IOCB_WRITE ;io_rw_done (kiocb,call_write_iter (file, kiocb, &iter)); } +out_free:kfree (iovec); out_fput:if (unlikely (ret))
@@ -1212,6 +1273,21 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,return 0 ; } +static struct async_list *io_async_list_from_sqe (struct io_ring_ctx *ctx, +const struct io_uring_sqe *sqe) +{ +switch (sqe->opcode) { +case IORING_OP_READV : +case IORING_OP_READ_FIXED : +return &ctx->pending_async[READ ]; +case IORING_OP_WRITEV : +case IORING_OP_WRITE_FIXED : +return &ctx->pending_async[WRITE ]; +default : +return NULL ; + } +} +static inline bool io_sqe_needs_user (const struct io_uring_sqe *sqe) { return !(sqe->opcode == IORING_OP_READ_FIXED ||
@@ -1221,50 +1297,124 @@ static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)static void io_sq_wq_submit_work (struct work_struct *work) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); -struct sqe_submit *s = &req->submit; -u64 user_data = s->sqe->user_data;struct io_ring_ctx *ctx = req->ctx; +struct mm_struct *cur_mm =NULL ;struct files_struct *old_files; +struct async_list *async_list; +LIST_HEAD (req_list);mm_segment_t old_fs; -bool needs_user;int ret; -/* Ensure we clear previously set forced non-block flag */ - req->flags &= ~REQ_F_FORCE_NONBLOCK ; - old_files = current->files; current->files = ctx->sqo_files; + async_list =io_async_list_from_sqe (ctx, req->submit.sqe); +restart: +do { +struct sqe_submit *s = &req->submit; +u64 user_data = s->sqe->user_data; + +/* Ensure we clear previously set forced non-block flag */ + req->flags &= ~REQ_F_FORCE_NONBLOCK ; + + ret =0 ; +if (io_sqe_needs_user (s->sqe) && !cur_mm) { +if (!mmget_not_zero (ctx->sqo_mm)) { + ret = -EFAULT ; + }else { + cur_mm = ctx->sqo_mm;; +use_mm (ctx->sqo_mm); + old_fs =get_fs (); +set_fs (USER_DS ); + } + } + +if (!ret) + ret =__io_submit_sqe (ctx, req, s, false,NULL ); +if (ret) { +io_cqring_add_event (ctx, user_data, ret,0 ); +io_free_req (req); + } +if (!async_list) +break ; +if (!list_empty (&req_list)) { + req = list_first_entry(&req_list, struct io_kiocb, + list); +list_del (&req->list); +continue ; + } +if (list_empty (&async_list->list)) +break ; + + req =NULL ; +spin_lock (&async_list->lock); +if (list_empty (&async_list->list)) { +spin_unlock (&async_list->lock); +break ; + } +list_splice_init (&async_list->list, &req_list); +spin_unlock (&async_list->lock); + + req = list_first_entry(&req_list, struct io_kiocb, list); +list_del (&req->list); + }while (req); +/* -* If we're doing IO to fixed buffers, we don't need to get/set -* user context +* Rare case of racing with a submitter. If we find the count has +* dropped to zero AND we have pending work items, then restart +* the processing. This is a tiny race window. */ - needs_user =io_sqe_needs_user (s->sqe); -if (needs_user) { -if (!mmget_not_zero (ctx->sqo_mm)) { - ret = -EFAULT ; - goto err; + ret =atomic_dec_return (&async_list->cnt); +while (!ret && !list_empty (&async_list->list)) { +spin_lock (&async_list->lock); +atomic_inc (&async_list->cnt); +list_splice_init (&async_list->list, &req_list); +spin_unlock (&async_list->lock); + +if (!list_empty (&req_list)) { + req = list_first_entry(&req_list, struct io_kiocb, + list); +list_del (&req->list); + goto restart; } -use_mm (ctx->sqo_mm); - old_fs =get_fs (); -set_fs (USER_DS ); + ret =atomic_dec_return (&async_list->cnt); } - ret =__io_submit_sqe (ctx, req, s, false,NULL ); - -if (needs_user) { +if (cur_mm) {set_fs (old_fs); -unuse_mm (ctx->sqo_mm); -mmput (ctx->sqo_mm); - } -err: -if (ret) { -io_cqring_add_event (ctx, user_data, ret,0 ); -io_free_req (req); +unuse_mm (cur_mm); +mmput (cur_mm); } current->files = old_files; } +/* +* See if we can piggy back onto previously submitted work, that is still +* running. We currently only allow this if the new request is sequential +* to the previous one we punted. +*/ +static bool io_add_to_prev_work (struct async_list *list,struct io_kiocb *req) +{ +bool ret = false; + +if (!list) +return false; +if (!(req->flags &REQ_F_SEQ_PREV )) +return false; +if (!atomic_read (&list->cnt)) +return false; + + ret = true; +spin_lock (&list->lock); +list_add_tail (&req->list, &list->list); +if (!atomic_read (&list->cnt)) { +list_del_init (&req->list); + ret = false; + } +spin_unlock (&list->lock); +return ret; +} +static int io_submit_sqe (struct io_ring_ctx *ctx,struct sqe_submit *s,struct io_submit_state *state) {
@@ -1281,9 +1431,16 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, ret =__io_submit_sqe (ctx, req, s, true, state);if (ret == -EAGAIN ) { +struct async_list *list; + + list =io_async_list_from_sqe (ctx, s->sqe);memcpy (&req->submit, s,sizeof (*s)); -INIT_WORK (&req->work, io_sq_wq_submit_work); -queue_work (ctx->sqo_wq, &req->work); +if (!io_add_to_prev_work (list, req)) { +if (list) +atomic_inc (&list->cnt); +INIT_WORK (&req->work, io_sq_wq_submit_work); +queue_work (ctx->sqo_wq, &req->work); + } ret =0 ; }if (ret)
Signature · 1 line
2.17.1
Jens Axboe
maintainerPATCH 18/18 io_uring: add io_uring_event cache hit information
In reply to Jens AxboeAdd hint on whether a read was served out of the page cache, or if it hit media. This is useful for buffered async IO, O_DIRECT reads would never have this set (for obvious reasons).
If the read hit page cache, cqe->flags will have IOCQE_FLAG_CACHEHIT set.
--- fs/io_uring.c | 7 ++++++- include/uapi/linux/io_uring.h | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-)
fs/io_uring.c +6 -1
@@ -561,11 +561,16 @@ static void io_fput(struct io_kiocb *req)static void io_complete_rw (struct kiocb *kiocb,long res,long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw); +unsigned ev_flags =0 ;kiocb_end_write (kiocb);io_fput (req); -io_cqring_add_event (req->ctx, req->user_data, res,0 ); + +if (res >0 && (req->flags &REQ_F_FORCE_NONBLOCK )) + ev_flags =IOCQE_FLAG_CACHEHIT ; + +io_cqring_add_event (req->ctx, req->user_data, res, ev_flags);io_free_req (req); }
include/uapi/linux/io_uring.h +5 -0
@@ -71,6 +71,11 @@ struct io_uring_cqe {__u32 flags; }; +/* +* io_uring_event->flags +*/ +#define IOCQE_FLAG_CACHEHIT (1 << 0)/* IO did not hit media */ +/* * Magic offsets for the application to mmap the data it needs */
Signature · 1 line
2.17.1