Cache async read/write and regular file state
Jens Axboe
maintainercover Cache async read/write and regular file state
Hi,
Patch 1 is just a regular fix/cleanup, there's no need to check for whether or not a file has io_uring fops if it's registered, as we explicitly disallow that at registration time.
Patch 2 uses the bottom 3 (or 2 for 32-bit) bits in the file pointer to cache state that we always use at read/write time. This prevents digging deep into the file, inode, superblock in the fast path.
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH 1/2 io_uring: don't check for io_uring_fops for fixed files
We don't allow them at registration time, so limit the check for needing inflight tracking in io_file_get() to the non-fixed path.
--- fs/io_uring.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
fs/io_uring.c +4 -2
@@ -6113,10 +6113,12 @@ static struct file *io_file_get(struct io_submit_state *state, }else {trace_io_uring_file_get (ctx, fd); file =__io_file_get (state, fd); + +/* we don't allow fixed io_uring files */ +if (file &&unlikely (file->f_op == &io_uring_fops)) +io_req_track_inflight (req); } -if (file &&unlikely (file->f_op == &io_uring_fops)) -io_req_track_inflight (req);return file; }
Signature · 1 line
2.30.2
Jens Axboe
maintainerPATCH 2/2 io_uring: cache async and regular file state for fixed files
In reply to Jens AxboeWe have to dig quite deep to check for particularly whether or not a file supports a fast-path nonblock attempt. For fixed files, we can do this lookup once and cache the state instead.
This adds two new bits to track whether we support async read/write attempt, and lines up the REQ_F_ISREG bit with those two. The file slot re-uses the last 3 (or 2, for 32-bit) of the file pointer to cache that state, and then we mask it in when we go and use a fixed file.
--- fs/io_uring.c | 76 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 12 deletions(-)
fs/io_uring.c +64 -12
@@ -673,13 +673,16 @@ enum {REQ_F_CUR_POS_BIT ,REQ_F_NOWAIT_BIT ,REQ_F_LINK_TIMEOUT_BIT , -REQ_F_ISREG_BIT ,REQ_F_NEED_CLEANUP_BIT ,REQ_F_POLLED_BIT ,REQ_F_BUFFER_SELECTED_BIT ,REQ_F_NO_FILE_TABLE_BIT ,REQ_F_LTIMEOUT_ACTIVE_BIT ,REQ_F_COMPLETE_INLINE_BIT , +/* keep async read/write and isreg together and in order */ +REQ_F_ASYNC_READ_BIT , +REQ_F_ASYNC_WRITE_BIT , +REQ_F_ISREG_BIT ,/* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT,
@@ -709,8 +712,6 @@ enum {REQ_F_NOWAIT =BIT (REQ_F_NOWAIT_BIT ),/* has or had linked timeout */ REQ_F_LINK_TIMEOUT =BIT (REQ_F_LINK_TIMEOUT_BIT ), -/* regular file */ -REQ_F_ISREG =BIT (REQ_F_ISREG_BIT ),/* needs cleanup */ REQ_F_NEED_CLEANUP = BIT(REQ_F_NEED_CLEANUP_BIT),/* already went through poll handler */
@@ -723,6 +724,12 @@ enum {REQ_F_LTIMEOUT_ACTIVE =BIT (REQ_F_LTIMEOUT_ACTIVE_BIT ),/* completion is deferred through io_comp_state */ REQ_F_COMPLETE_INLINE =BIT (REQ_F_COMPLETE_INLINE_BIT ), +/* supports async reads */ +REQ_F_ASYNC_READ =BIT (REQ_F_ASYNC_READ_BIT ), +/* supports async writes */ +REQ_F_ASYNC_WRITE =BIT (REQ_F_ASYNC_WRITE_BIT ), +/* regular file */ + REQ_F_ISREG = BIT(REQ_F_ISREG_BIT), }; struct async_poll {
@@ -2617,7 +2624,7 @@ static bool io_bdev_nowait(struct block_device *bdev)* any file. For now, just ensure that anything potentially problematic is done * inline. */ -static bool io_file_supports_async (struct file *file,int rw) +static bool __io_file_supports_async (struct file *file,int rw) {umode_t mode =file_inode (file)->i_mode;
@@ -2650,6 +2657,16 @@ static bool io_file_supports_async(struct file *file, int rw)return file->f_op->write_iter !=NULL ; } +static bool io_file_supports_async (struct io_kiocb *req,int rw) +{ +if (rw ==READ && (req->flags &REQ_F_ASYNC_READ )) +return true; +else if (rw ==WRITE && (req->flags &REQ_F_ASYNC_WRITE )) +return true; + +return __io_file_supports_async (req->file, rw); +} +static int io_prep_rw (struct io_kiocb *req,const struct io_uring_sqe *sqe) {struct io_ring_ctx *ctx = req->ctx;
@@ -2658,7 +2675,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)unsigned ioprio;int ret; -if (S_ISREG (file_inode (file)->i_mode)) +if (!(req->flags &REQ_F_ISREG ) &&S_ISREG (file_inode (file)->i_mode)) req->flags |=REQ_F_ISREG ; kiocb->ki_pos =READ_ONCE (sqe->off);
@@ -3242,7 +3259,7 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags) kiocb->ki_flags |=IOCB_NOWAIT ;/* If the file doesn't support async, just async punt */ -if (force_nonblock && !io_file_supports_async (req->file,READ )) { +if (force_nonblock && !io_file_supports_async (req,READ )) { ret =io_setup_async_rw (req, iovec, inline_vecs, iter, true);return ret ?: -EAGAIN ; }
@@ -3348,7 +3365,7 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags) kiocb->ki_flags |=IOCB_NOWAIT ;/* If the file doesn't support async, just async punt */ -if (force_nonblock && !io_file_supports_async (req->file,WRITE )) +if (force_nonblock && !io_file_supports_async (req,WRITE )) goto copy_iov;/* file path doesn't support NOWAIT for non-direct_IO */
@@ -5114,7 +5131,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req)else return false;/* if we can't nonblock try, then no point in arming a poll handler */ -if (!io_file_supports_async (req->file, rw)) +if (!io_file_supports_async (req, rw))return false; apoll =kmalloc (sizeof (*apoll),GFP_ATOMIC );
@@ -6083,8 +6100,17 @@ static void io_wq_submit_work(struct io_wq_work *work) } } -static inline struct file **io_fixed_file_slot (struct fixed_rsrc_data *file_data, -unsigned i) +#define FFS_ASYNC_READ 0x1UL +#define FFS_ASYNC_WRITE 0x2UL +#ifdef CONFIG_64BIT +#define FFS_ISREG 0x4UL +#else +#define FFS_ISREG 0x0UL +#endif +#define FFS_MASK ~(FFS_ASYNC_READ|FFS_ASYNC_WRITE|FFS_ISREG) + +static inline void *__io_fixed_file_slot (struct fixed_rsrc_data *file_data, +unsigned i) {struct fixed_rsrc_table *table;
@@ -6092,6 +6118,17 @@ static inline struct file **io_fixed_file_slot(struct fixed_rsrc_data *file_datareturn &table->files[i &IORING_FILE_TABLE_MASK ]; } +static inline struct file **io_fixed_file_slot (struct fixed_rsrc_data *file_data, +unsigned i) +{ +struct file **file; + + file =__io_fixed_file_slot (file_data, i); +if (*file) + *file = (struct file *) ((unsigned long ) *file &FFS_MASK ); +return file; +} +static inline struct file *io_file_from_index (struct io_ring_ctx *ctx,int index) {
@@ -6105,10 +6142,16 @@ static struct file *io_file_get(struct io_submit_state *state,struct file *file;if (fixed) { +unsigned long file_ptr; +if (unlikely ((unsigned int )fd >= ctx->nr_user_files))return NULL ; fd =array_index_nospec (fd, ctx->nr_user_files); - file =io_file_from_index (ctx, fd); + file_ptr = (unsigned long ) *io_fixed_file_slot (ctx->file_data, fd); + file = (struct file *) (file_ptr &FFS_MASK ); + file_ptr &= ~FFS_MASK ; +/* mask in overlapping REQ_F and FFS bits */ + req->flags |= (file_ptr <<REQ_F_ASYNC_READ_BIT );io_set_resource_node (req); }else {trace_io_uring_file_get (ctx, fd);
@@ -7432,6 +7475,8 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, goto out_free;for (i =0 ; i < nr_args; i++, ctx->nr_user_files++) { +unsigned long file_ptr; +if (copy_from_user (&fd, &fds[i],sizeof (fd))) { ret = -EFAULT ; goto out_fput;
@@ -7456,7 +7501,14 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,fput (file); goto out_fput; } - *io_fixed_file_slot (file_data, i) = file; + file_ptr = (unsigned long ) file; +if (__io_file_supports_async (file,READ )) + file_ptr |=FFS_ASYNC_READ ; +if (__io_file_supports_async (file,WRITE )) + file_ptr |=FFS_ASYNC_WRITE ; +if (S_ISREG (file_inode (file)->i_mode)) + file_ptr |=FFS_ISREG ; + *io_fixed_file_slot (file_data, i) = (struct file *) file_ptr; } ret =io_sqe_files_scm (ctx);
Signature · 1 line
2.30.2