aio cleanups and optimizations for 4.21
PATCH 1/1 aio cleanups and optimizations for 4.21
Please consider this series for 4.21. It's been part of my aio series that adds buffered and polled aio support for a long time, but these are standalone fixes and optimizations that we should apply for 4.21. Last one is just a prep patch for the series.
Signature · 1 line
Jens Axboe
PATCH 1/7 aio: use assigned completion handler
We know this is a read/write request, but in preparation for having different kinds of those, ensure that we call the assigned handler instead of assuming it's aio_complete_rq().
Reviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/aio.c +1 -1
@@ -1490,7 +1490,7 @@ static inline void aio_rw_done(struct kiocb *req, ssize_t ret) ret = -EINTR ;/*FALLTHRU*/ default : -aio_complete_rw (req, ret,0 ); + req->ki_complete (req, ret,0 ); } }
Signature · 1 line
2.17.1
PATCH 2/7 aio: separate out ring reservation from req allocation
In reply to Jens AxboeFrom: Christoph Hellwig <[email protected]>
This is in preparation for certain types of IO not needing a ring reserveration.
Signed-off-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-)
fs/aio.c +17 -13
@@ -901,7 +901,7 @@ static void put_reqs_available(struct kioctx *ctx, unsigned nr)local_irq_restore (flags); } -static bool get_reqs_available (struct kioctx *ctx) +static bool __get_reqs_available (struct kioctx *ctx) {struct kioctx_cpu *kcpu;bool ret = false;
@@ -993,6 +993,14 @@ static void user_refill_reqs_available(struct kioctx *ctx)spin_unlock_irq (&ctx->completion_lock); } +static bool get_reqs_available (struct kioctx *ctx) +{ +if (__get_reqs_available (ctx)) +return true; +user_refill_reqs_available (ctx); +return __get_reqs_available (ctx); +} +/* aio_get_req * Allocate a slot for an aio request. * Returns NULL if no requests are free.
@@ -1001,24 +1009,15 @@ static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) {struct aio_kiocb *req; -if (!get_reqs_available (ctx)) { -user_refill_reqs_available (ctx); -if (!get_reqs_available (ctx)) -return NULL ; - } - req =kmem_cache_alloc (kiocb_cachep,GFP_KERNEL |__GFP_ZERO);if (unlikely (!req)) - goto out_put; +return NULL ;percpu_ref_get (&ctx->reqs);INIT_LIST_HEAD (&req->ki_list);refcount_set (&req->ki_refcnt,0 ); req->ki_ctx = ctx;return req; -out_put: -put_reqs_available (ctx,1 ); -return NULL ; }static struct kioctx *lookup_ioctx (unsigned long ctx_id)
@@ -1805,9 +1804,13 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,return -EINVAL ; } +if (!get_reqs_available (ctx)) +return -EAGAIN ; + + ret = -EAGAIN ; req =aio_get_req (ctx);if (unlikely (!req)) -return -EAGAIN ; + goto out_put_reqs_available;if (iocb.aio_flags &IOCB_FLAG_RESFD ) {/*
@@ -1870,11 +1873,12 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, goto out_put_req;return 0 ; out_put_req: -put_reqs_available (ctx,1 );percpu_ref_put (&ctx->reqs);if (req->ki_eventfd)eventfd_ctx_put (req->ki_eventfd);kmem_cache_free (kiocb_cachep, req); +out_put_reqs_available: +put_reqs_available (ctx,1 );return ret; }
Signature · 1 line
2.17.1
PATCH 3/7 aio: don't zero entire aio_kiocb aio_get_req()
In reply to Jens AxboeIt's 192 bytes, fairly substantial. Most items don't need to be cleared, especially not upfront. Clear the ones we do need to clear, and leave the other ones for setup when the iocb is prepared and submitted.
Reviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)
fs/aio.c +7 -2
@@ -1009,14 +1009,15 @@ static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) {struct aio_kiocb *req; - req =kmem_cache_alloc (kiocb_cachep,GFP_KERNEL |__GFP_ZERO); + req =kmem_cache_alloc (kiocb_cachep,GFP_KERNEL );if (unlikely (!req))return NULL ;percpu_ref_get (&ctx->reqs); + req->ki_ctx = ctx;INIT_LIST_HEAD (&req->ki_list);refcount_set (&req->ki_refcnt,0 ); - req->ki_ctx = ctx; + req->ki_eventfd =NULL ;return req; }
@@ -1730,6 +1731,10 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb)if (unlikely (!req->file))return -EBADF ; + req->head =NULL ; + req->woken = false; + req->cancelled = false; + apt.pt._qproc = aio_poll_queue_proc; apt.pt._key = req->events; apt.iocb = aiocb;
Signature · 1 line
2.17.1
PATCH 4/7 aio: only use blk plugs for > 2 depth submissions
In reply to Jens AxboePlugging is meant to optimize submission of a string of IOs, if we don't have more than 2 being submitted, don't bother setting up a plug.
Reviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-)
fs/aio.c +14 -4
@@ -69,6 +69,12 @@ struct aio_ring {struct io_event io_events[0 ]; };/* 128 bytes + ring size */ +/* +* Plugging is meant to work with larger batches of IOs. If we don't +* have more than the below, then don't bother setting up a plug. +*/ +#define AIO_PLUG_THRESHOLD 2 +#define AIO_RING_PAGES 8struct kioctx_table {
@@ -1919,7 +1925,8 @@ SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,if (nr > ctx->nr_events) nr = ctx->nr_events; -blk_start_plug (&plug); +if (nr >AIO_PLUG_THRESHOLD ) +blk_start_plug (&plug);for (i =0 ; i < nr; i++) {struct iocb __user *user_iocb;
@@ -1932,7 +1939,8 @@ SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,if (ret)break ; } -blk_finish_plug (&plug); +if (nr >AIO_PLUG_THRESHOLD ) +blk_finish_plug (&plug);percpu_ref_put (&ctx->users);return i ? i : ret;
@@ -1959,7 +1967,8 @@ COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,if (nr > ctx->nr_events) nr = ctx->nr_events; -blk_start_plug (&plug); +if (nr >AIO_PLUG_THRESHOLD ) +blk_start_plug (&plug);for (i =0 ; i < nr; i++) {compat_uptr_t user_iocb;
@@ -1972,7 +1981,8 @@ COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id,if (ret)break ; } -blk_finish_plug (&plug); +if (nr >AIO_PLUG_THRESHOLD ) +blk_finish_plug (&plug);percpu_ref_put (&ctx->users);return i ? i : ret;
Signature · 1 line
2.17.1
PATCH 5/7 aio: use iocb_put() instead of open coding it
In reply to Jens AxboeReplace the percpu_ref_put() + kmem_cache_free() with a call to iocb_put() instead.
Reviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
fs/aio.c +1 -2
@@ -1884,10 +1884,9 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, goto out_put_req;return 0 ; out_put_req: -percpu_ref_put (&ctx->reqs);if (req->ki_eventfd)eventfd_ctx_put (req->ki_eventfd); -kmem_cache_free (kiocb_cachep, req); +iocb_put (req); out_put_reqs_available:put_reqs_available (ctx,1 );return ret;
Signature · 1 line
2.17.1
PATCH 6/7 aio: split out iocb copy from io_submit_one()
In reply to Jens AxboeIn preparation of handing in iocbs in a different fashion as well. Also make it clear that the iocb being passed in isn't modified, by marking it const throughout.
Reviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 68 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 30 deletions(-)
fs/aio.c +38 -30
@@ -1420,7 +1420,7 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2)aio_complete (iocb, res, res2); } -static int aio_prep_rw (struct kiocb *req,struct iocb *iocb) +static int aio_prep_rw (struct kiocb *req,const struct iocb *iocb) {int ret;
@@ -1461,7 +1461,7 @@ static int aio_prep_rw(struct kiocb *req, struct iocb *iocb)return ret; } -static int aio_setup_rw (int rw,struct iocb *iocb,struct iovec **iovec, +static int aio_setup_rw (int rw,const struct iocb *iocb,struct iovec **iovec,bool vectored,bool compat,struct iov_iter *iter) {void __user *buf = (void __user *)(uintptr_t )iocb->aio_buf;
@@ -1500,8 +1500,8 @@ static inline void aio_rw_done(struct kiocb *req, ssize_t ret) } } -static ssize_t aio_read (struct kiocb *req,struct iocb *iocb,bool vectored, -bool compat) +static ssize_t aio_read (struct kiocb *req,const struct iocb *iocb, +bool vectored,bool compat) {struct iovec inline_vecs[UIO_FASTIOV ], *iovec = inline_vecs;struct iov_iter iter;
@@ -1533,8 +1533,8 @@ static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored,return ret; } -static ssize_t aio_write (struct kiocb *req,struct iocb *iocb,bool vectored, -bool compat) +static ssize_t aio_write (struct kiocb *req,const struct iocb *iocb, +bool vectored,bool compat) {struct iovec inline_vecs[UIO_FASTIOV ], *iovec = inline_vecs;struct iov_iter iter;
@@ -1589,7 +1589,8 @@ static void aio_fsync_work(struct work_struct *work) aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); } -static int aio_fsync (struct fsync_iocb *req,struct iocb *iocb,bool datasync) +static int aio_fsync (struct fsync_iocb *req,const struct iocb *iocb, +bool datasync) {if (unlikely (iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags))
@@ -1717,7 +1718,7 @@ aio_poll_queue_proc(struct file *file, struct wait_queue_head *head,add_wait_queue (head, &pt->iocb->poll.wait); } -static ssize_t aio_poll (struct aio_kiocb *aiocb,struct iocb *iocb) +static ssize_t aio_poll (struct aio_kiocb *aiocb,const struct iocb *iocb) {struct kioctx *ctx = aiocb->ki_ctx;struct poll_iocb *req = &aiocb->poll;
@@ -1789,27 +1790,23 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb)return 0 ; } -static int io_submit_one (struct kioctx *ctx,struct iocb __user *user_iocb, -bool compat) +static int __io_submit_one (struct kioctx *ctx,const struct iocb *iocb, +struct iocb __user *user_iocb,bool compat) {struct aio_kiocb *req; -struct iocb iocb;ssize_t ret; -if (unlikely (copy_from_user (&iocb, user_iocb,sizeof (iocb)))) -return -EFAULT ; -/* enforce forwards compatibility on users */ -if (unlikely (iocb.aio_reserved2)) { +if (unlikely (iocb->aio_reserved2)) {pr_debug ("EINVAL: reserve field set\n" );return -EINVAL ; }/* prevent overflows */ if (unlikely ( - (iocb.aio_buf != (unsigned long )iocb.aio_buf) || - (iocb.aio_nbytes != (size_t )iocb.aio_nbytes) || - ((ssize_t )iocb.aio_nbytes <0 ) + (iocb->aio_buf != (unsigned long )iocb->aio_buf) || + (iocb->aio_nbytes != (size_t )iocb->aio_nbytes) || + ((ssize_t )iocb->aio_nbytes <0 ) )) {pr_debug ("EINVAL: overflow check\n" );return -EINVAL ;
@@ -1823,14 +1820,14 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,if (unlikely (!req)) goto out_put_reqs_available; -if (iocb.aio_flags &IOCB_FLAG_RESFD ) { +if (iocb->aio_flags &IOCB_FLAG_RESFD ) {/* * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an * instance of the file* now. The file descriptor must be * an eventfd() fd, and will be signaled for each completed * event using the eventfd_signal() function. */ - req->ki_eventfd =eventfd_ctx_fdget ((int ) iocb.aio_resfd); + req->ki_eventfd =eventfd_ctx_fdget ((int ) iocb->aio_resfd);if (IS_ERR (req->ki_eventfd)) { ret =PTR_ERR (req->ki_eventfd); req->ki_eventfd =NULL ;
@@ -1845,32 +1842,32 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, } req->ki_user_iocb = user_iocb; - req->ki_user_data = iocb.aio_data; + req->ki_user_data = iocb->aio_data; -switch (iocb.aio_lio_opcode) { +switch (iocb->aio_lio_opcode) {case IOCB_CMD_PREAD : - ret =aio_read (&req->rw, &iocb, false, compat); + ret =aio_read (&req->rw, iocb, false, compat);break ;case IOCB_CMD_PWRITE : - ret =aio_write (&req->rw, &iocb, false, compat); + ret =aio_write (&req->rw, iocb, false, compat);break ;case IOCB_CMD_PREADV : - ret =aio_read (&req->rw, &iocb, true, compat); + ret =aio_read (&req->rw, iocb, true, compat);break ;case IOCB_CMD_PWRITEV : - ret =aio_write (&req->rw, &iocb, true, compat); + ret =aio_write (&req->rw, iocb, true, compat);break ;case IOCB_CMD_FSYNC : - ret =aio_fsync (&req->fsync, &iocb, false); + ret =aio_fsync (&req->fsync, iocb, false);break ;case IOCB_CMD_FDSYNC : - ret =aio_fsync (&req->fsync, &iocb, true); + ret =aio_fsync (&req->fsync, iocb, true);break ;case IOCB_CMD_POLL : - ret =aio_poll (req, &iocb); + ret =aio_poll (req, iocb);break ;default : -pr_debug ("invalid aio operation %d\n" , iocb.aio_lio_opcode); +pr_debug ("invalid aio operation %d\n" , iocb->aio_lio_opcode); ret = -EINVAL ;break ; }
@@ -1892,6 +1889,17 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,return ret; } +static int io_submit_one (struct kioctx *ctx,struct iocb __user *user_iocb, +bool compat) +{ +struct iocb iocb; + +if (unlikely (copy_from_user (&iocb, user_iocb,sizeof (iocb)))) +return -EFAULT ; + +return __io_submit_one (ctx, &iocb, user_iocb, compat); +} +/* sys_io_submit: * Queue the nr iocbs pointed to by iocbpp for processing. Returns * the number of iocbs queued. May return -EINVAL if the aio_context
Signature · 1 line
2.17.1
PATCH 7/7 aio: abstract out io_event filler helper
In reply to Jens AxboeReviewed-by Christoph Hellwig <[email protected]>
Signed-off-by Jens Axboe <[email protected]>
--- fs/aio.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-)
fs/aio.c +10 -4
@@ -1063,6 +1063,15 @@ static inline void iocb_put(struct aio_kiocb *iocb) } } +static void aio_fill_event (struct io_event *ev,struct aio_kiocb *iocb, +long res,long res2) +{ + ev->obj = (u64 )(unsigned long )iocb->ki_user_iocb; + ev->data = iocb->ki_user_data; + ev->res = res; + ev->res2 = res2; +} +/* aio_complete * Called when the io request on the given iocb is complete. */
@@ -1090,10 +1099,7 @@ static void aio_complete(struct aio_kiocb *iocb, long res, long res2) ev_page =kmap_atomic (ctx->ring_pages[pos /AIO_EVENTS_PER_PAGE ]); event = ev_page + pos %AIO_EVENTS_PER_PAGE ; - event->obj = (u64 )(unsigned long )iocb->ki_user_iocb; - event->data = iocb->ki_user_data; - event->res = res; - event->res2 = res2; +aio_fill_event (event, iocb, res, res2);kunmap_atomic (ev_page);flush_dcache_page (ctx->ring_pages[pos /AIO_EVENTS_PER_PAGE ]);
Signature · 1 line
2.17.1
Quoted 4 lines · Jens Axboe
Please consider this series for 4.21. It's been part of my aio series that adds buffered and polled aio support for a long time, but these are standalone fixes and optimizations that we should apply for 4.21. Last one is just a prep patch for the series.
Tested-by Jeff Moyer <[email protected]>
Quoted 1 line
From the "better late than never" department.
Happy holidays!
-Jeff
Quoted 7 lines · Jeff Moyer
Quoted 4 lines · Jens Axboe
Please consider this series for 4.21. It's been part of my aio series that adds buffered and polled aio support for a long time, but these are standalone fixes and optimizations that we should apply for 4.21. Last one is just a prep patch for the series.
Quoted 5 lines
Tested-by Jeff Moyer <[email protected]>From the "better late than never" department.
Happy holidays!
Thanks Jeff! I did already ship this one, but I don't think it's been pulled. I provided an alternate tag with your Tested-by added.
Signature · 1 line
Jens Axboe