aio: allocate kiocbs in batches
PATCH v3 1/1 aio: allocate kiocbs in batches
Hi,
In testing aio on a fast storage device, I found that the context lock takes up a fair amount of cpu time in the I/O submission path. The reason is that we take it for every I/O submitted (see __aio_get_req). Since we know how many I/Os are passed to io_submit, we can preallocate the kiocbs in batches, reducing the number of times we take and release the lock. In my testing, I was able to reduce the amount of time spent in _raw_spin_lock_irq by .56% (average of 3 runs). The command I used to test this was:
aio-stress -O -o 2 -o 3 -r 8 -d 128 -b 32 -i 32 -s 16384 <dev>
I also tested the patch with various numbers of events passed to io_submit, and I ran the xfstests aio group of tests to ensure I didn't break anything.
---
Changes from v2 -> v3:
- got rid of an extraneous structure member in the kiocb_batch.
- fixed up some haphazard variable types
- fixed a build warning
Changes from rfc -> v2:
- folded in akpm's incremental patch which fixes coding style and variable names
- tried to clarify a comment about a starvation case
- fixed up my breaking of the handling of that starvation case
- moved from an on-stack array to a list at the suggestion of akpm
fs/aio.c +107 -29
@@ -440,8 +440,6 @@ void exit_aio(struct mm_struct *mm)static struct kiocb *__aio_get_req (struct kioctx *ctx) {struct kiocb *req =NULL ; -struct aio_ring *ring; -int okay =0 ; req =kmem_cache_alloc (kiocb_cachep,GFP_KERNEL );if (unlikely (!req))
@@ -459,39 +457,114 @@ static struct kiocb *__aio_get_req(struct kioctx *ctx)INIT_LIST_HEAD (&req->ki_run_list); req->ki_eventfd =NULL ; -/* Check if the completion queue has enough free space to -* accept an event from this io. -*/ +return req; +} + +/* +* struct kiocb's are allocated in batches to reduce the number of +* times the ctx lock is acquired and released. +*/ +#define KIOCB_BATCH_SIZE 32L +struct kiocb_batch { +struct list_head head; +long count;/* number of requests left to allocate */ +}; + +static void kiocb_batch_init (struct kiocb_batch *batch,long total) +{ +INIT_LIST_HEAD (&batch->head); + batch->count = total; +} + +static void kiocb_batch_free (struct kiocb_batch *batch) +{ +struct kiocb *req, *n; + +list_for_each_entry_safe (req, n, &batch->head, ki_batch) { +list_del (&req->ki_batch); +kmem_cache_free (kiocb_cachep, req); + } +} + +/* +* Allocate a batch of kiocbs. This avoids taking and dropping the +* context lock a lot during setup. +*/ +static int kiocb_batch_refill (struct kioctx *ctx,struct kiocb_batch *batch) +{ +unsigned short allocated, to_alloc; +long avail; +bool called_fput = false; +struct kiocb *req, *n; +struct aio_ring *ring; + + to_alloc =min (batch->count,KIOCB_BATCH_SIZE ); +for (allocated =0 ; allocated < to_alloc; allocated++) { + req =__aio_get_req (ctx); +if (!req) +/* allocation failed, go with what we've got */ +break ; +list_add (&req->ki_batch, &batch->head); + } + +if (allocated ==0 ) + goto out; + +retry:spin_lock_irq (&ctx->ctx_lock); - ring =kmap_atomic (ctx->ring_info.ring_pages[0 ],KM_USER0 ); -if (ctx->reqs_active <aio_ring_avail (&ctx->ring_info, ring)) { + ring =kmap_atomic (ctx->ring_info.ring_pages[0 ]); + + avail =aio_ring_avail (&ctx->ring_info, ring) - ctx->reqs_active; +BUG_ON (avail <0 ); +if (avail ==0 && !called_fput) { +/* +* Handle a potential starvation case. It is possible that +* we hold the last reference on a struct file, causing us +* to delay the final fput to non-irq context. In this case, +* ctx->reqs_active is artificially high. Calling the fput +* routine here may free up a slot in the event completion +* ring, allowing this allocation to succeed. +*/ +kunmap_atomic (ring); +spin_unlock_irq (&ctx->ctx_lock); +aio_fput_routine (NULL ); + called_fput = true; + goto retry; + } + +if (avail < allocated) { +/* Trim back the number of requests. */ +list_for_each_entry_safe (req, n, &batch->head, ki_batch) { +list_del (&req->ki_batch); +kmem_cache_free (kiocb_cachep, req); +if (--allocated <= avail) +break ; + } + } + + batch->count -= allocated; +list_for_each_entry (req, &batch->head, ki_batch) {list_add (&req->ki_list, &ctx->active_reqs); ctx->reqs_active++; - okay =1 ; } -kunmap_atomic (ring,KM_USER0 ); -spin_unlock_irq (&ctx->ctx_lock); -if (!okay) { -kmem_cache_free (kiocb_cachep, req); - req =NULL ; - } +kunmap_atomic (ring); +spin_unlock_irq (&ctx->ctx_lock); -return req; +out: +return allocated; } -static inline struct kiocb *aio_get_req (struct kioctx *ctx) +static inline struct kiocb *aio_get_req (struct kioctx *ctx, +struct kiocb_batch *batch) {struct kiocb *req; -/* Handle a potential starvation case -- should be exceedingly rare as -* requests will be stuck on fput_head only if the aio_fput_routine is -* delayed and the requests were the last user of the struct file. -*/ - req =__aio_get_req (ctx); -if (unlikely (NULL == req)) { -aio_fput_routine (NULL ); - req =__aio_get_req (ctx); - } + +if (list_empty (&batch->head)) +if (kiocb_batch_refill (ctx, batch) ==0 ) +return NULL ; + req = list_first_entry(&batch->head, struct kiocb, ki_batch); +list_del (&req->ki_batch);return req; }
@@ -1515,7 +1588,8 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) }static int io_submit_one (struct kioctx *ctx,struct iocb __user *user_iocb, -struct iocb *iocb,bool compat) +struct iocb *iocb,struct kiocb_batch *batch, +bool compat) {struct kiocb *req;struct file *file;
@@ -1541,7 +1615,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,if (unlikely (!file))return -EBADF ; - req =aio_get_req (ctx);/* returns with 2 references to req */ + req =aio_get_req (ctx, batch);/* returns with 2 references to req */ if (unlikely (!req)) {fput (file);return -EAGAIN ;
@@ -1621,8 +1695,9 @@ long do_io_submit(aio_context_t ctx_id, long nr, {struct kioctx *ctx;long ret =0 ; -int i; +int i =0 ;struct blk_plug plug; +struct kiocb_batch batch;if (unlikely (nr <0 ))return -EINVAL ;
@@ -1639,6 +1714,8 @@ long do_io_submit(aio_context_t ctx_id, long nr,return -EINVAL ; } +kiocb_batch_init (&batch, nr); +blk_start_plug (&plug);/*
@@ -1659,12 +1736,13 @@ long do_io_submit(aio_context_t ctx_id, long nr,break ; } - ret =io_submit_one (ctx, user_iocb, &tmp, compat); + ret =io_submit_one (ctx, user_iocb, &tmp, &batch, compat);if (ret)break ; }blk_finish_plug (&plug); +kiocb_batch_free (&batch);put_ioctx (ctx);return i ? i : ret; }
include/linux/aio.h +1 -0
@@ -117,6 +117,7 @@ struct kiocb {struct list_head ki_list;/* the aio core uses this * for cancellation */ +struct list_head ki_batch;/* batch allocation */ /* * If the aio_resfd field of the userspace iocb is not zero,
Quoted 12 lines · Jeff Moyer
Hi,
In testing aio on a fast storage device, I found that the context lock takes up a fair amount of cpu time in the I/O submission path. The reason is that we take it for every I/O submitted (see
__aio_get_req). Since we know how many I/Os are passed toio_submit, we can preallocate the kiocbs in batches, reducing the number of times we take and release the lock. In my testing, I was able to reduce the amount of time spent in_raw_spin_lock_irqby .56% (average of 3 runs). The command I used to test this was:
aio-stress -O -o 2 -o 3 -r 8 -d 128 -b 32 -i 32 -s 16384 <dev>
Quoted 4 lines
I also tested the patch with various numbers of events passed to
io_submit, and I ran the xfstests aio group of tests to ensure I didn't break anything.
This patch looks great! It produces a significant speedup on a benchmark I care about.
I have been looking into improving the efficiency of io_submit, and I believe there is still more work that can be done.
Dan