blk: reimplement elevator switch
PATCH 1/1 blk: reimplement elevator switch
Hello, Jens.
This patch reimplements elevator switch. This patch assumes generic
dispatch queue patchset is applied.
* Each request is tagged with REQ_ELVPRIV flag if it has its elevator private data set.
* Requests which doesn't have REQ_ELVPRIV flag set never enter iosched. They are always directly back inserted to dispatch queue. Of course, elevator_put_req_fn is called only for requests which have its REQ_ELVPRIV set.
* Request queue maintains the current number of requests which have its elevator data set (elevator_set_req_fn called) in q->rq->elvpriv.
* If a request queue has QUEUE_FLAG_BYPASS set, elevator private data is not allocated for new requests.
To switch to another iosched, we set QUEUE_FLAG_BYPASS and wait until
elvpriv goes to zero; then, we attach the new iosched and clears QUEUE_FLAG_BYPASS. New implementation is much simpler and main code paths are less cluttered, IMHO.
drivers/block/elevator.c +35 -45
@@ -34,6 +34,7 @@#include <linux/slab.h> #include <linux/init.h> #include <linux/compiler.h> +#include <linux/delay.h> #include <asm/uaccess.h>
@@ -135,13 +136,7 @@ static int elevator_attach(request_queuememset (eq,0 ,sizeof (*eq)); eq->ops = &e->ops; eq->elevator_type = e; - -INIT_LIST_HEAD (&q->queue_head); - q->last_merge =NULL ; q->elevator = eq; - q->last_sector =0 ; - q->boundary_rq =NULL ; - q->max_back_kb =0 ;if (eq->ops->elevator_init_fn) ret = eq->ops->elevator_init_fn (q, eq);
@@ -190,6 +185,12 @@ int elevator_init(request_queue_t *q, chstruct elevator_queue *eq;int ret =0 ; +INIT_LIST_HEAD (&q->queue_head); + q->last_merge =NULL ; + q->last_sector =0 ; + q->boundary_rq =NULL ; + q->max_back_kb =0 ; +elevator_setup_default ();if (!name)
@@ -353,23 +354,14 @@ void __elv_add_request(request_queue_t * q->last_sector =rq_last_sector (rq); q->boundary_rq = rq; } - } + }else if (!(rq->flags &REQ_ELVPRIV ) && where ==ELEVATOR_INSERT_SORT ) + where =ELEVATOR_INSERT_BACK ;if (plug)blk_plug_device (q); rq->q = q; -if (unlikely (test_bit (QUEUE_FLAG_DRAIN , &q->queue_flags))) { -/* -* if drain is set, store the request "locally". when the drain -* is finished, the requests will be handed ordered to the io -* scheduler -*/ -list_add_tail (&rq->queuelist, &q->drain_list); -return ; - } -switch (where) {case ELEVATOR_INSERT_FRONT : rq->flags |=REQ_SOFTBARRIER ;
@@ -676,25 +668,36 @@ EXPORT_SYMBOL_GPL(elv_unregister);* switch to new_e io scheduler. be careful not to introduce deadlocks - * we don't free the old io scheduler, before we have allocated what we * need for the new one. this way we have a chance of going back to the old -* one, if the new one fails init for some reason. we also do an intermediate -* switch to noop to ensure safety with stack-allocated requests, since they -* don't originate from the block layer allocator. noop is safe here, because -* it never needs to touch the elevator itself for completion events. DRAIN -* flags will make sure we don't touch it for additions either. +* one, if the new one fails init for some reason. */ static void elevator_switch (request_queue_t *q,struct elevator_type *new_e) { -elevator_t *e =kmalloc (sizeof (elevator_t),GFP_KERNEL ); -struct elevator_type *noop_elevator =NULL ; -elevator_t *old_elevator; +elevator_t *old_elevator, *e; +/* +* Allocate new elevator +*/ + e =kmalloc (sizeof (elevator_t),GFP_KERNEL );if (!e) goto error;/* -* first step, drain requests from the block freelist +* Turn on BYPASS and drain all requests w/ elevator private data */ -blk_wait_queue_drained (q,0 ); +spin_lock_irq (q->queue_lock); + +set_bit (QUEUE_FLAG_BYPASS , &q->queue_flags); + +while (q->elevator->ops->elevator_dispatch_fn (q,1 )) + ; + +while (q->rq.elvpriv) { +spin_unlock_irq (q->queue_lock); +msleep (100 ); +spin_lock_irq (q->queue_lock); + } + +spin_unlock_irq (q->queue_lock);/* * unregister old elevator data
@@ -703,18 +706,6 @@ static void elevator_switch(request_queu old_elevator = q->elevator;/* -* next step, switch to noop since it uses no private rq structures -* and doesn't allocate any memory for anything. then wait for any -* non-fs requests in-flight -*/ - noop_elevator =elevator_get ("noop" ); -spin_lock_irq (q->queue_lock); -elevator_attach (q, noop_elevator, e); -spin_unlock_irq (q->queue_lock); - -blk_wait_queue_drained (q,1 ); - -/* * attach and start new elevator */ if (elevator_attach (q, new_e, e))
@@ -724,11 +715,10 @@ static void elevator_switch(request_queu goto fail_register;/* -* finally exit old elevator and start queue again +* finally exit old elevator and turn off BYPASS. */ elevator_exit (old_elevator); -blk_finish_queue_drain (q); -elevator_put (noop_elevator); +clear_bit (QUEUE_FLAG_BYPASS , &q->queue_flags);return ; fail_register:
@@ -737,13 +727,13 @@ fail_register:* one again (along with re-adding the sysfs dir) */ elevator_exit (e); + e =NULL ; fail: q->elevator = old_elevator;elv_register_queue (q); -blk_finish_queue_drain (q); +clear_bit (QUEUE_FLAG_BYPASS , &q->queue_flags); +kfree (e); error: -if (noop_elevator) -elevator_put (noop_elevator);elevator_put (new_e);printk (KERN_ERR "elevator: switch to %s failed\n" ,new_e->elevator_name); }
drivers/block/ll_rw_blk.c +27 -115
236 lines not on this page · raw message
include/linux/blkdev.h +4 -6
@@ -107,9 +107,9 @@ typedef void (rq_end_io_fn)(struct requestruct request_list {int count[2 ];int starved[2 ]; +int elvpriv;mempool_t *rq_pool;wait_queue_head_t wait[2 ]; -wait_queue_head_t drain; };#define BLK_MAX_CDB 16
@@ -211,6 +211,7 @@ enum rq_flag_bits { __REQ_STARTED,/* drive already may have started this one */ __REQ_DONTPREP,/* don't call prep for this one */ __REQ_QUEUED,/* uses queueing */ + __REQ_ELVPRIV, /* elevator private data attached *//* * for ATA/ATAPI devices */
@@ -244,6 +245,7 @@ enum rq_flag_bits {#define REQ_STARTED (1 << __REQ_STARTED)#define REQ_DONTPREP (1 << __REQ_DONTPREP)#define REQ_QUEUED (1 << __REQ_QUEUED) +#define REQ_ELVPRIV (1 << __REQ_ELVPRIV)#define REQ_PC (1 << __REQ_PC)#define REQ_BLOCK_PC (1 << __REQ_BLOCK_PC)#define REQ_SENSE (1 << __REQ_SENSE)
@@ -414,8 +416,6 @@ struct request_queueunsigned int sg_reserved_size;int node; -struct list_head drain_list; -/* * reserved for flush operations */
@@ -443,7 +443,7 @@ enum {#define QUEUE_FLAG_DEAD 5/* queue being torn down */ #define QUEUE_FLAG_REENTER 6/* Re-entrancy avoidance */ #define QUEUE_FLAG_PLUGGED 7/* queue is plugged */ -#define QUEUE_FLAG_DRAIN 8/* draining queue for sched switch */ +#define QUEUE_FLAG_BYPASS 8/* don't use elevator, just do FIFO */ #define QUEUE_FLAG_FLUSH 9/* doing barrier flush sequence */ #define blk_queue_plugged (q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags)
@@ -655,8 +655,6 @@ extern void blk_dump_rq_flags(struct reqextern void generic_unplug_device (request_queue_t *);extern void __generic_unplug_device (request_queue_t *);extern long nr_blockdev_pages (void ); -extern void blk_wait_queue_drained (request_queue_t *,int ); -extern void blk_finish_queue_drain (request_queue_t *);int blk_get_queue (request_queue_t *);request_queue_t *blk_alloc_queue (int gfp_mask);
Quoted 17 lines · Tejun Heo
Hello, Jens.
This patch reimplements elevator switch. This patch assumes generic
dispatch queue patchset is applied.* Each request is tagged with
REQ_ELVPRIVflag if it has its elevator private data set.
* Requests which doesn't haveREQ_ELVPRIVflag set never enter iosched. They are always directly back inserted to dispatch queue. Of course,elevator_put_req_fnis called only for requests which have itsREQ_ELVPRIVset.
* Request queue maintains the current number of requests which have its elevator data set (elevator_set_req_fncalled) inq->rq->elvpriv.
* If a request queue hasQUEUE_FLAG_BYPASSset, elevator private data is not allocated for new requests.
Quoted 4 lines
To switch to another iosched, we set
QUEUE_FLAG_BYPASSand wait until
elvpriv goes to zero; then, we attach the new iosched and clearsQUEUE_FLAG_BYPASS. New implementation is much simpler and main code paths are less cluttered, IMHO.
Wonderful! Applied as-is, I didn't make any changes to this one. I agree it's much cleaner than the previous approach, both in the code and in killing the request_queue and request_list members.
I'm going to make a little few tweaks:
- The naming, QUEUE_FLAG_BYPASS isn't really clear. I don't know what this means without looking at specific parts of the code. Testing of same flag in various locations would also be preferred instead of passing priv around and cluttering the function parameters, however we should split the queue flags a little for this. Basically into an atomic and non-atomic part. So I'll leave that alone for now.
- The msleep(100) seems a little too slow. With the switching being more efficient now, in 100msecs we can complete lots of requests.
Signature · 1 line
Jens Axboe
Quoted 23 lines · Jens Axboe
Quoted 17 lines · Tejun Heo
Hello, Jens.
This patch reimplements elevator switch. This patch assumes generic
dispatch queue patchset is applied.* Each request is tagged with
REQ_ELVPRIVflag if it has its elevator private data set.
* Requests which doesn't haveREQ_ELVPRIVflag set never enter iosched. They are always directly back inserted to dispatch queue. Of course,elevator_put_req_fnis called only for requests which have itsREQ_ELVPRIVset.
* Request queue maintains the current number of requests which have its elevator data set (elevator_set_req_fncalled) inq->rq->elvpriv.
* If a request queue hasQUEUE_FLAG_BYPASSset, elevator private data is not allocated for new requests.Quoted 4 lines
To switch to another iosched, we set
QUEUE_FLAG_BYPASSand wait until
elvpriv goes to zero; then, we attach the new iosched and clearsQUEUE_FLAG_BYPASS. New implementation is much simpler and main code paths are less cluttered, IMHO.
Quoted 8 lines
Wonderful! Applied as-is, I didn't make any changes to this one. I agree it's much cleaner than the previous approach, both in the code and in killing the
request_queueandrequest_listmembers.I'm going to make a little few tweaks:
- The naming,
QUEUE_FLAG_BYPASSisn't really clear. I don't know what this means without looking at specific parts of the code. Testing of
It means to bypass ioscheds and go directly into dispatch queue.
Quoted 4 lines
same flag in various locations would also be preferred instead of passing priv around and cluttering the function parameters, however we should split the queue flags a little for this. Basically into an atomic and non-atomic part. So I'll leave that alone for now.
Hmmm...
Quoted 2 lines
- The msleep(100) seems a little too slow. With the switching being more efficient now, in 100msecs we can complete lots of requests.
Yeap, agreed.
Signature · 1 line
tejun
Quoted 28 lines · Tejun Heo
Quoted 27 lines · Jens Axboe
Quoted 17 lines · Tejun Heo
Hello, Jens.
This patch reimplements elevator switch. This patch assumes generic
dispatch queue patchset is applied.* Each request is tagged with
REQ_ELVPRIVflag if it has its elevator private data set.
* Requests which doesn't haveREQ_ELVPRIVflag set never enter iosched. They are always directly back inserted to dispatch queue. Of course,elevator_put_req_fnis called only for requests which have itsREQ_ELVPRIVset.
* Request queue maintains the current number of requests which have its elevator data set (elevator_set_req_fncalled) inq->rq->elvpriv.
* If a request queue hasQUEUE_FLAG_BYPASSset, elevator private data is not allocated for new requests.Quoted 4 lines
To switch to another iosched, we set
QUEUE_FLAG_BYPASSand wait until
elvpriv goes to zero; then, we attach the new iosched and clearsQUEUE_FLAG_BYPASS. New implementation is much simpler and main code paths are less cluttered, IMHO.Wonderful! Applied as-is, I didn't make any changes to this one. I agree it's much cleaner than the previous approach, both in the code and in killing the
request_queueandrequest_listmembers.
Quoted 6 lines
Quoted 4 lines
I'm going to make a little few tweaks:
- The naming,
QUEUE_FLAG_BYPASSisn't really clear. I don't know what this means without looking at specific parts of the code. Testing ofIt means to bypass ioscheds and go directly into dispatch queue.
Certainly :-) I renamed it to QUEUE_FLAG_ELVSWITCH for now, if another use comes of this we can name it something appropriately generic then.
Quoted 6 lines
Quoted 4 lines
same flag in various locations would also be preferred instead of passing priv around and cluttering the function parameters, however we should split the queue flags a little for this. Basically into an atomic and non-atomic part. So I'll leave that alone for now.
Hmmm...
There's room for optimization there, lots of places we check queue flags (and set them) inside the queue lock, we don't need to use the bit operations for those. But we cannot safely mix them either.
Quoted 4 lines
Quoted 2 lines
- The msleep(100) seems a little too slow. With the switching being more efficient now, in 100msecs we can complete lots of requests.
Yeap, agreed.
Cool
Signature · 1 line
Jens Axboe