rq-dyn-alloc, dynamic request allocation
PATCH 1/1 rq-dyn-alloc, dynamic request allocation
Hi,
This patch adds dynamic request allocation to the block io path. On systems with lots of disks (and thus queues) it saves a non-significant amount of low memory. It also allows for much better experimentation with larger queue lengths, this experimental patch tops the queue depth off at 16384 (vs 128 before).
Please play with it. Andrew, want a version for -mm?
axboe@smithers:/mnt/kernel/iosched $ diffstat rq-dyn-alloc-3
drivers/block/deadline-iosched.c | 105 +++++++++-------- drivers/block/elevator.c | 19 +++ drivers/block/ll_rw_blk.c | 238 +++++++++++---------------------------- include/linux/blkdev.h | 12 + include/linux/elevator.h | 7 + 5 files changed, 167 insertions(+), 214 deletions(-)
# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.1013 -> 1.1014
# include/linux/elevator.h 1.18 -> 1.19
# drivers/block/ll_rw_blk.c 1.161 -> 1.162
# include/linux/blkdev.h 1.98 -> 1.99
# drivers/block/elevator.c 1.40 -> 1.41
# drivers/block/deadline-iosched.c 1.17 -> 1.18
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 03/04/01 [email protected] 1.1014
# dynamic request allocation # -------------------------------------------- # diff -Nru a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c --- a/drivers/block/deadline-iosched.c Tue Apr 1 12:22:05 2003 +++ b/drivers/block/deadline-iosched.c Tue Apr 1 12:22:05 2003 @@ -28,7 +28,7 @@ static int fifo_batch = 16; /* # of sequential requests treated as one by the above parameters. For throughput. */
-static const int deadline_hash_shift = 10;
+static const int deadline_hash_shift = 5;
#define DL_HASH_BLOCK(sec) ((sec) >> 3)
#define DL_HASH_FN(sec) (hash_long(DL_HASH_BLOCK((sec)), deadline_hash_shift))
#define DL_HASH_ENTRIES (1 << deadline_hash_shift)
@@ -71,6 +71,8 @@
int fifo_batch;
int writes_starved;
int front_merges;
+
+ mempool_t *drq_pool;
};
/*
@@ -130,6 +132,21 @@
list_add(&drq->hash, &dd->hash[DL_HASH_FN(rq_hash_key(rq))]);
}
+/*
+ * move hot entry to front of chain
+ */
+static inline void
+deadline_hot_drq_hash(struct deadline_data *dd, struct deadline_rq *drq) +{
+ struct request *rq = drq->request;
+ struct list_head *head = &dd->hash[DL_HASH_FN(rq_hash_key(rq))];
+
+ if (ON_HASH(drq) && drq->hash.prev != head) {
+ list_del(&drq->hash);
+ list_add(&drq->hash, head);
+ }
+}
+
static struct request *
deadline_find_drq_hash(struct deadline_data *dd, sector_t offset) {
@@ -353,6 +370,8 @@
out:
q->last_merge = &__rq->queuelist;
out_insert:
+ if (ret)
+ deadline_hot_drq_hash(dd, RQ_DATA(__rq));
*insert = &__rq->queuelist;
return ret;
}
@@ -673,28 +692,11 @@
static void deadline_exit(request_queue_t *q, elevator_t *e) {
struct deadline_data *dd = e->elevator_data;
- struct deadline_rq *drq;
- struct request *rq;
- int i;
BUG_ON(!list_empty(&dd->fifo_list[READ])); BUG_ON(!list_empty(&dd->fifo_list[WRITE]));
- for (i = READ; i <= WRITE; i++) {
- struct request_list *rl = &q->rq[i];
- struct list_head *entry;
-
- list_for_each(entry, &rl->free) {
- rq = list_entry_rq(entry);
-
- if ((drq = RQ_DATA(rq)) == NULL)
- continue;
-
- rq->elevator_private = NULL;
- kmem_cache_free(drq_pool, drq);
- }
- }
-
+ mempool_destroy(dd->drq_pool);
kfree(dd->hash);
kfree(dd);
}
@@ -706,9 +708,7 @@
static int deadline_init(request_queue_t *q, elevator_t *e) {
struct deadline_data *dd;
- struct deadline_rq *drq;
- struct request *rq;
- int i, ret = 0;
+ int i;
if (!drq_pool)
return -ENOMEM;
@@ -724,6 +724,13 @@
return -ENOMEM;
}
+ dd->drq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, drq_pool);
+ if (!dd->drq_pool) {
+ kfree(dd->hash);
+ kfree(dd);
+ return -ENOMEM;
+ }
+
for (i = 0; i < DL_HASH_ENTRIES; i++)
INIT_LIST_HEAD(&dd->hash[i]);
@@ -739,33 +746,41 @@
dd->front_merges = 1;
dd->fifo_batch = fifo_batch;
e->elevator_data = dd;
+ return 0;
+}
- for (i = READ; i <= WRITE; i++) {
- struct request_list *rl = &q->rq[i];
- struct list_head *entry;
-
- list_for_each(entry, &rl->free) {
- rq = list_entry_rq(entry);
-
- drq = kmem_cache_alloc(drq_pool, GFP_KERNEL);
- if (!drq) {
- ret = -ENOMEM;
- break;
- }
+static void deadline_put_request(request_queue_t *q, struct request *rq) +{
+ struct deadline_data *dd = q->elevator.elevator_data;
+ struct deadline_rq *drq = RQ_DATA(rq);
- memset(drq, 0, sizeof(*drq));
- INIT_LIST_HEAD(&drq->fifo);
- INIT_LIST_HEAD(&drq->hash);
- RB_CLEAR(&drq->rb_node);
- drq->request = rq;
- rq->elevator_private = drq;
- }
+ if (drq) {
+ mempool_free(drq, dd->drq_pool);
+ rq->elevator_private = NULL;
}
+}
- if (ret)
- deadline_exit(q, e);
+static int
+deadline_set_request(request_queue_t *q, struct request *rq, int gfp_mask) +{
+ struct deadline_data *dd = q->elevator.elevator_data;
+ struct deadline_rq *drq;
- return ret;
+ drq = mempool_alloc(dd->drq_pool, gfp_mask);
+ if (drq) {
+ RB_CLEAR(&drq->rb_node);
+ drq->request = rq;
+
+ INIT_LIST_HEAD(&drq->hash);
+ drq->hash_valid_count = 0;
+
+ INIT_LIST_HEAD(&drq->fifo);
+
+ rq->elevator_private = drq;
+ return 0;
+ }
+
+ return 1;
}
/*
@@ -916,6 +931,8 @@
.elevator_queue_empty_fn = deadline_queue_empty,
.elevator_former_req_fn = deadline_former_request,
.elevator_latter_req_fn = deadline_latter_request,
+ .elevator_set_req_fn = deadline_set_request,
+ .elevator_put_req_fn = deadline_put_request,
.elevator_init_fn = deadline_init,
.elevator_exit_fn = deadline_exit,
drivers/block/elevator.c +19 -0
@@ -408,6 +408,25 @@return NULL ; } +int elv_set_request (request_queue_t *q,struct request *rq,int gfp_mask) +{ +elevator_t *e = &q->elevator; + +if (e->elevator_set_req_fn) +return e->elevator_set_req_fn (q, rq, gfp_mask); + + rq->elevator_private =NULL ; +return 0 ; +} + +void elv_put_request (request_queue_t *q,struct request *rq) +{ +elevator_t *e = &q->elevator; + +if (e->elevator_put_req_fn) + e->elevator_put_req_fn (q, rq); +} +int elv_register_queue (struct gendisk *disk) {request_queue_t *q = disk->queue;
drivers/block/ll_rw_blk.c +73 -165
398 lines not on this page · raw message
include/linux/blkdev.h +7 -5
@@ -10,6 +10,7 @@#include <linux/pagemap.h> #include <linux/backing-dev.h> #include <linux/wait.h> +#include <linux/mempool.h> #include <asm/scatterlist.h>
@@ -18,10 +19,12 @@struct elevator_s ;typedef struct elevator_s elevator_t ; +#define BLKDEV_MIN_RQ 4 +#define BLKDEV_MAX_RQ 16384 +struct request_list { -unsigned int count; -struct list_head free; -wait_queue_head_t wait; +int count[2 ]; +mempool_t *rq_pool; };/*
@@ -180,7 +183,7 @@/* * the queue request freelist, one for reads and one for writes */ -struct request_list rq[2 ]; +struct request_list rq;request_fn_proc *request_fn;merge_request_fn *back_merge_fn;
@@ -330,7 +333,6 @@extern void blk_attempt_remerge (request_queue_t *,struct request *);extern void __blk_attempt_remerge (request_queue_t *,struct request *);extern struct request *blk_get_request (request_queue_t *,int ,int ); -extern struct request *__blk_get_request (request_queue_t *,int );extern void blk_put_request (struct request *);extern void blk_insert_request (request_queue_t *,struct request *,int ,void *);extern void blk_plug_device (request_queue_t *);
include/linux/elevator.h +7 -0
@@ -15,6 +15,8 @@typedef void (elevator_remove_req_fn ) (request_queue_t *,struct request *);typedef struct request *(elevator_request_list_fn ) (request_queue_t *,struct request *);typedef struct list_head *(elevator_get_sort_head_fn ) (request_queue_t *,struct request *); +typedef int (elevator_set_req_fn ) (request_queue_t *,struct request *,int ); +typedef void (elevator_put_req_fn ) (request_queue_t *,struct request *);typedef int (elevator_init_fn ) (request_queue_t *,elevator_t *);typedef void (elevator_exit_fn ) (request_queue_t *,elevator_t *);
@@ -34,6 +36,9 @@elevator_request_list_fn *elevator_former_req_fn;elevator_request_list_fn *elevator_latter_req_fn; +elevator_set_req_fn *elevator_set_req_fn; +elevator_put_req_fn *elevator_put_req_fn; +elevator_init_fn *elevator_init_fn;elevator_exit_fn *elevator_exit_fn;
@@ -58,6 +63,8 @@extern struct request *elv_latter_request (request_queue_t *,struct request *);extern int elv_register_queue (struct gendisk *);extern void elv_unregister_queue (struct gendisk *); +extern int elv_set_request (request_queue_t *,struct request *,int ); +extern void elv_put_request (request_queue_t *,struct request *);#define __elv_add_request_pos (q, rq, pos) \ (q)->elevator.elevator_add_req_fn((q), (rq), (pos))
Signature · 1 line
Jens Axboe
Quoted 8 lines · Jens Axboe
Hi,
This patch adds dynamic request allocation to the block io path. On systems with lots of disks (and thus queues) it saves a non-significant amount of low memory. It also allows for much better experimentation with larger queue lengths, this experimental patch tops the queue depth off at 16384 (vs 128 before).
heh, 16k requests per queue? Last time I played with 1024 certain popular benchmarks ran like a bullet.
Quoted 1 line
Please play with it. Andrew, want a version for -mm?
Would be much appreciated, thanks.
Quoted 14 lines
*/
static struct request *get_request_wait(request_queue_t *q, int rw) {drivers/block/ll_rw_blk.c +1
-5-DEFINE_WAIT (wait); -struct request_list *rl = &q->rq[rw];struct request *rq; -spin_lock_prefetch (q->queue_lock); -generic_unplug_device (q); do { - int block = 0; + rq = get_request(q, rw, GFP_NOIO);
drivers/block/ll_rw_blk.c line 1297
Quoted 10 lines
drivers/block/ll_rw_blk.c +1
-8-prepare_to_wait_exclusive (&rl->wait, &wait, -TASK_UNINTERRUPTIBLE ); -spin_lock_irq (q->queue_lock); -if (!rl->count) - block =1 ; -spin_unlock_irq (q->queue_lock); - -if (block) +if (!rq)io_schedule ();
hmm. I fear that if a SCHED_FIFO/SCHED_RR task hits this, it will just pick itself to run again in the schedule() and the box locks up.
A blk_congestion_wait(WRITE, HZ/50) may be better here. It will send the caller to sleep until someone puts a write request back, which seems appropriate.
Quoted 4 lines · Andrew Morton
Quoted 3 lines · Jens Axboe
Hi,
Quoted 8 lines
Quoted 5 lines
This patch adds dynamic request allocation to the block io path. On systems with lots of disks (and thus queues) it saves a non-significant amount of low memory. It also allows for much better experimentation with larger queue lengths, this experimental patch tops the queue depth off at 16384 (vs 128 before).
heh, 16k requests per queue? Last time I played with 1024 certain popular benchmarks ran like a bullet.
I have one such unmentionable benchmark running right now, current depth is 5127 :-)
Quoted 3 lines
Quoted 1 line
Please play with it. Andrew, want a version for -mm?
Would be much appreciated, thanks.
Ok, getting on it.
Quoted 28 lines
Quoted 14 lines
*/
static struct request *get_request_wait(request_queue_t *q, int rw) {drivers/block/ll_rw_blk.c +1
-5-DEFINE_WAIT (wait); -struct request_list *rl = &q->rq[rw];struct request *rq; -spin_lock_prefetch (q->queue_lock); -generic_unplug_device (q); do { - int block = 0; + rq = get_request(q, rw, GFP_NOIO);drivers/block/ll_rw_blk.c line 1297
Quoted 10 lines
drivers/block/ll_rw_blk.c +1
-8-prepare_to_wait_exclusive (&rl->wait, &wait, -TASK_UNINTERRUPTIBLE ); -spin_lock_irq (q->queue_lock); -if (!rl->count) - block =1 ; -spin_unlock_irq (q->queue_lock); - -if (block) +if (!rq)io_schedule ();hmm. I fear that if a SCHED_FIFO/SCHED_RR task hits this, it will just pick itself to run again in the
schedule()and the box locks up.
Quoted 3 lines
A
blk_congestion_wait(WRITE, HZ/50)may be better here. It will send the caller to sleep until someone puts a write request back, which seems appropriate.
Yes good point, I'll make that change.
Signature · 1 line
Jens Axboe