Make diskstats per-cpu using kmalloc_percpu
Subsystems (2)
PATCH 1/1 Make diskstats per-cpu using kmalloc_percpu
Hi Andrew,
Sometime back you'd asked for this on irc. This version makes the disk stats on struct gendisk per-cpu. I am working on making the per partition stats per-cpu too (struct hd_struct), Meanwhile, I was wondering if this can get some testing on the -mm tree. Patch has been tested on a PIII 4way in smp and up configurations. Applies on 2.5.63.
Thanks,
Kiran
drivers/block/genhd.c | 23 +++++++--- drivers/block/ll_rw_blk.c | 29 +++++++------ drivers/md/md.c | 4 + fs/partitions/check.c | 8 --- include/linux/genhd.h | 97 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 124 insertions(+), 37 deletions(-)
drivers/block/genhd.c +15 -8
@@ -328,14 +328,16 @@"%8u %8u %8llu %8u " "%8u %8u %8u" "\n" , - disk->reads, disk->read_merges, - (unsigned long long )disk->read_sectors, -jiffies_to_msec (disk->read_ticks), - disk->writes, disk->write_merges, - (unsigned long long )disk->write_sectors, -jiffies_to_msec (disk->write_ticks), - disk->in_flight,jiffies_to_msec (disk->io_ticks), - jiffies_to_msec(disk->time_in_queue)); +DISK_STAT_READ (disk, reads),DISK_STAT_READ (disk, read_merges), + (unsigned long long )DISK_STAT_READ (disk, read_sectors), +jiffies_to_msec (DISK_STAT_READ (disk, read_ticks)), +DISK_STAT_READ (disk, writes), +DISK_STAT_READ (disk, write_merges), + (unsigned long long )DISK_STAT_READ (disk, write_sectors), +jiffies_to_msec (DISK_STAT_READ (disk, write_ticks)), +DISK_STAT_READ (disk, in_flight), +jiffies_to_msec (DISK_STAT_READ (disk, io_ticks)), + jiffies_to_msec(DISK_STAT_READ(disk, time_in_queue))); }static struct disk_attribute disk_attr_dev = { .attr = {.name ="dev" , .mode =S_IRUGO },
@@ -367,6 +369,7 @@struct gendisk *disk =to_disk (kobj);kfree (disk->random);kfree (disk->part); +free_disk_stats (disk);kfree (disk); }
@@ -386,6 +389,10 @@struct gendisk *disk =kmalloc (sizeof (struct gendisk ),GFP_KERNEL );if (disk) {memset (disk,0 ,sizeof (struct gendisk )); +if (!init_disk_stats (disk)) { +kfree (disk); +return NULL ; + }if (minors >1 ) {int size = (minors -1 ) *sizeof (struct hd_struct ); disk->part =kmalloc (size,GFP_KERNEL );
drivers/block/ll_rw_blk.c +15 -14
@@ -1474,17 +1474,17 @@return ;if (rw ==READ ) { - rq->rq_disk->read_sectors += nr_sectors; +DISK_STAT_ADD (rq->rq_disk, read_sectors, nr_sectors);if (!new_io) - rq->rq_disk->read_merges++; +DISK_STAT_INC (rq->rq_disk, read_merges); }else if (rw ==WRITE ) { - rq->rq_disk->write_sectors += nr_sectors; +DISK_STAT_ADD (rq->rq_disk, write_sectors, nr_sectors);if (!new_io) - rq->rq_disk->write_merges++; +DISK_STAT_INC (rq->rq_disk, write_merges); }if (new_io) {disk_round_stats (rq->rq_disk); - rq->rq_disk->in_flight++; +DISK_STAT_INC (rq->rq_disk, in_flight); } }
@@ -1524,11 +1524,12 @@ {unsigned long now = jiffies; - disk->time_in_queue += disk->in_flight * (now - disk->stamp); +DISK_STAT_ADD (disk, time_in_queue, +DISK_STAT_READ (disk, in_flight) * (now - disk->stamp)); disk->stamp = now; -if (disk->in_flight) - disk->io_ticks += (now - disk->stamp_idle); +if (DISK_STAT_READ (disk, in_flight)) +DISK_STAT_ADD (disk, io_ticks, (now - disk->stamp_idle)); disk->stamp_idle = now; }
@@ -1646,7 +1647,7 @@if (req->rq_disk) {disk_round_stats (req->rq_disk); - req->rq_disk->in_flight--; +DISK_STAT_DEC (req->rq_disk, in_flight); }__blk_put_request (q, next);
@@ -2198,16 +2199,16 @@unsigned long duration = jiffies - req->start_time;switch (rq_data_dir (req)) {case WRITE : - disk->writes++; - disk->write_ticks += duration; +DISK_STAT_INC (disk, writes); +DISK_STAT_ADD (disk, write_ticks, duration);break ;case READ : - disk->reads++; - disk->read_ticks += duration; +DISK_STAT_INC (disk, reads); +DISK_STAT_ADD (disk, read_ticks, duration);break ; }disk_round_stats (disk); - disk->in_flight--; +DISK_STAT_DEC (disk, in_flight); }__blk_put_request (req->q, req); }
drivers/md/md.c +3 -1
@@ -2771,7 +2771,9 @@ idle =1 ; ITERATE_RDEV(mddev,rdev,tmp) {struct gendisk *disk = rdev->bdev->bd_contains->bd_disk; - curr_events = disk->read_sectors + disk->write_sectors - disk->sync_io; + curr_events =DISK_STAT_READ (disk, read_sectors) + +DISK_STAT_READ (disk, write_sectors) - + disk->sync_io;if ((curr_events - rdev->last_events) >32 ) { rdev->last_events = curr_events; idle =0 ;
fs/partitions/check.c +1 -7
@@ -503,13 +503,7 @@ disk->capacity =0 ; disk->flags &= ~GENHD_FL_UP ;unlink_gendisk (disk); - disk->reads = disk->writes =0 ; - disk->read_sectors = disk->write_sectors =0 ; - disk->read_merges = disk->write_merges =0 ; - disk->read_ticks = disk->write_ticks =0 ; - disk->in_flight =0 ; - disk->io_ticks =0 ; - disk->time_in_queue =0 ; +disk_stat_set_all (disk,0 ); disk->stamp = disk->stamp_idle =0 ;devfs_remove_partitions (disk);if (disk->driverfs_dev) {
include/linux/genhd.h +90 -7
112 lines not on this page · raw message
Quoted 2 lines · Ravikiran G Thirumalai
This version makes the disk stats on struct gendisk per-cpu.
Well OK, but it has lots of SCARY UPPER-CASE MACROS and disk_stat_inc(), disk_stat_dec() and disk_stat_sub() can all be defined in terms of disk_stat_add().
I did this to it:
drivers/block/genhd.c | 27 ++++++++----- drivers/block/ll_rw_blk.c | 29 +++++++------- drivers/md/md.c | 4 + fs/partitions/check.c | 8 --- include/linux/genhd.h | 94 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 123 insertions(+), 39 deletions(-)
drivers/block/genhd.c renamed from drivers/block/genhd.c~per-cpu-disk-stats +17 -10
@@ -318,7 +318,7 @@ static inline unsigned jiffies_to_msec(ureturn (jif /HZ ) *1000 + (jif %HZ ) *1000 /HZ ;#endif } -static ssize_t disk_stat_read (struct gendisk * disk,char *page) +static ssize_t disk_stats_read (struct gendisk * disk,char *page) {disk_round_stats (disk); return sprintf(page,
@@ -326,14 +326,16 @@ static ssize_t disk_stat_read(struct gen"%8u %8u %8llu %8u " "%8u %8u %8u" "\n" , - disk->reads, disk->read_merges, - (unsigned long long )disk->read_sectors, -jiffies_to_msec (disk->read_ticks), - disk->writes, disk->write_merges, - (unsigned long long )disk->write_sectors, -jiffies_to_msec (disk->write_ticks), - disk->in_flight,jiffies_to_msec (disk->io_ticks), - jiffies_to_msec(disk->time_in_queue)); +disk_stat_read (disk, reads),disk_stat_read (disk, read_merges), + (unsigned long long )disk_stat_read (disk, read_sectors), +jiffies_to_msec (disk_stat_read (disk, read_ticks)), +disk_stat_read (disk, writes), +disk_stat_read (disk, write_merges), + (unsigned long long )disk_stat_read (disk, write_sectors), +jiffies_to_msec (disk_stat_read (disk, write_ticks)), +disk_stat_read (disk, in_flight), +jiffies_to_msec (disk_stat_read (disk, io_ticks)), + jiffies_to_msec(disk_stat_read(disk, time_in_queue))); }static struct disk_attribute disk_attr_dev = { .attr = {.name ="dev" , .mode =S_IRUGO },
@@ -349,7 +351,7 @@ static struct disk_attribute disk_attr_s };static struct disk_attribute disk_attr_stat = { .attr = {.name ="stat" , .mode =S_IRUGO }, - .show = disk_stat_read + .show = disk_stats_read };static struct attribute * default_attrs[] = {
@@ -365,6 +367,7 @@ static void disk_release(struct kobjectstruct gendisk *disk =to_disk (kobj);kfree (disk->random);kfree (disk->part); +free_disk_stats (disk);kfree (disk); }
@@ -384,6 +387,10 @@ struct gendisk *alloc_disk(int minors)struct gendisk *disk =kmalloc (sizeof (struct gendisk ),GFP_KERNEL );if (disk) {memset (disk,0 ,sizeof (struct gendisk )); +if (!init_disk_stats (disk)) { +kfree (disk); +return NULL ; + }if (minors >1 ) {int size = (minors -1 ) *sizeof (struct hd_struct ); disk->part =kmalloc (size,GFP_KERNEL );
drivers/block/ll_rw_blk.c renamed from drivers/block/ll_rw_blk.c~per-cpu-disk-stats +15 -14
@@ -1458,17 +1458,17 @@ void drive_stat_acct(struct request *rq,return ;if (rw ==READ ) { - rq->rq_disk->read_sectors += nr_sectors; +disk_stat_add (rq->rq_disk, read_sectors, nr_sectors);if (!new_io) - rq->rq_disk->read_merges++; +disk_stat_inc (rq->rq_disk, read_merges); }else if (rw ==WRITE ) { - rq->rq_disk->write_sectors += nr_sectors; +disk_stat_add (rq->rq_disk, write_sectors, nr_sectors);if (!new_io) - rq->rq_disk->write_merges++; +disk_stat_inc (rq->rq_disk, write_merges); }if (new_io) {disk_round_stats (rq->rq_disk); - rq->rq_disk->in_flight++; +disk_stat_inc (rq->rq_disk, in_flight); } }
@@ -1508,11 +1508,12 @@ void disk_round_stats(struct gendisk *di {unsigned long now = jiffies; - disk->time_in_queue += disk->in_flight * (now - disk->stamp); +disk_stat_add (disk, time_in_queue, +disk_stat_read (disk, in_flight) * (now - disk->stamp)); disk->stamp = now; -if (disk->in_flight) - disk->io_ticks += (now - disk->stamp_idle); +if (disk_stat_read (disk, in_flight)) +disk_stat_add (disk, io_ticks, (now - disk->stamp_idle)); disk->stamp_idle = now; }
@@ -1628,7 +1629,7 @@ static int attempt_merge(request_queue_tif (req->rq_disk) {disk_round_stats (req->rq_disk); - req->rq_disk->in_flight--; +disk_stat_dec (req->rq_disk, in_flight); }__blk_put_request (q, next);
@@ -2180,16 +2181,16 @@ void end_that_request_last(struct requesunsigned long duration = jiffies - req->start_time;switch (rq_data_dir (req)) {case WRITE : - disk->writes++; - disk->write_ticks += duration; +disk_stat_inc (disk, writes); +disk_stat_add (disk, write_ticks, duration);break ;case READ : - disk->reads++; - disk->read_ticks += duration; +disk_stat_inc (disk, reads); +disk_stat_add (disk, read_ticks, duration);break ; }disk_round_stats (disk); - disk->in_flight--; +disk_stat_dec (disk, in_flight); }__blk_put_request (req->q, req); }
drivers/md/md.c renamed from drivers/md/md.c~per-cpu-disk-stats +3 -1
@@ -2803,7 +2803,9 @@ static int is_mddev_idle(mddev_t *mddev) idle =1 ; ITERATE_RDEV(mddev,rdev,tmp) {struct gendisk *disk = rdev->bdev->bd_contains->bd_disk; - curr_events = disk->read_sectors + disk->write_sectors - disk->sync_io; + curr_events =disk_stat_read (disk, read_sectors) + +disk_stat_read (disk, write_sectors) - + disk->sync_io;if ((curr_events - rdev->last_events) >32 ) { rdev->last_events = curr_events; idle =0 ;
fs/partitions/check.c renamed from fs/partitions/check.c~per-cpu-disk-stats +1 -7
@@ -503,13 +503,7 @@ void del_gendisk(struct gendisk *disk) disk->capacity =0 ; disk->flags &= ~GENHD_FL_UP ;unlink_gendisk (disk); - disk->reads = disk->writes =0 ; - disk->read_sectors = disk->write_sectors =0 ; - disk->read_merges = disk->write_merges =0 ; - disk->read_ticks = disk->write_ticks =0 ; - disk->in_flight =0 ; - disk->io_ticks =0 ; - disk->time_in_queue =0 ; +disk_stat_set_all (disk,0 ); disk->stamp = disk->stamp_idle =0 ;devfs_remove_partitions (disk);if (disk->driverfs_dev) {
include/linux/genhd.h renamed from include/linux/genhd.h~per-cpu-disk-stats +87 -7
109 lines not on this page · raw message
_
Quoted 10 lines · Andrew Morton
Quoted 2 lines · Ravikiran G Thirumalai
This version makes the disk stats on struct gendisk per-cpu.
Well OK, but it has lots of SCARY UPPER-CASE MACROS and
disk_stat_inc(),disk_stat_dec()anddisk_stat_sub()can all be defined in terms ofdisk_stat_add().I did this to it:
Ok, I don't have a preference one way or the other. I used upper case since mib stats used upper case macros (it was that way before I twiddled with it).
I'll use lower case macros for the per partition stats too.
Thanks,
Kiran
This version makes the disk stats on struct gendisk per-cpu. I am working on making the per partition stats per-cpu too (struct hd_struct).
In general I'm in favor of this. It seems intuitive to me that counters of this type should be per-cpu. But the question is, do we actually see any gains? At the very least, are we sure we've not introduced any degradation? Has any of your testing so far been measuring performance or just checking for correctness?
Rick
Quoted 9 lines · Rick Lindsley
This version makes the disk stats on struct gendisk per-cpu. I am working on making the per partition stats per-cpu too (struct
hd_struct).In general I'm in favor of this. It seems intuitive to me that counters of this type should be per-cpu. But the question is, do we actually see any gains? At the very least, are we sure we've not introduced any degradation? Has any of your testing so far been measuring performance or just checking for correctness?
Tests on this patch have been for correctness. However, we had done some measurements sometime back (actually loong time back) to verify if we actually saw any gains with similar statistics counters. The results are at
http://lse.sourceforge.net/counters/statctr.html
Since, we already know that per-cpu counters are good from our experiments earlier, I did not bother doing it for this patch.