Recent changes (master)
The following changes since commit 8116fd24b737c9d878ccb6a4cc13cc4f974dc2dc:
Update documentation for random_distribution=gauss (2016-03-03 14:00:54 -0700)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 7c4d0fb7e9e038000978b43b5674f9ad049d36b9:
Fix double free of td zone state index (2016-03-04 19:50:41 -0700)
----------------------------------------------------------------
Alan C (1):
Log to parent instead of twice to child td
Jens Axboe (3):
Add support for zones of random IO, with varying frequency of access
options: clean number of zones if we fail parsing
Fix double free of td zone state index
HOWTO | 20 ++++ backend.c | 9 ++ cconv.c | 32 +++++++ examples/rand-zones.fio | 18 ++++ fio.1 | 34 ++++++- fio.h | 10 ++ init.c | 1 + io_u.c | 108 ++++++++++++++++----- lib/rand.h | 11 +++ options.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++ server.h | 2 +- thread_options.h | 13 ++- 12 files changed, 477 insertions(+), 29 deletions(-) create mode 100644 examples/rand-zones.fio
---
Diff of recent changes:
HOWTO +20 -0
@@ -964,6 +964,7 @@ random_distribution=str:float By default, fio will use a completely uniform zipf Zipf distribution pareto Pareto distribution gauss Normal (guassian) distribution + zoned Zoned random distribution When using a zipf or pareto distribution, an input value is also needed to define the access pattern. For zipf, this
@@ -976,6 +977,25 @@ random_distribution=str:float By default, fio will use a completely uniform the gauss distribution, a normal deviation is supplied as a value between 0 and 100. + For a zoned distribution, fio supports specifying percentages + of IO access that should fall within what range of the file or + device. For example, given a criteria of: + + 60% of accesses should be to the first 10% + 30% of accesses should be to the next 20% + 8% of accesses should be to to the next 30% + 2% of accesses should be to the next 40% + + we can define that through zoning of the random accesses. For + the above example, the user would do: + + random_distribution=zoned:60/10:30/20:8/30:2/40 + + similarly to how bssplit works for setting ranges and + percentages of block sizes. Like bssplit, it's possible to + specify separate zones for reads, writes, and trims. If just + one set is given, it'll apply to all of them. + percentage_random=int For a random workload, set how big a percentage should be random. This defaults to 100%, in which case the workload is fully random. It can be set from anywhere from 0 to 100.
backend.c +9 -0
@@ -1711,6 +1711,15 @@ err:cgroup_shutdown (td, &cgroup_mnt);verify_free_state (td); +if (td->zone_state_index) { +int i; + +for (i =0 ; i <DDIR_RWDIR_CNT ; i++) +free (td->zone_state_index[i]); +free (td->zone_state_index); + td->zone_state_index =NULL ; + } +if (fio_option_is_set (o, cpumask)) { ret =fio_cpuset_exit (&o->cpumask);if (ret)
cconv.c +32 -0
@@ -23,6 +23,8 @@ static void __string_to_net(uint8_t *dst, const char *src, size_t dst_size)static void free_thread_options_to_cpu (struct thread_options *o) { +int i; +free (o->description);free (o->name);free (o->wait_for);
@@ -43,6 +45,11 @@ static void free_thread_options_to_cpu(struct thread_options *o)free (o->ioscheduler);free (o->profile);free (o->cgroup); + +for (i =0 ; i <DDIR_RWDIR_CNT ; i++) { +free (o->bssplit[i]); +free (o->zone_split[i]); + } } void convert_thread_options_to_cpu(struct thread_options *o,
@@ -111,6 +118,16 @@ void convert_thread_options_to_cpu(struct thread_options *o, } } + o->zone_split_nr[i] =le32_to_cpu (top->zone_split_nr[i]); + +if (o->zone_split_nr[i]) { + o->zone_split[i] =malloc (o->zone_split_nr[i] *sizeof (struct zone_split )); +for (j =0 ; j < o->zone_split_nr[i]; j++) { + o->zone_split[i][j].access_perc = top->zone_split[i][j].access_perc; + o->zone_split[i][j].size_perc = top->zone_split[i][j].size_perc; + } + } + o->rwmix[i] =le32_to_cpu (top->rwmix[i]); o->rate[i] =le32_to_cpu (top->rate[i]); o->ratemin[i] =le32_to_cpu (top->ratemin[i]);
@@ -453,6 +470,21 @@ void convert_thread_options_to_net(struct thread_options_pack *top, } } + top->zone_split_nr[i] =cpu_to_le32 (o->zone_split_nr[i]); + +if (o->zone_split_nr[i]) { +unsigned int zone_split_nr = o->zone_split_nr[i]; + +if (zone_split_nr >ZONESPLIT_MAX ) { +log_err ("fio: ZONESPLIT_MAX is too small\n" ); + zone_split_nr =ZONESPLIT_MAX ; + } +for (j =0 ; j < zone_split_nr; j++) { + top->zone_split[i][j].access_perc = o->zone_split[i][j].access_perc; + top->zone_split[i][j].size_perc = o->zone_split[i][j].size_perc; + } + } + top->rwmix[i] =cpu_to_le32 (o->rwmix[i]); top->rate[i] =cpu_to_le32 (o->rate[i]); top->ratemin[i] =cpu_to_le32 (o->ratemin[i]);
examples/rand-zones.fio new file +18 -0
@@ -0,0 +1,18 @@ +# Sample job file demonstrating how to use zoned random distributionss +# to have skewed random accesses. This example has 50% of the accesses +# to the first 5% of the file (50/5), 30% to the next 15% (30/15), and +# finally 20% of the IO will end up in the remaining 80%. +[zones] +size=2g +direct=1 +bs=4k +rw=randread +norandommap +random_distribution=zoned:50/5:30/15:20/ + +# The above applies to all of reads/writes/trims. If we wanted to do +# something differently for writes, let's say 50% for the first 10% +# and 50% for the remaining 90%, we could do it by adding a new section +# after a a comma. + +# random_distribution=zoned:50/5:30/15:20/,50/10:50/90
fio.1 +33 -1
@@ -877,8 +877,10 @@ Pareto distribution .B gauss Normal (gaussian) distribution .TP +.B zoned +Zoned random distribution +.TP .RE -.P When using a \fBzipf\fR or \fBpareto\fR distribution, an input value is also needed to define the access pattern. For \fBzipf\fR, this is the zipf theta. For \fBpareto\fR, it's the pareto power. Fio includes a test program, genzipf,
@@ -887,6 +889,36 @@ hit rates. If you wanted to use \fBzipf\fR with a theta of 1.2, you would use random_distribution=zipf:1.2 as the option. If a non-uniform model is used, fio will disable use of the random map. For the \fBgauss\fR distribution, a normal deviation is supplied as a value between 0 and 100. +.P +.RS +For a \fBzoned\fR distribution, fio supports specifying percentages of IO +access that should fall within what range of the file or device. For example, +given a criteria of: +.P +.RS +60% of accesses should be to the first 10% +.RE +.RS +30% of accesses should be to the next 20% +.RE +.RS +8% of accesses should be to to the next 30% +.RE +.RS +2% of accesses should be to the next 40% +.RE +.P +we can define that through zoning of the random accesses. For the above +example, the user would do: +.P +.RS +.B random_distribution=zoned:60/10:30/20:8/30:2/40 +.RE +.P +similarly to how \fBbssplit\fR works for setting ranges and percentages of block +sizes. Like \fBbssplit\fR, it's possible to specify separate zones for reads, +writes, and trims. If just one set is given, it'll apply to all of them. +.RE .TP .BI percentage_random \fR=\fPint For a random workload, set how big a percentage should be random. This defaults
fio.h +10 -0
@@ -96,6 +96,7 @@ enum {FIO_RAND_START_DELAY ,FIO_DEDUPE_OFF ,FIO_RAND_POISSON_OFF , +FIO_RAND_ZONE_OFF , FIO_RAND_NR_OFFS, };
@@ -115,6 +116,11 @@ struct sk_out;void sk_out_assign (struct sk_out *);void sk_out_drop (void ); +struct zone_split_index { +uint8_t size_perc; +uint8_t size_perc_prev; +}; +/* * This describes a single thread/process executing a fio job. */
@@ -200,6 +206,9 @@ struct thread_data {struct frand_state buf_state;struct frand_state buf_state_prev;struct frand_state dedupe_state; +struct frand_state zone_state; + +struct zone_split_index **zone_state_index;unsigned int verify_batch;unsigned int trim_batch;
@@ -712,6 +721,7 @@ enum {FIO_RAND_DIST_ZIPF ,FIO_RAND_DIST_PARETO ,FIO_RAND_DIST_GAUSS , + FIO_RAND_DIST_ZONED, };#define FIO_DEF_ZIPF 1.1
init.c +1 -0
@@ -968,6 +968,7 @@ void td_fill_rand_seeds(struct thread_data *td)frand_copy (&td->buf_state_prev, &td->buf_state);init_rand_seed (&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF ], use64); +init_rand_seed (&td->zone_state, td->rand_seeds[FIO_RAND_ZONE_OFF ], use64); }/*
io_u.c +82 -26
179 lines not on this page · raw message
lib/rand.h +11 -0
@@ -117,6 +117,17 @@ static inline double __rand_0_1(struct frand_state *state) } } +/* +* Generate a random value between 'start' and 'end', both inclusive +*/ +static inline int rand_between (struct frand_state *state,int start,int end) +{ +uint64_t r; + + r =__rand (state); +return start + (int ) ((double )end * (r / (rand_max (state) +1.0 ))); +} +extern void init_rand (struct frand_state *,int );extern void init_rand_seed (struct frand_state *,unsigned int seed,int );extern void __fill_random_buf (void *buf,unsigned int len,unsigned long seed);
options.c +248 -0
266 lines not on this page · raw message
server.h +1 -1
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =52 , +FIO_SERVER_VER =53 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +12 -1
@@ -25,12 +25,18 @@ enum fio_memtype {#define ERROR_STR_MAX 128#define BSSPLIT_MAX 64 +#define ZONESPLIT_MAX 64struct bssplit {uint32_t bs;uint32_t perc; }; +struct zone_split { +uint8_t access_perc; +uint8_t size_perc; +}; +#define NR_OPTS_SZ (FIO_MAX_OPTS / (8 * sizeof(uint64_t)))#define OPT_MAGIC 0x4f50544e
@@ -135,6 +141,9 @@ struct thread_options {unsigned int random_distribution;unsigned int exitall_error; +struct zone_split *zone_split[DDIR_RWDIR_CNT ]; +unsigned int zone_split_nr[DDIR_RWDIR_CNT ]; +fio_fp64_t zipf_theta;fio_fp64_t pareto_h;fio_fp64_t gauss_dev;
@@ -382,7 +391,9 @@ struct thread_options_pack {uint32_t random_distribution;uint32_t exitall_error; -uint32_t pad0; + +struct zone_split zone_split[DDIR_RWDIR_CNT ][ZONESPLIT_MAX ]; +uint32_t zone_split_nr[DDIR_RWDIR_CNT ];fio_fp64_t zipf_theta;fio_fp64_t pareto_h;