Recent changes (master)
The following changes since commit 5c8f0ba56837a0b848cbbbc5a8673589d099ded3:
verify: unroll string copy (2016-05-10 19:50:00 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 15a0c8ee4e1a5434075ebc2c9f48e96e5e892196:
Windows crash in ctime_r() (2016-05-16 19:25:48 -0600)
----------------------------------------------------------------
Jens Axboe (5):
stat: add blocksize to averaged log, if it's consistent zipf/pareto/gauss: add option to disable hashing Add support for non-uniformly random file service type
options: 0.00 is a valid gauss dev zipf/pareto/gauss: hash cleanup
Michael Schoberg (mschoberg) (1):
Windows crash in ctime_r()
HOWTO | 21 ++++++++++--- file.h | 17 +++++++---- fio.1 | 18 +++++++++-- fio.h | 9 ++++++ init.c | 16 ++++++++++ io_u.c | 47 +++++++++++++++++++++++------ lib/gauss.c | 10 ++++++- lib/gauss.h | 2 ++ lib/zipf.c | 21 +++++++++++-- lib/zipf.h | 2 ++ options.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++---- os/windows/posix.c | 6 ++-- stat.c | 20 ++++++++++--- 13 files changed, 240 insertions(+), 36 deletions(-)
---
Diff of recent changes:
HOWTO +17 -4
@@ -673,10 +673,23 @@ file_service_type=str Defines how fio decides which file from a job to the next. Multiple files can still be open depending on 'openfiles'. - The string can have a number appended, indicating how - often to switch to a new file. So if option random:4 is - given, fio will switch to a new random file after 4 ios - have been issued. + zipf Use a zipfian distribution to decide what file + to access. + + pareto Use a pareto distribution to decide what file + to access. + + gauss Use a gaussian (normal) distribution to decide + what file to access. + + For random, roundrobin, and sequential, a postfix can be + appended to tell fio how many I/Os to issue before switching + to a new file. For example, specifying + 'file_service_type=random:8' would cause fio to issue 8 I/Os + before selecting a new file at random. For the non-uniform + distributions, a floating point postfix can be given to + influence how the distribution is skewed. See + 'random_distribution' for a description of how that would work. ioengine=str Defines how the job issues io to the file. The following types are defined:
file.h +12 -5
@@ -39,13 +39,20 @@ enum file_lock_mode { };/* -* roundrobin available files, or choose one at random, or do each one -* serially. +* How fio chooses what file to service next. Choice of uniformly random, or +* some skewed random variants, or just sequentially go through them or +* roundrobing. */ enum { -FIO_FSERVICE_RANDOM =1 , -FIO_FSERVICE_RR =2 , -FIO_FSERVICE_SEQ =3 , +FIO_FSERVICE_RANDOM =1 , +FIO_FSERVICE_RR =2 , +FIO_FSERVICE_SEQ =3 , + __FIO_FSERVICE_NONUNIFORM =0x100 , +FIO_FSERVICE_ZIPF = __FIO_FSERVICE_NONUNIFORM |4 , +FIO_FSERVICE_PARETO = __FIO_FSERVICE_NONUNIFORM |5 , +FIO_FSERVICE_GAUSS = __FIO_FSERVICE_NONUNIFORM |6 , + +FIO_FSERVICE_SHIFT =10 , };/*
fio.1 +16 -2
@@ -566,10 +566,24 @@ Round robin over opened files (default). .TP .B sequential Do each file in the set sequentially. +.TP +.B zipf +Use a zipfian distribution to decide what file to access. +.TP +.B pareto +Use a pareto distribution to decide what file to access. +.TP +.B gauss +Use a gaussian (normal) distribution to decide what file to access. .RE .P -The number of I/Os to issue before switching to a new file can be specified by -appending `:\fIint\fR' to the service type. +For \fBrandom\fR, \fBroundrobin\fR, and \fBsequential\fR, a postfix can be +appended to tell fio how many I/Os to issue before switching to a new file. +For example, specifying \fBfile_service_type=random:8\fR would cause fio to +issue \fI8\fR I/Os before selecting a new file at random. For the non-uniform +distributions, a floating point postfix can be given to influence how the +distribution is skewed. See \fBrandom_distribution\fR for a description of how +that would work. .RE .TP .BI ioengine \fR=\fPstr
fio.h +9 -0
@@ -170,6 +170,15 @@ struct thread_data {unsigned int next_file;struct frand_state next_file_state; }; +union { +struct zipf_state next_file_zipf; +struct gauss_state next_file_gauss; + }; +union { +double zipf_theta; +double pareto_h; +double gauss_dev; + };int error;int sig;int done;
init.c +16 -0
@@ -929,6 +929,22 @@ static void td_fill_rand_seeds_internal(struct thread_data *td, bool use64)if (td->o.file_service_type ==FIO_FSERVICE_RANDOM )init_rand_seed (&td->next_file_state, td->rand_seeds[FIO_RAND_FILE_OFF ], use64); +else if (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM) { +unsigned long nranges; + + nranges = td->o.nr_files <<FIO_FSERVICE_SHIFT ; + +if (td->o.file_service_type ==FIO_FSERVICE_ZIPF ) { +zipf_init (&td->next_file_zipf, nranges, td->zipf_theta, td->rand_seeds[FIO_RAND_FILE_OFF ]); +zipf_disable_hash (&td->next_file_zipf); + }else if (td->o.file_service_type ==FIO_FSERVICE_PARETO ) { +pareto_init (&td->next_file_zipf, nranges, td->pareto_h, td->rand_seeds[FIO_RAND_FILE_OFF ]); +zipf_disable_hash (&td->next_file_zipf); + }else if (td->o.file_service_type ==FIO_FSERVICE_GAUSS ) { +gauss_init (&td->next_file_gauss, nranges, td->gauss_dev, td->rand_seeds[FIO_RAND_FILE_OFF ]); +gauss_disable_hash (&td->next_file_gauss); + } + }init_rand_seed (&td->file_size_state, td->rand_seeds[FIO_RAND_FILE_SIZE_OFF ], use64);init_rand_seed (&td->trim_state, td->rand_seeds[FIO_RAND_TRIM_OFF ], use64);
io_u.c +38 -9
78 lines not on this page · raw message
lib/gauss.c +9 -1
@@ -38,7 +38,10 @@ unsigned long long gauss_next(struct gauss_state *gs) sum += dev; } -return __hash_u64 (sum) % gs->nranges; +if (!gs->disable_hash) + sum =__hash_u64 (sum); + +return sum % gs->nranges; } void gauss_init(struct gauss_state *gs, unsigned long nranges, double dev,
@@ -54,3 +57,8 @@ void gauss_init(struct gauss_state *gs, unsigned long nranges, double dev, gs->stddev = nranges /2 ; } } + +void gauss_disable_hash (struct gauss_state *gs) +{ + gs->disable_hash = true; +}
lib/gauss.h +2 -0
@@ -8,10 +8,12 @@ struct gauss_state {struct frand_state r;uint64_t nranges;unsigned int stddev; + bool disable_hash; };void gauss_init (struct gauss_state *gs,unsigned long nranges,double dev,unsigned int seed);unsigned long long gauss_next (struct gauss_state *gs); +void gauss_disable_hash (struct gauss_state *gs);#endif
lib/zipf.c +18 -3
@@ -69,7 +69,12 @@ unsigned long long zipf_next(struct zipf_state *zs)else val =1 + (unsigned long long )(n *pow (eta*rand_uni - eta +1.0 , alpha)); -return (__hash_u64 (val -1 ) + zs->rand_off) % zs->nranges; + val--; + +if (!zs->disable_hash) + val =__hash_u64 (val); + +return (val + zs->rand_off) % zs->nranges; } void pareto_init(struct zipf_state *zs, unsigned long nranges, double h,
@@ -82,7 +87,17 @@ void pareto_init(struct zipf_state *zs, unsigned long nranges, double h,unsigned long long pareto_next (struct zipf_state *zs) {double rand = (double )__rand (&zs->rand) / (double )FRAND32_MAX ; -unsigned long long n = zs->nranges -1 ; +unsigned long long n; + + n = (zs->nranges -1 ) *pow (rand, zs->pareto_pow); + +if (!zs->disable_hash) + n =__hash_u64 (n); -return (__hash_u64 (n *pow (rand, zs->pareto_pow)) + zs->rand_off) % zs->nranges; +return (n + zs->rand_off) % zs->nranges; +} + +void zipf_disable_hash (struct zipf_state *zs) +{ + zs->disable_hash = true; }
lib/zipf.h +2 -0
@@ -12,6 +12,7 @@ struct zipf_state {double pareto_pow;struct frand_state rand;uint64_t rand_off; +bool disable_hash; };void zipf_init (struct zipf_state *zs,unsigned long nranges,double theta,unsigned int seed);
@@ -19,5 +20,6 @@ unsigned long long zipf_next(struct zipf_state *zs);void pareto_init (struct zipf_state *zs,unsigned long nranges,double h,unsigned int seed);unsigned long long pareto_next (struct zipf_state *zs); +void zipf_disable_hash (struct zipf_state *zs);#endif
options.c +82 -5
108 lines not on this page · raw message
os/windows/posix.c +3 -3
@@ -243,12 +243,12 @@ void Time_tToSystemTime(time_t dosTime, SYSTEMTIME *systemTime)char *ctime_r (const time_t *t,char *buf) {SYSTEMTIME systime; -const char *const dayOfWeek[] = {"Mon" ,"Tue" ,"Wed" ,"Thu" ,"Fri" ,"Sat" ,"Sun" }; +const char *const dayOfWeek[] = {"Sun" ,"Mon" ,"Tue" ,"Wed" ,"Thu" ,"Fri" ,"Sat" };const char *const monthOfYear[] = {"Jan" ,"Feb" ,"Mar" ,"Apr" ,"May" ,"Jun" ,"Jul" ,"Aug" ,"Sep" ,"Oct" ,"Nov" ,"Dec" };Time_tToSystemTime (*t, &systime);/* We don't know how long `buf` is, but assume it's rounded up from the minimum of 25 to 32 */ -StringCchPrintfA (buf,32 ,"%s %s %d %02d:%02d:%02d %04d" , dayOfWeek[systime.wDayOfWeek -1 ], monthOfYear[systime.wMonth -1 ], +StringCchPrintfA (buf,31 ,"%s %s %d %02d:%02d:%02d %04d" , dayOfWeek[systime.wDayOfWeek %7 ], monthOfYear[(systime.wMonth -1 ) %12 ], systime.wDay, systime.wHour, systime.wMinute, systime.wSecond, systime.wYear);return buf; }
@@ -888,7 +888,7 @@ struct dirent *readdir(DIR *dirp)if (dirp->find_handle ==INVALID_HANDLE_VALUE ) {char search_pattern[MAX_PATH ]; -StringCchPrintfA (search_pattern,MAX_PATH ,"%s\\*" , dirp->dirname); +StringCchPrintfA (search_pattern,MAX_PATH -1 ,"%s\\*" , dirp->dirname); dirp->find_handle =FindFirstFileA (search_pattern, &find_data);if (dirp->find_handle ==INVALID_HANDLE_VALUE )return NULL ;
stat.c +16 -4
@@ -2169,8 +2169,14 @@ static int add_bw_samples(struct thread_data *td, struct timeval *t)add_stat_sample (&ts->bw_stat[ddir], rate); -if (td->bw_log) -add_log_sample (td, td->bw_log, rate, ddir,0 ,0 ); +if (td->bw_log) { +unsigned int bs =0 ; + +if (td->o.min_bs[ddir] == td->o.max_bs[ddir]) + bs = td->o.min_bs[ddir]; + +add_log_sample (td, td->bw_log, rate, ddir, bs,0 ); + } td->stat_io_bytes[ddir] = td->this_io_bytes[ddir]; }
@@ -2234,8 +2240,14 @@ static int add_iops_samples(struct thread_data *td, struct timeval *t)add_stat_sample (&ts->iops_stat[ddir], iops); -if (td->iops_log) -add_log_sample (td, td->iops_log, iops, ddir,0 ,0 ); +if (td->iops_log) { +unsigned int bs =0 ; + +if (td->o.min_bs[ddir] == td->o.max_bs[ddir]) + bs = td->o.min_bs[ddir]; + +add_log_sample (td, td->iops_log, iops, ddir, bs,0 ); + } td->stat_io_blocks[ddir] = td->this_io_blocks[ddir]; }