Recent changes (master)
Subsystems (2)
The following changes since commit 674428a527931d86bfb164abcc847508b3be2742:
Merge branch 'num2str-patch' of https://github.com/gloit042/fio (2021-01-09 15:28:44 -0700)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 4008b7fc8e2bff60a4e98de0005e6bc71b1a8641:
Merge branch 'zipf-pareto-lock' of https://github.com/aclamk/fio (2021-01-12 10:52:54 -0700)
----------------------------------------------------------------
Adam Kupczyk (1):
distibutions: Extend flexibility of non-uniform random distributions
Jens Axboe (3):
Merge branch 'fuzz' of https://github.com/catenacyber/fio
Merge branch 'osx_fix' of https://github.com/sitsofe/fio
Merge branch 'zipf-pareto-lock' of https://github.com/aclamk/fio
Philippe Antoine (2):
fuzz: Adds fuzz target for parse_jobs_ini
options: Fix buffer over read in fio_keyword_replace
Sitsofe Wheeler (1):
configure: fix compilation on recent macOS Xcode versions
HOWTO | 10 +++++++++- Makefile | 26 +++++++++++++++++++++++++ cconv.c | 2 ++ configure | 12 ++++++------ filesetup.c | 6 +++--- fio.1 | 10 +++++++++- fio.h | 1 + init.c | 8 +++++--- lib/gauss.c | 8 ++++++-- lib/gauss.h | 3 ++- lib/zipf.c | 12 +++++++----- lib/zipf.h | 6 ++++-- options.c | 42 ++++++++++++++++++++++++++++++++++++++--- server.h | 2 +- t/fuzz/fuzz_parseini.c | 41 ++++++++++++++++++++++++++++++++++++++++ t/fuzz/onefile.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ t/genzipf.c | 6 +++--- thread_options.h | 2 ++ 18 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 t/fuzz/fuzz_parseini.c create mode 100644 t/fuzz/onefile.c
---
Diff of recent changes:
HOWTO +9 -1
@@ -1361,7 +1361,7 @@ I/O type limit reads or writes to a certain rate. If that is the case, then the distribution may be skewed. Default: 50. -.. option:: random_distribution=str:float[,str:float][,str:float] +.. option:: random_distribution=str:float[:float][,str:float][,str:float] By default, fio will use a completely uniform random distribution when asked to perform random I/O. Sometimes it is useful to skew the distribution in
@@ -1396,6 +1396,14 @@ I/O type map. For the **normal** distribution, a normal (Gaussian) deviation is supplied as a value between 0 and 100. + The second, optional float is allowed for **pareto**, **zipf** and **normal** distributions. + It allows to set base of distribution in non-default place, giving more control + over most probable outcome. This value is in range [0-1] which maps linearly to + range of possible random values. + Defaults are: random for **pareto** and **zipf**, and 0.5 for **normal**. + If you wanted to use **zipf** with a `theta` of 1.2 centered on 1/4 of allowed value range, + you would use ``random_distibution=zipf:1.2:0.25``. + For a **zoned** distribution, fio supports specifying percentages of I/O access that should fall within what range of the file or device. For example, given a criteria of:
Makefile +26 -0
@@ -346,6 +346,23 @@ T_MEMLOCK_PROGS = t/memlock T_TT_OBJS = t/time-test.o T_TT_PROGS = t/time-test +T_FUZZ_OBJS = t/fuzz/fuzz_parseini.o +T_FUZZ_OBJS += $(OBJS) +ifdef CONFIG_ARITHMETIC +T_FUZZ_OBJS += lex.yy.o y.tab.o +endif +# in case there is no fuzz driver defined by environment variable LIB_FUZZING_ENGINE, use a simple one +# For instance, with compiler clang, address sanitizer and libFuzzer as a fuzzing engine, you should define +# export CFLAGS="-fsanitize=address,fuzzer-no-link" +# export LIB_FUZZING_ENGINE="-fsanitize=address" +# export CC=clang +# before running configure && make +# You can adapt this with different compilers, sanitizers, and fuzzing engines +ifndef LIB_FUZZING_ENGINE +T_FUZZ_OBJS += t/fuzz/onefile.o +endif +T_FUZZ_PROGS = t/fuzz/fuzz_parseini + T_OBJS = $(T_SMALLOC_OBJS) T_OBJS += $(T_IEEE_OBJS) T_OBJS += $(T_ZIPF_OBJS)
@@ -359,6 +376,7 @@ T_OBJS += $(T_PIPE_ASYNC_OBJS) T_OBJS += $(T_MEMLOCK_OBJS) T_OBJS += $(T_TT_OBJS) T_OBJS += $(T_IOU_RING_OBJS) +T_OBJS += $(T_FUZZ_OBJS) ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS))) T_DEDUPE_OBJS += $(WINDOWS_OBJS)
@@ -382,6 +400,7 @@ endif ifneq (,$(findstring Linux,$(CONFIG_TARGET_OS))) T_TEST_PROGS += $(T_IOU_RING_PROGS) endif +T_TEST_PROGS += $(T_FUZZ_PROGS) PROGS += $(T_PROGS)
@@ -533,6 +552,13 @@ t/ieee754: $(T_IEEE_OBJS) fio: $(FIO_OBJS) $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(FIO_OBJS) $(LIBS) $(HDFSLIB) +t/fuzz/fuzz_parseini: $(T_FUZZ_OBJS) +ifndef LIB_FUZZING_ENGINE + $(QUIET_LINK)$(CC) $(LDFLAGS) -o $@ $(T_FUZZ_OBJS) $(LIBS) $(HDFSLIB) +else + $(QUIET_LINK)$(CXX) $(LDFLAGS) -o $@ $(T_FUZZ_OBJS) $(LIB_FUZZING_ENGINE) $(LIBS) $(HDFSLIB) +endif + gfio: $(GFIO_OBJS) $(QUIET_LINK)$(CC) $(filter-out -static, $(LDFLAGS)) -o gfio $(GFIO_OBJS) $(LIBS) $(GFIO_LIBS) $(GTK_LDFLAGS) $(HDFSLIB)
cconv.c +2 -0
@@ -203,6 +203,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->zipf_theta.u.f =fio_uint64_to_double (le64_to_cpu (top->zipf_theta.u.i)); o->pareto_h.u.f =fio_uint64_to_double (le64_to_cpu (top->pareto_h.u.i)); o->gauss_dev.u.f =fio_uint64_to_double (le64_to_cpu (top->gauss_dev.u.i)); + o->random_center.u.f =fio_uint64_to_double (le64_to_cpu (top->random_center.u.i)); o->random_generator =le32_to_cpu (top->random_generator); o->hugepage_size =le32_to_cpu (top->hugepage_size); o->rw_min_bs =le64_to_cpu (top->rw_min_bs);
@@ -423,6 +424,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->zipf_theta.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->zipf_theta.u.f)); top->pareto_h.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->pareto_h.u.f)); top->gauss_dev.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->gauss_dev.u.f)); + top->random_center.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->random_center.u.f)); top->random_generator =cpu_to_le32 (o->random_generator); top->hugepage_size =cpu_to_le32 (o->hugepage_size); top->rw_min_bs =__cpu_to_le64 (o->rw_min_bs);
configure +6 -6
@@ -45,6 +45,7 @@ print_config() { # Default CFLAGS CFLAGS="-D_GNU_SOURCE -include config-host.h $CFLAGS" +CONFIGURE_CFLAGS="-Werror-implicit-function-declaration" BUILD_CFLAGS="" # Print a helpful header at the top of config.log
@@ -88,14 +89,14 @@ do_cc() { } compile_object() { - do_cc $CFLAGS -Werror-implicit-function-declaration -c -o $TMPO $TMPC + do_cc $CFLAGS $CONFIGURE_CFLAGS -c -o $TMPO $TMPC } compile_prog() { local_cflags="$1" local_ldflags="$2 $LIBS" echo "Compiling test case $3" >> config.log - do_cc $CFLAGS -Werror-implicit-function-declaration $local_cflags -o $TMPE $TMPC $LDFLAGS $local_ldflags + do_cc $CFLAGS $CONFIGURE_CFLAGS $local_cflags -o $TMPE $TMPC $LDFLAGS $local_ldflags } feature_not_found() {
@@ -360,16 +361,15 @@ Darwin) if test -z "$cpu" && test "$(sysctl -n hw.optional.x86_64)" = "1"; then cpu="x86_64" fi - # Error at compile time linking of weak/partial symbols if possible... + # Avoid configure feature detection of features provided by weak symbols cat > $TMPC <<EOF int main(void) { return 0; } EOF - if compile_prog "" "-Wl,-no_weak_imports" "disable weak symbols"; then - echo "Disabling weak symbols" - LDFLAGS="$LDFLAGS -Wl,-no_weak_imports" + if compile_prog "" "-Werror=partial-availability" "error on weak symbols"; then + CONFIGURE_CFLAGS="$CONFIGURE_CFLAGS -Werror=partial-availability" fi ;; SunOS)
filesetup.c +3 -3
@@ -1319,11 +1319,11 @@ static void __init_rand_distribution(struct thread_data *td, struct fio_file *f) seed = td->rand_seeds[4 ];if (td->o.random_distribution ==FIO_RAND_DIST_ZIPF ) -zipf_init (&f->zipf, nranges, td->o.zipf_theta.u.f, seed); +zipf_init (&f->zipf, nranges, td->o.zipf_theta.u.f, td->o.random_center.u.f, seed);else if (td->o.random_distribution ==FIO_RAND_DIST_PARETO ) -pareto_init (&f->zipf, nranges, td->o.pareto_h.u.f, seed); +pareto_init (&f->zipf, nranges, td->o.pareto_h.u.f, td->o.random_center.u.f, seed);else if (td->o.random_distribution ==FIO_RAND_DIST_GAUSS ) -gauss_init (&f->gauss, nranges, td->o.gauss_dev.u.f, seed); +gauss_init (&f->gauss, nranges, td->o.gauss_dev.u.f, td->o.random_center.u.f, seed); }static bool init_rand_distribution (struct thread_data *td)
fio.1 +9 -1
@@ -1132,7 +1132,7 @@ first. This may interfere with a given rate setting, if fio is asked to limit reads or writes to a certain rate. If that is the case, then the distribution may be skewed. Default: 50. .TP -.BI random_distribution \fR=\fPstr:float[,str:float][,str:float] +.BI random_distribution \fR=\fPstr:float[:float][,str:float][,str:float] By default, fio will use a completely uniform random distribution when asked to perform random I/O. Sometimes it is useful to skew the distribution in specific ways, ensuring that some parts of the data is more hot than others.
@@ -1168,6 +1168,14 @@ option. If a non\-uniform model is used, fio will disable use of the random map. For the \fBnormal\fR distribution, a normal (Gaussian) deviation is supplied as a value between 0 and 100. .P +The second, optional float is allowed for \fBpareto\fR, \fBzipf\fR and \fBnormal\fR +distributions. It allows to set base of distribution in non-default place, giving +more control over most probable outcome. This value is in range [0-1] which maps linearly to +range of possible random values. +Defaults are: random for \fBpareto\fR and \fBzipf\fR, and 0.5 for \fBnormal\fR. +If you wanted to use \fBzipf\fR with a `theta` of 1.2 centered on 1/4 of allowed value range, +you would use `random_distibution=zipf:1.2:0.25`. +.P For a \fBzoned\fR distribution, fio supports specifying percentages of I/O access that should fall within what range of the file or device. For example, given a criteria of:
fio.h +1 -0
@@ -229,6 +229,7 @@ struct thread_data {double pareto_h;double gauss_dev; }; +double random_center;int error;int sig;int done;
init.c +5 -3
@@ -327,6 +327,7 @@ void free_threads_shm(void)static void free_shm (void ) { +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (nr_segments) {flow_exit (); fio_debug_jobp =NULL ;
@@ -343,6 +344,7 @@ static void free_shm(void)fio_filelock_exit ();file_hash_exit ();scleanup (); +#endif }static int add_thread_segment (void )
@@ -971,13 +973,13 @@ static void init_rand_file_service(struct thread_data *td)const unsigned int seed = td->rand_seeds[FIO_RAND_FILE_OFF ];if (td->o.file_service_type ==FIO_FSERVICE_ZIPF ) { -zipf_init (&td->next_file_zipf, nranges, td->zipf_theta, seed); +zipf_init (&td->next_file_zipf, nranges, td->zipf_theta, td->random_center, seed);zipf_disable_hash (&td->next_file_zipf); }else if (td->o.file_service_type ==FIO_FSERVICE_PARETO ) { -pareto_init (&td->next_file_zipf, nranges, td->pareto_h, seed); +pareto_init (&td->next_file_zipf, nranges, td->pareto_h, td->random_center, seed);zipf_disable_hash (&td->next_file_zipf); }else if (td->o.file_service_type ==FIO_FSERVICE_GAUSS ) { -gauss_init (&td->next_file_gauss, nranges, td->gauss_dev, seed); +gauss_init (&td->next_file_gauss, nranges, td->gauss_dev, td->random_center, seed);gauss_disable_hash (&td->next_file_gauss); } }
lib/gauss.c +6 -2
@@ -40,11 +40,11 @@ unsigned long long gauss_next(struct gauss_state *gs)if (!gs->disable_hash) sum =__hash_u64 (sum); -return sum % gs->nranges; +return (sum + gs->rand_off) % gs->nranges; }void gauss_init (struct gauss_state *gs,unsigned long nranges,double dev, -unsigned int seed) +double center,unsigned int seed) {memset (gs,0 ,sizeof (*gs));init_rand_seed (&gs->r, seed,0 );
@@ -55,6 +55,10 @@ void gauss_init(struct gauss_state *gs, unsigned long nranges, double dev,if (gs->stddev > nranges /2 ) gs->stddev = nranges /2 ; } +if (center ==-1 ) + gs->rand_off =0 ; +else + gs->rand_off = nranges * (center -0.5 ); }void gauss_disable_hash (struct gauss_state *gs)
lib/gauss.h +2 -1
@@ -8,11 +8,12 @@ struct gauss_state {struct frand_state r;uint64_t nranges;unsigned int stddev; +unsigned int rand_off;bool disable_hash; };void gauss_init (struct gauss_state *gs,unsigned long nranges,double dev, -unsigned int seed); +double center,unsigned int seed);unsigned long long gauss_next (struct gauss_state *gs);void gauss_disable_hash (struct gauss_state *gs);
lib/zipf.c +7 -5
@@ -23,19 +23,21 @@ static void zipf_update(struct zipf_state *zs) }static void shared_rand_init (struct zipf_state *zs,uint64_t nranges, -unsigned int seed) +double center,unsigned int seed) {memset (zs,0 ,sizeof (*zs)); zs->nranges = nranges;init_rand_seed (&zs->rand, seed,0 ); zs->rand_off =__rand (&zs->rand); +if (center !=-1 ) + zs->rand_off = nranges * center; }void zipf_init (struct zipf_state *zs,uint64_t nranges,double theta, -unsigned int seed) +double center,unsigned int seed) { -shared_rand_init (zs, nranges, seed); +shared_rand_init (zs, nranges, center, seed); zs->theta = theta; zs->zeta2 =pow (1.0 , zs->theta) +pow (0.5 , zs->theta);
@@ -71,9 +73,9 @@ uint64_t zipf_next(struct zipf_state *zs) }void pareto_init (struct zipf_state *zs,uint64_t nranges,double h, -unsigned int seed) +double center,unsigned int seed) { -shared_rand_init (zs, nranges, seed); +shared_rand_init (zs, nranges, center, seed); zs->pareto_pow =log (h) /log (1.0 - h); }
lib/zipf.h +4 -2
@@ -16,10 +16,12 @@ struct zipf_state {bool disable_hash; }; -void zipf_init (struct zipf_state *zs,uint64_t nranges,double theta,unsigned int seed); +void zipf_init (struct zipf_state *zs,uint64_t nranges,double theta, +double center,unsigned int seed);uint64_t zipf_next (struct zipf_state *zs); -void pareto_init (struct zipf_state *zs,uint64_t nranges,double h,unsigned int seed); +void pareto_init (struct zipf_state *zs,uint64_t nranges,double h, +double center,unsigned int seed);uint64_t pareto_next (struct zipf_state *zs);void zipf_disable_hash (struct zipf_state *zs);
options.c +39 -3
96 lines not on this page · raw message
server.h +1 -1
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =86 , +FIO_SERVER_VER =87 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
t/fuzz/fuzz_parseini.c new file +41 -0
41 lines not on this page · raw message
t/fuzz/onefile.c new file +51 -0
51 lines not on this page · raw message
t/genzipf.c +3 -3
@@ -297,11 +297,11 @@ int main(int argc, char *argv[]) nranges /= block_size;if (dist_type ==TYPE_ZIPF ) -zipf_init (&zs, nranges, dist_val,1 ); +zipf_init (&zs, nranges, dist_val,-1 ,1 );else if (dist_type ==TYPE_PARETO ) -pareto_init (&zs, nranges, dist_val,1 ); +pareto_init (&zs, nranges, dist_val,-1 ,1 );else -gauss_init (&gs, nranges, dist_val,1 ); +gauss_init (&gs, nranges, dist_val,-1 ,1 ); hash_bits =0 ; hash_size = nranges;
thread_options.h +2 -0
@@ -166,6 +166,7 @@ struct thread_options {fio_fp64_t zipf_theta;fio_fp64_t pareto_h;fio_fp64_t gauss_dev; +fio_fp64_t random_center;unsigned int random_generator;
@@ -467,6 +468,7 @@ struct thread_options_pack {fio_fp64_t zipf_theta;fio_fp64_t pareto_h;fio_fp64_t gauss_dev; +fio_fp64_t random_center;uint32_t random_generator;