Recent changes (master)
The following changes since commit 07ed2b57741afa53afa7b2b9fa742c652f1ed8c1:
Merge branch 'libaio-hang' of https://github.com/lrumancik/fio (2023-04-10 15:40:45 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 7624d58953d38612c11496551a855a1aeee7ad24:
docs: update documentation for randrepeat and allrandrepeat (2023-04-13 13:38:52 -0400)
----------------------------------------------------------------
Vincent Fu (7):
rand: print out random seeds for debugging
init: refactor random seed setting
init: get rid of td_fill_rand_seeds_internal
init: clean up random seed options
t/random_seed: python script to test random seed options
test: improve evaluation of t0020.fio and t0021.fio
docs: update documentation for randrepeat and allrandrepeat
Xiaoguang Wang (1):
t/io_uring: fix max_blocks calculation in nvme passthrough mode
HOWTO.rst | 7 +- cconv.c | 2 - ci/actions-install.sh | 3 +- ci/appveyor-install.sh | 2 +- fio.1 | 7 +- fio.h | 1 - init.c | 91 ++++-------- options.c | 11 +- server.h | 2 +- t/io_uring.c | 2 +- t/random_seed.py | 394 +++++++++++++++++++++++++++++++++++++++++++++++++ t/run-fio-tests.py | 26 ++-- thread_options.h | 3 - 13 files changed, 454 insertions(+), 97 deletions(-) create mode 100755 t/random_seed.py
---
Diff of recent changes:
HOWTO.rst +3 -4
@@ -1232,13 +1232,12 @@ I/O type .. option:: randrepeat=bool - Seed the random number generator used for random I/O patterns in a - predictable way so the pattern is repeatable across runs. Default: true. + Seed all random number generators in a predictable way so the pattern + is repeatable across runs. Default: true. .. option:: allrandrepeat=bool - Seed all random number generators in a predictable way so results are - repeatable across runs. Default: false. + Alias for :option:`randrepeat`. Default: true. .. option:: randseed=int
cconv.c +0 -2
@@ -206,7 +206,6 @@ int convert_thread_options_to_cpu(struct thread_options *o, o->do_disk_util =le32_to_cpu (top->do_disk_util); o->override_sync =le32_to_cpu (top->override_sync); o->rand_repeatable =le32_to_cpu (top->rand_repeatable); - o->allrand_repeatable =le32_to_cpu (top->allrand_repeatable); o->rand_seed =le64_to_cpu (top->rand_seed); o->log_entries =le32_to_cpu (top->log_entries); o->log_avg_msec =le32_to_cpu (top->log_avg_msec);
@@ -446,7 +445,6 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->do_disk_util =cpu_to_le32 (o->do_disk_util); top->override_sync =cpu_to_le32 (o->override_sync); top->rand_repeatable =cpu_to_le32 (o->rand_repeatable); - top->allrand_repeatable =cpu_to_le32 (o->allrand_repeatable); top->rand_seed =__cpu_to_le64 (o->rand_seed); top->log_entries =cpu_to_le32 (o->log_entries); top->log_avg_msec =cpu_to_le32 (o->log_avg_msec);
ci/actions-install.sh +2 -1
@@ -62,6 +62,7 @@ DPKGCFG pkgs+=( python3-scipy python3-sphinx + python3-statsmodels ) echo "Updating APT..."
@@ -85,7 +86,7 @@ install_macos() { echo "Installing packages..." HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs sphinx-doc brew link sphinx-doc --force - pip3 install scipy six + pip3 install scipy six statsmodels } main() {
ci/appveyor-install.sh +1 -1
@@ -37,7 +37,7 @@ case "${DISTRO}" in ;; esac -python.exe -m pip install scipy six +python.exe -m pip install scipy six statsmodels echo "Python3 path: $(type -p python3 2>&1)" echo "Python3 version: $(python3 -V 2>&1)"
fio.1 +3 -4
@@ -1022,12 +1022,11 @@ Alias for \fBboth\fR. .RE .TP .BI randrepeat \fR=\fPbool -Seed the random number generator used for random I/O patterns in a -predictable way so the pattern is repeatable across runs. Default: true. +Seed all random number generators in a predictable way so the pattern is +repeatable across runs. Default: true. .TP .BI allrandrepeat \fR=\fPbool -Seed all random number generators in a predictable way so results are -repeatable across runs. Default: false. +Alias for \fBrandrepeat\fR. Default: true. .TP .BI randseed \fR=\fPint Seed the random number generators based on this seed value, to be able to
fio.h +0 -1
@@ -638,7 +638,6 @@ extern void fio_options_dup_and_init(struct option *);extern char *fio_option_dup_subs (const char *);extern void fio_options_mem_dupe (struct thread_data *);extern void td_fill_rand_seeds (struct thread_data *); -extern void td_fill_verify_state_seed (struct thread_data *);extern void add_job_opts (const char **,int );extern int ioengine_load (struct thread_data *);extern bool parse_dryrun (void );
init.c +32 -59
136 lines not on this page · raw message
options.c +1 -10
@@ -2465,6 +2465,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { }, { .name = "randrepeat", + .alias = "allrandrepeat", .lname = "Random repeatable", .type = FIO_OPT_BOOL, .off1 = offsetof(struct thread_options, rand_repeatable),
@@ -2594,16 +2595,6 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_IO, .group = FIO_OPT_G_RANDOM, }, - { - .name = "allrandrepeat", - .lname = "All Random Repeat", - .type = FIO_OPT_BOOL, - .off1 = offsetof(struct thread_options, allrand_repeatable), - .help = "Use repeatable random numbers for everything", - .def = "0", - .category = FIO_OPT_C_IO, - .group = FIO_OPT_G_RANDOM, - }, { .name = "nrfiles", .lname = "Number of files",
server.h +1 -1
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =99 , +FIO_SERVER_VER =100 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
t/io_uring.c +1 -1
@@ -704,7 +704,7 @@ static int get_file_size(struct file *f) bs, lbs);return -1 ; } - f->max_blocks = nlba / bs; + f->max_blocks = nlba; f->max_size = nlba; f->lba_shift =ilog2 (lbs);return 0 ;
t/random_seed.py new file +394 -0
394 lines not on this page · raw message
t/run-fio-tests.py +16 -10
@@ -53,6 +53,7 @@ import traceback import subprocess import multiprocessing from pathlib import Path +from statsmodels.sandbox.stats.runs import runstest_1samp class FioTest():
@@ -598,24 +599,16 @@ class FioJobTest_t0020(FioJobTest): log_lines = file_data.split('\n') - seq_count = 0 - offsets = set() + offsets = [] prev = int(log_lines[0].split(',')[4]) for line in log_lines[1:]: - offsets.add(prev/4096) + offsets.append(prev/4096) if len(line.strip()) == 0: continue cur = int(line.split(',')[4]) - if cur - prev == 4096: - seq_count += 1 prev = cur - # 10 is an arbitrary threshold - if seq_count > 10: - self.passed = False - self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count) - if len(offsets) != 256: self.passed = False self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
@@ -625,6 +618,11 @@ class FioJobTest_t0020(FioJobTest): self.passed = False self.failure_reason += " missing offset {0}".format(i*4096) + (z, p) = runstest_1samp(list(offsets)) + if p < 0.05: + self.passed = False + self.failure_reason += f" runs test failed with p = {p}" + class FioJobTest_t0022(FioJobTest): """Test consists of fio test job t0022"""
@@ -1361,6 +1359,14 @@ TEST_LIST = [ 'success': SUCCESS_DEFAULT, 'requirements': [], }, + { + 'test_id': 1013, + 'test_class': FioExeTest, + 'exe': 't/random_seed.py', + 'parameters': ['-f', '{fio_path}'], + 'success': SUCCESS_DEFAULT, + 'requirements': [], + }, ]
thread_options.h +0 -3
@@ -162,7 +162,6 @@ struct thread_options {unsigned int do_disk_util;unsigned int override_sync;unsigned int rand_repeatable; -unsigned int allrand_repeatable;unsigned long long rand_seed;unsigned int log_avg_msec;unsigned int log_hist_msec;
@@ -485,8 +484,6 @@ struct thread_options_pack {uint32_t do_disk_util;uint32_t override_sync;uint32_t rand_repeatable; -uint32_t allrand_repeatable; -uint32_t pad2;uint64_t rand_seed;uint32_t log_avg_msec;uint32_t log_hist_msec;