Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Apr 14, 2023

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 07ed2b57741afa53afa7b2b9fa742c652f1ed8c1:

Merge branch 'libaio-hang' of https://github.com/lrumancik/fio (2023-04-10 15:40:45 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 7624d58953d38612c11496551a855a1aeee7ad24:

docs: update documentation for randrepeat and allrandrepeat (2023-04-13 13:38:52 -0400)

----------------------------------------------------------------
Vincent Fu (7):
rand: print out random seeds for debugging
init: refactor random seed setting
init: get rid of td_fill_rand_seeds_internal
init: clean up random seed options
t/random_seed: python script to test random seed options
test: improve evaluation of t0020.fio and t0021.fio
docs: update documentation for randrepeat and allrandrepeat

Xiaoguang Wang (1):
t/io_uring: fix max_blocks calculation in nvme passthrough mode

 HOWTO.rst              |   7 +-
 cconv.c                |   2 -
 ci/actions-install.sh  |   3 +-
 ci/appveyor-install.sh |   2 +-
 fio.1                  |   7 +-
 fio.h                  |   1 -
 init.c                 |  91 ++++--------
 options.c              |  11 +-
 server.h               |   2 +-
 t/io_uring.c           |   2 +-
 t/random_seed.py       | 394 +++++++++++++++++++++++++++++++++++++++++++++++++
 t/run-fio-tests.py     |  26 ++--
 thread_options.h       |   3 -
 13 files changed, 454 insertions(+), 97 deletions(-)
 create mode 100755 t/random_seed.py

---

Diff of recent changes:

HOWTO.rst +3 -4
index cb0f9834..0a6e60c7 100644
@@ -1232,13 +1232,12 @@ I/O type
 
 .. option:: randrepeat=bool
 
-	Seed the random number generator used for random I/O patterns in a
-	predictable way so the pattern is repeatable across runs. Default: true.
+        Seed all random number generators in a predictable way so the pattern
+        is repeatable across runs. Default: true.
 
 .. option:: allrandrepeat=bool
 
-	Seed all random number generators in a predictable way so results are
-	repeatable across runs.  Default: false.
+	Alias for :option:`randrepeat`. Default: true.
 
 .. option:: randseed=int
 
cconv.c +0 -2
index 1ae38b1b..9095d519 100644
@@ -206,7 +206,6 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->do_disk_util = le32_to_cpu(top->do_disk_util);
 	o->override_sync = le32_to_cpu(top->override_sync);
 	o->rand_repeatable = le32_to_cpu(top->rand_repeatable);
-	o->allrand_repeatable = le32_to_cpu(top->allrand_repeatable);
 	o->rand_seed = le64_to_cpu(top->rand_seed);
 	o->log_entries = le32_to_cpu(top->log_entries);
 	o->log_avg_msec = le32_to_cpu(top->log_avg_msec);
@@ -446,7 +445,6 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->do_disk_util = cpu_to_le32(o->do_disk_util);
 	top->override_sync = cpu_to_le32(o->override_sync);
 	top->rand_repeatable = cpu_to_le32(o->rand_repeatable);
-	top->allrand_repeatable = cpu_to_le32(o->allrand_repeatable);
 	top->rand_seed = __cpu_to_le64(o->rand_seed);
 	top->log_entries = cpu_to_le32(o->log_entries);
 	top->log_avg_msec = cpu_to_le32(o->log_avg_msec);
ci/actions-install.sh +2 -1
index 5057fca3..fb3bd141 100755
@@ -62,6 +62,7 @@ DPKGCFG
     pkgs+=(
         python3-scipy
 	python3-sphinx
+	python3-statsmodels
     )
 
     echo "Updating APT..."
@@ -85,7 +86,7 @@ install_macos() {
     echo "Installing packages..."
     HOMEBREW_NO_AUTO_UPDATE=1 brew install cunit libnfs sphinx-doc
     brew link sphinx-doc --force
-    pip3 install scipy six 
+    pip3 install scipy six statsmodels
 }
 
 main() {
ci/appveyor-install.sh +1 -1
index 3137f39e..1e28c454 100755
@@ -37,7 +37,7 @@ case "${DISTRO}" in
         ;;
 esac
 
-python.exe -m pip install scipy six
+python.exe -m pip install scipy six statsmodels
 
 echo "Python3 path: $(type -p python3 2>&1)"
 echo "Python3 version: $(python3 -V 2>&1)"
fio.1 +3 -4
index 311b16d8..4207814b 100644
@@ -1022,12 +1022,11 @@ Alias for \fBboth\fR.
 .RE
 .TP
 .BI randrepeat \fR=\fPbool
-Seed the random number generator used for random I/O patterns in a
-predictable way so the pattern is repeatable across runs. Default: true.
+Seed all random number generators in a predictable way so the pattern is
+repeatable across runs. Default: true.
 .TP
 .BI allrandrepeat \fR=\fPbool
-Seed all random number generators in a predictable way so results are
-repeatable across runs. Default: false.
+Alias for \fBrandrepeat\fR. Default: true.
 .TP
 .BI randseed \fR=\fPint
 Seed the random number generators based on this seed value, to be able to
fio.h +0 -1
index 6b841e9c..6fc7fb9c 100644
@@ -638,7 +638,6 @@ extern void fio_options_dup_and_init(struct option *);
 extern char *fio_option_dup_subs(const char *);
 extern void fio_options_mem_dupe(struct thread_data *);
 extern void td_fill_rand_seeds(struct thread_data *);
-extern void td_fill_verify_state_seed(struct thread_data *);
 extern void add_job_opts(const char **, int);
 extern int ioengine_load(struct thread_data *);
 extern bool parse_dryrun(void);
init.c +32 -59

136 lines not on this page · raw message

options.c +1 -10
index 440bff37..8193fb29 100644
@@ -2465,6 +2465,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 	},
 	{
 		.name	= "randrepeat",
+		.alias	= "allrandrepeat",
 		.lname	= "Random repeatable",
 		.type	= FIO_OPT_BOOL,
 		.off1	= offsetof(struct thread_options, rand_repeatable),
@@ -2594,16 +2595,6 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_IO,
 		.group	= FIO_OPT_G_RANDOM,
 	},
-	{
-		.name	= "allrandrepeat",
-		.lname	= "All Random Repeat",
-		.type	= FIO_OPT_BOOL,
-		.off1	= offsetof(struct thread_options, allrand_repeatable),
-		.help	= "Use repeatable random numbers for everything",
-		.def	= "0",
-		.category = FIO_OPT_C_IO,
-		.group	= FIO_OPT_G_RANDOM,
-	},
 	{
 		.name	= "nrfiles",
 		.lname	= "Number of files",
server.h +1 -1
index 898a893d..601d3340 100644
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 99,
+	FIO_SERVER_VER			= 100,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
t/io_uring.c +1 -1
index 504f8ce9..f9f4b840 100644
@@ -704,7 +704,7 @@ static int get_file_size(struct file *f)
 					bs, lbs);
 			return -1;
 		}
-		f->max_blocks = nlba / bs;
+		f->max_blocks = nlba;
 		f->max_size = nlba;
 		f->lba_shift = ilog2(lbs);
 		return 0;
t/random_seed.py new file +394 -0

394 lines not on this page · raw message

t/run-fio-tests.py +16 -10
index c3091b68..4fe6fe46 100755
@@ -53,6 +53,7 @@ import traceback
 import subprocess
 import multiprocessing
 from pathlib import Path
+from statsmodels.sandbox.stats.runs import runstest_1samp
 
 
 class FioTest():
@@ -598,24 +599,16 @@ class FioJobTest_t0020(FioJobTest):
 
         log_lines = file_data.split('\n')
 
-        seq_count = 0
-        offsets = set()
+        offsets = []
 
         prev = int(log_lines[0].split(',')[4])
         for line in log_lines[1:]:
-            offsets.add(prev/4096)
+            offsets.append(prev/4096)
             if len(line.strip()) == 0:
                 continue
             cur = int(line.split(',')[4])
-            if cur - prev == 4096:
-                seq_count += 1
             prev = cur
 
-        # 10 is an arbitrary threshold
-        if seq_count > 10:
-            self.passed = False
-            self.failure_reason = "too many ({0}) consecutive offsets".format(seq_count)
-
         if len(offsets) != 256:
             self.passed = False
             self.failure_reason += " number of offsets is {0} instead of 256".format(len(offsets))
@@ -625,6 +618,11 @@ class FioJobTest_t0020(FioJobTest):
                 self.passed = False
                 self.failure_reason += " missing offset {0}".format(i*4096)
 
+        (z, p) = runstest_1samp(list(offsets))
+        if p < 0.05:
+            self.passed = False
+            self.failure_reason += f" runs test failed with p = {p}"
+
 
 class FioJobTest_t0022(FioJobTest):
     """Test consists of fio test job t0022"""
@@ -1361,6 +1359,14 @@ TEST_LIST = [
         'success':          SUCCESS_DEFAULT,
         'requirements':     [],
     },
+    {
+        'test_id':          1013,
+        'test_class':       FioExeTest,
+        'exe':              't/random_seed.py',
+        'parameters':       ['-f', '{fio_path}'],
+        'success':          SUCCESS_DEFAULT,
+        'requirements':     [],
+    },
 ]
 
 
thread_options.h +0 -3
index 6670cbbf..a24ebee6 100644
@@ -162,7 +162,6 @@ struct thread_options {
 	unsigned int do_disk_util;
 	unsigned int override_sync;
 	unsigned int rand_repeatable;
-	unsigned int allrand_repeatable;
 	unsigned long long rand_seed;
 	unsigned int log_avg_msec;
 	unsigned int log_hist_msec;
@@ -485,8 +484,6 @@ struct thread_options_pack {
 	uint32_t do_disk_util;
 	uint32_t override_sync;
 	uint32_t rand_repeatable;
-	uint32_t allrand_repeatable;
-	uint32_t pad2;
 	uint64_t rand_seed;
 	uint32_t log_avg_msec;
 	uint32_t log_hist_msec;