Recent changes (master)
The following changes since commit df60924e4d854cd6fdf4b730d37282c6b6b4c0a1:
Merge branch 'update-docs-for-compare' of https://github.com/minwooim/fio (2025-03-05 12:05:52 -0500)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to a72fed7a4900e43035cf4e7e653fd11b6671c726:
ci: add nightly test for verify (2025-03-06 13:58:43 -0500)
----------------------------------------------------------------
Ankit Kumar (14):
filesetup: remove unnecessary check
verify: add missing client/server support for verify_write_sequence
init: write sequence behavior change for verify_only mode
fio: add verify_header_seed option
verify: disable header seed checking instead of overwriting it
verify: enable header seed check for 100% write jobs
verify: disable header seed check for verify_only jobs
verify: header seed check for read only workloads
verify: fix verify issues with norandommap
verify: disable write sequence checks with norandommap and iodepth > 1
backend: fix verify issue during readwrite
init: fixup verify_offset option
verify: fix verify issue with offest modifiers
verify: adjust fio_offset_overlap_risk to include randommap
Vincent Fu (7):
t/fiotestcommon: do not require nvmecdev argument for Requirements t/fiotestlib: improve JSON decoding
t/fiotestlib: display stderr size when it is not empty but should be t/verify.py: Add verify test script
t/fiotestcommon: add a success pattern for long tests t/run-fio-test: add t/verify.py
ci: add nightly test for verify
.github/workflows/ci.yml | 2 + HOWTO.rst | 52 ++- backend.c | 18 +- cconv.c | 4 + ci/actions-full-test.sh | 13 + filesetup.c | 14 +- fio.1 | 46 ++- fio.h | 9 + init.c | 43 ++- iolog.c | 9 +- options.c | 11 + server.h | 2 +- t/fiotestcommon.py | 7 +- t/fiotestlib.py | 20 +- t/run-fio-tests.py | 8 + t/verify.py | 803 +++++++++++++++++++++++++++++++++++++++++++++++ thread_options.h | 3 + verify.c | 10 +- 18 files changed, 1009 insertions(+), 65 deletions(-) create mode 100755 t/verify.py
---
Diff of recent changes:
.github/workflows/ci.yml +2 -0
@@ -4,6 +4,8 @@ on: push: pull_request: workflow_dispatch: + schedule: + - cron: "35 5 * * *" # 5:35 UTC which is 0:35 ET jobs: build-containers:
HOWTO.rst +38 -14
94 lines not on this page · raw message
backend.c +16 -2
@@ -978,8 +978,11 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)if (td_write (td) &&td_random (td) && td->o.norandommap) total_bytes =max (total_bytes, (uint64_t ) td->o.io_size); -/* Don't break too early if io_size > size */ -if (td_rw (td) && !td_random (td)) +/* +* Don't break too early if io_size > size. The exception is when +* verify is enabled. +*/ +if (td_rw (td) && !td_random (td) && td->o.verify ==VERIFY_NONE ) total_bytes =max (total_bytes, (uint64_t )td->o.io_size);/*
@@ -1069,6 +1072,17 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)if (td->o.verify !=VERIFY_NONE && io_u->ddir ==DDIR_READ && ((io_u->flags &IO_U_F_VER_LIST ) || !td_rw (td))) { +/* +* For read only workloads generate the seed. This way +* we can still verify header seed at any later +* invocation. +*/ +if (!td_write (td) && !td->o.verify_pattern_bytes) { + io_u->rand_seed =__rand (&td->verify_state); +if (sizeof (int ) !=sizeof (long *)) + io_u->rand_seed *=__rand (&td->verify_state); + } +if (verify_state_should_stop (td, io_u)) {put_io_u (td, io_u);break ;
cconv.c +4 -0
@@ -182,6 +182,8 @@ int convert_thread_options_to_cpu(struct thread_options *o, o->verify_state =le32_to_cpu (top->verify_state); o->verify_interval =le32_to_cpu (top->verify_interval); o->verify_offset =le32_to_cpu (top->verify_offset); + o->verify_write_sequence =le32_to_cpu (top->verify_write_sequence); + o->verify_header_seed =le32_to_cpu (top->verify_header_seed); o->verify_pattern_bytes =le32_to_cpu (top->verify_pattern_bytes); o->buffer_pattern_bytes =le32_to_cpu (top->buffer_pattern_bytes);
@@ -442,6 +444,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->verify_state =cpu_to_le32 (o->verify_state); top->verify_interval =cpu_to_le32 (o->verify_interval); top->verify_offset =cpu_to_le32 (o->verify_offset); + top->verify_write_sequence =cpu_to_le32 (o->verify_write_sequence); + top->verify_header_seed =cpu_to_le32 (o->verify_header_seed); top->verify_pattern_bytes =cpu_to_le32 (o->verify_pattern_bytes); top->verify_fatal =cpu_to_le32 (o->verify_fatal); top->verify_dump =cpu_to_le32 (o->verify_dump);
ci/actions-full-test.sh +13 -0
@@ -33,6 +33,19 @@ main() { fi + # If we are running a nightly test just run the verify tests. + # Otherwise skip the verify test script because it takes so long. + if [ "${GITHUB_EVENT_NAME}" == "schedule" ]; then + args+=( + --run-only + 1017 + ) + else + skip+=( + 1017 + ) + fi + echo python3 t/run-fio-tests.py --skip "${skip[@]}" "${args[@]}" python3 t/run-fio-tests.py --skip "${skip[@]}" "${args[@]}" make -C doc html
filesetup.c +4 -10
@@ -1388,16 +1388,10 @@ int setup_files(struct thread_data *td)if (err) goto err_out; -/* -* iolog already set the total io size, if we read back -* stored entries. -*/ -if (!o->read_iolog_file) { -if (o->io_size) - td->total_io_size = o->io_size * o->loops; -else - td->total_io_size = o->size * o->loops; - } +if (o->io_size) + td->total_io_size = o->io_size * o->loops; +else + td->total_io_size = o->size * o->loops; done:if (td->o.zone_mode ==ZONE_MODE_ZBD ) {
fio.1 +33 -13
89 lines not on this page · raw message
fio.h +9 -0
@@ -800,6 +800,15 @@ extern void lat_target_reset(struct thread_data *); (i) < (td)->o.nr_files && ((f) = (td)->files[i]) !=NULL ; \ (i)++) +static inline bool fio_offset_overlap_risk (struct thread_data *td) +{ +if (td->o.norandommap || td->o.softrandommap || + td->o.ddir_seq_add || (td->o.ddir_seq_nr >1 )) +return true; + +return false; +} +static inline bool fio_fill_issue_time (struct thread_data *td) { if (td->o.read_iolog_file ||
init.c +41 -2
49 lines not on this page · raw message
iolog.c +5 -4
@@ -301,11 +301,12 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u) }/* -* Only sort writes if we don't have a random map in which case we need -* to check for duplicate blocks and drop the old one, which we rely on -* the rb insert/lookup for handling. +* Sort writes if we don't have a random map in which case we need to +* check for duplicate blocks and drop the old one, which we rely on +* the rb insert/lookup for handling. Sort writes if we have offset +* modifier which can also create duplicate blocks. */ -if (file_randommap (td, ipo->file)) { +if (!fio_offset_overlap_risk (td)) {INIT_FLIST_HEAD (&ipo->list);flist_add_tail (&ipo->list, &td->io_hist_list); ipo->flags |=IP_F_ONLIST ;
options.c +11 -0
@@ -3408,6 +3408,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_IO, .group = FIO_OPT_G_VERIFY, }, + { + .name = "verify_header_seed", + .lname = "Verify header seed", + .off1 = offsetof(struct thread_options, verify_header_seed), + .type = FIO_OPT_BOOL, + .def = "1", + .help = "Verify the header seed used to generate the buffer contents", + .parent = "verify", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_VERIFY, + }, #ifdef FIO_HAVE_TRIM { .name = "trim_percentage",
server.h +1 -1
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =107 , +FIO_SERVER_VER =109 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
t/fiotestcommon.py +6 -1
@@ -19,6 +19,11 @@ SUCCESS_DEFAULT = { 'stderr_empty': True, 'timeout': 600, } +SUCCESS_LONG = { + 'zero_return': True, + 'stderr_empty': True, + 'timeout': 1800, + } SUCCESS_NONZERO = { 'zero_return': False, 'stderr_empty': False,
@@ -101,7 +106,7 @@ class Requirements(): Requirements._unittests = os.path.exists(unittest_path) Requirements._cpucount4 = multiprocessing.cpu_count() >= 4 - Requirements._nvmecdev = args.nvmecdev + Requirements._nvmecdev = args.nvmecdev if hasattr(args, 'nvmecdev') else False req_list = [ Requirements.linux,
t/fiotestlib.py +11 -9
@@ -139,7 +139,7 @@ class FioExeTest(FioTest): if 'stderr_empty' in self.success: if self.success['stderr_empty']: if stderr_size != 0: - self.failure_reason = f"{self.failure_reason} stderr not empty," + self.failure_reason = f"{self.failure_reason} stderr not empty size {stderr_size}," self.passed = False else: if stderr_size == 0:
@@ -260,12 +260,13 @@ class FioJobFileTest(FioExeTest): return # - # Sometimes fio informational messages are included at the top of the - # JSON output, especially under Windows. Try to decode output as JSON - # data, skipping everything until the first { + # Sometimes fio informational messages are included outside the JSON + # output, especially under Windows. Try to decode output as JSON data, + # skipping outside the first { and last } # lines = file_data.splitlines() - file_data = '\n'.join(lines[lines.index("{"):]) + last = len(lines) - lines[::-1].index("}") + file_data = '\n'.join(lines[lines.index("{"):last]) try: self.json_data = json.loads(file_data) except json.JSONDecodeError:
@@ -320,12 +321,13 @@ class FioJobCmdTest(FioExeTest): file_data = file.read() # - # Sometimes fio informational messages are included at the top of the - # JSON output, especially under Windows. Try to decode output as JSON - # data, skipping everything until the first { + # Sometimes fio informational messages are included outside the JSON + # output, especially under Windows. Try to decode output as JSON data, + # skipping outside the first { and last } # lines = file_data.splitlines() - file_data = '\n'.join(lines[lines.index("{"):]) + last = len(lines) - lines[::-1].index("}") + file_data = '\n'.join(lines[lines.index("{"):last]) try: self.json_data = json.loads(file_data) except json.JSONDecodeError:
t/run-fio-tests.py +8 -0
@@ -1107,6 +1107,14 @@ TEST_LIST = [ 'success': SUCCESS_DEFAULT, 'requirements': [Requirements.linux], }, + { + 'test_id': 1017, + 'test_class': FioExeTest, + 'exe': 't/verify.py', + 'parameters': ['-f', '{fio_path}'], + 'success': SUCCESS_LONG, + 'requirements': [], + }, ]
t/verify.py new file +803 -0
803 lines not on this page · raw message
thread_options.h +3 -0
@@ -157,6 +157,7 @@ struct thread_options {unsigned int verify_state;unsigned int verify_state_save;unsigned int verify_write_sequence; +unsigned int verify_header_seed;unsigned int use_thread;unsigned int unlink;unsigned int unlink_each_loop;
@@ -484,6 +485,8 @@ struct thread_options_pack {uint32_t experimental_verify;uint32_t verify_state;uint32_t verify_state_save; +uint32_t verify_write_sequence; +uint32_t verify_header_seed;uint32_t use_thread;uint32_t unlink;uint32_t unlink_each_loop;
verify.c +1 -9
@@ -833,7 +833,7 @@ static int verify_header(struct io_u *io_u, struct thread_data *td, hdr->len, hdr_len); goto err; } -if (hdr->rand_seed != io_u->rand_seed) { +if (td->o.verify_header_seed && (hdr->rand_seed != io_u->rand_seed)) {log_err ("verify: bad header rand_seed %" PRIu64", wanted %" PRIu64, hdr->rand_seed, io_u->rand_seed);
@@ -934,14 +934,6 @@ int verify_io_u(struct thread_data *td, struct io_u **io_u_ptr)memswp (p, p + td->o.verify_offset, header_size); hdr = p; -/* -* Make rand_seed check pass when have verify_backlog or -* zone reset frequency for zonemode=zbd. -*/ -if (!td_rw (td) || (td->flags &TD_F_VER_BACKLOG ) || - td->o.zrf.u.f) - io_u->rand_seed = hdr->rand_seed; -if (td->o.verify !=VERIFY_PATTERN_NO_HDR ) { ret =verify_header (io_u, td, hdr, hdr_num, hdr_inc);if (ret)