Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message May 21

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 4de5b267a716a9b7b71b212e0b948743f70c4d54:

Merge branch 'pr-free-td-init' of https://github.com/malikoyv/fio (2026-05-15 08:03:55 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to d569883e281076e0dec5f5768b828ef44392a11a:

Merge branch 'skip-errored-offset' of https://github.com/minwooim/fio (2026-05-20 09:17:25 -0400)

----------------------------------------------------------------
Minwoo Im (1):
verify: skip errored WRITE offset in verify

Vincent Fu (2):
ci: run all non-QEMU tests on push/PR/schedule
Merge branch 'skip-errored-offset' of https://github.com/minwooim/fio

 backend.c               | 58 ++++++++++++++++++++++++++++--
 ci/actions-full-test.sh | 16 ---------
 fio.h                   |  7 ++++
 io_u.c                  | 23 ++++++++++++
 verify-state.h          |  3 +-
 verify.c                | 95 ++++++++++++++++++++++++++++++++++++++++++++-----
 6 files changed, 173 insertions(+), 29 deletions(-)

---

Diff of recent changes:

backend.c +55 -3
index 7a02ea11..f6023d0e 100644
@@ -621,6 +621,7 @@ static enum fio_q_status io_u_submit(struct thread_data *td, struct io_u *io_u)
  */
 static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 {
+	uint64_t expected_numberio;
 	struct fio_file *f;
 	struct io_u *io_u;
 	unsigned int i;
@@ -649,6 +650,7 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 	td_set_runstate(td, TD_VERIFYING);
 
 	io_u = NULL;
+	expected_numberio = UINT64_MAX;
 	while (!td->terminate) {
 		enum fio_ddir ddir;
 		int full;
@@ -677,6 +679,33 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 				break;
 			}
 
+			/*
+			 * Advance verify_state to the seed that was used when
+			 * this block was written.  Any numberio gap between
+			 * @expected_numberio and @io_u->numberio represents
+			 * skipped writes due to errors (e.g., suppressed by
+			 * --ignore_error= in online verification, or skipped
+			 *  offsets from do_dry_run in offline verification).
+			 */
+			if (td->o.verify_header_seed && !td->o.verify_pattern_bytes) {
+				uint64_t seed = 0;
+				uint64_t from;
+
+				if (io_u->numberio < expected_numberio)
+					from = io_u->numberio;
+				else
+					from = expected_numberio;
+
+				for (uint64_t n = from; n <= io_u->numberio; n++) {
+					seed = __rand(&td->verify_state);
+					if (sizeof(int) != sizeof(long *))
+						seed *= __rand(&td->verify_state);
+				}
+
+				io_u->rand_seed = seed;
+				expected_numberio = io_u->numberio + 1;
+			}
+
 			if (td_io_prep(td, io_u)) {
 				put_io_u(td, io_u);
 				break;
@@ -1207,7 +1236,26 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 					io_u->rand_seed *= __rand(&td->verify_state);
 			}
 
-			if (verify_state_should_stop(td, td->io_issues[io_u->ddir])) {
+			/*
+			 * Assign numberio for read-only verify workloads. This
+			 * handles two cases: (1) rw=[rand]write with verify_only=1
+			 * and verify_state_load=1, where we replay a prior write
+			 * run; (2) a prior job ran with rw=[rand]write and
+			 * verify_state_save=1, and the current job runs with
+			 * rw=[rand]read and verify_state_load=1 to verify those
+			 * writes.
+			 */
+			if (!td_rw(td) && !(io_u->flags & IO_U_F_VER_LIST))
+				io_u->numberio = td->io_issues[io_u->ddir];
+
+			if (verify_state_should_skip(td, io_u->numberio)) {
+				/* Account for this I/O so we move to the next sequence */
+				td->io_issues[io_u->ddir]++;
+				put_io_u(td, io_u);
+				continue;
+			}
+
+			if (verify_state_should_stop(td, io_u->numberio)) {
 				put_io_u(td, io_u);
 				break;
 			}
@@ -1380,6 +1428,8 @@ static void free_inflight_logging(struct thread_data *td)
 {
 	if (td->inflight_numberio)
 		sfree(td->inflight_numberio);
+	if (td->failed_numberio)
+		free(td->failed_numberio);
 }
 
 static void cleanup_io_u(struct thread_data *td)
@@ -1780,8 +1830,10 @@ static uint64_t do_dry_run(struct thread_data *td)
 		if (td_write(td) && io_u->ddir == DDIR_WRITE &&
 		    td->o.do_verify &&
 		    td->o.verify != VERIFY_NONE &&
-		    !td->o.experimental_verify)
-			log_io_piece(td, io_u);
+		    !td->o.experimental_verify) {
+			if (!verify_state_should_skip(td, io_u->numberio))
+				log_io_piece(td, io_u);
+		}
 
 		ret = io_u_sync_complete(td, io_u);
 		(void) ret;
ci/actions-full-test.sh +0 -16
index 14c2fcf7..56b5b31f 100755
@@ -40,22 +40,6 @@ main() {
 
     fi
 
-    # If we are running a nightly test just run the verify tests.  Skip the
-    # verify test script with pull requests and pushes because it takes so
-    # long. When this workflow is run manually everything will be run.
-    if [ "${GITHUB_EVENT_NAME}" == "schedule" ]; then
-	args+=(
-	    --run-only
-	    1017
-	    -p
-	    "1017:--complete"
-	)
-    elif [ "${GITHUB_EVENT_NAME}" == "pull_request" ] || [ "${GITHUB_EVENT_NAME}" == "push" ]; then
-	skip+=(
-	    1017
-	)
-    fi
-
     echo python3 t/run-fio-tests.py --skip "${skip[@]}" "${args[@]}"
     python3 t/run-fio-tests.py -c --skip "${skip[@]}" "${args[@]}"
     make -C doc html
fio.h +7 -0
index 18196123..3b6ff6c6 100644
@@ -386,6 +386,13 @@ struct thread_data {
 	unsigned int next_inflight_numberio_idx;
 	uint64_t inflight_issued;
 
+	/*
+	 * Track failed write I/Os for offline verification (to exclude from verify state)
+	 */
+	uint64_t *failed_numberio;
+	unsigned int failed_numberio_count;
+	unsigned int failed_numberio_alloc;
+
 	/*
 	 * Completions
 	 */
io_u.c +23 -0
index 24d8b7de..1725a9ce 100644
@@ -2137,6 +2137,29 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
 	    !td_ioengine_flagged(td, FIO_SYNCIO))
 		zbd_recover_write_error(td, io_u);
 
+	/*
+	 * Track failed write I/Os for offline verification (independent of io_u->ipo).
+	 * This works even when do_verify=0 is set.  In case of online
+	 * verification, `unlog_io_piece()` will be called so that the ipo will
+	 * be removed from the `io_hist`.
+	 */
+	if (io_u->error && ddir == DDIR_WRITE && io_u->numberio != INVALID_NUMBERIO) {
+		if (td->failed_numberio_count >= td->failed_numberio_alloc) {
+			unsigned int new_alloc;
+
+			if (td->failed_numberio_alloc)
+				new_alloc = td->failed_numberio_alloc * 2;
+			else
+				new_alloc = 16;
+			td->failed_numberio = realloc(td->failed_numberio,
+						new_alloc * sizeof(uint64_t));
+			td->failed_numberio_alloc = new_alloc;
+		}
+		td->failed_numberio[td->failed_numberio_count++] = io_u->numberio;
+		dprint(FD_VERIFY, "Recorded failed write numberio=%"PRIu64"\n",
+			io_u->numberio);
+	}
+
 	/*
 	 * Mark IO ok to verify
 	 */
verify-state.h +2 -1
index 27eb9e9a..03093ecd 100644
@@ -41,7 +41,7 @@ struct all_io_list {
 	struct thread_io_list state[0];
 };
 
-#define VSTATE_HDR_VERSION	0x05
+#define VSTATE_HDR_VERSION	0x06
 
 struct verify_state_hdr {
 	uint64_t version;
@@ -57,6 +57,7 @@ extern void __verify_save_state(struct all_io_list *, const char *);
 extern void verify_save_state(int mask);
 extern int verify_load_state(struct thread_data *, const char *);
 extern void verify_free_state(struct thread_data *);
+extern int verify_state_should_skip(struct thread_data *, uint64_t);
 extern int verify_state_should_stop(struct thread_data *, uint64_t);
 extern void verify_assign_state(struct thread_data *, void *);
 extern int verify_state_hdr(struct verify_state_hdr *, struct thread_io_list *);
verify.c +86 -9

145 lines not on this page · raw message