Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Aug 6, 2025

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit ef740f7cbe47d5552677fb018b15b4ab1f1de4f4:

t/io_uring_pi: test script for io_uring PI (2025-08-02 09:40:12 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to b1b07c8dfbb562a949afd127d693e9c0cb009827:

ioengines: bump version number (2025-08-05 15:56:45 -0600)

----------------------------------------------------------------
Jens Axboe (2):
Merge branch 'verify_inflight' of https://github.com/noclip-code/fio
ioengines: bump version number

Riley Thomasson (5):
verify: make numberio uint64_t
verify: rework write tracking for use with verify_save_state()
verify: plumb inflight write information through verify state
verify: clear inflight log in between loops
verify: add versioning to verify_header

 backend.c        | 157 +++++++++++++++++++++++++++++++++++++++++--------------
 file.h           |   7 ---
 fio.h            |  17 +++++-
 io_u.c           |  11 +---
 io_u.h           |   4 +-
 ioengines.h      |   2 +-
 iolog.h          |   2 +-
 libfio.c         |   2 +
 t/verify-state.c |  26 ++++-----
 verify-state.h   |  24 ++++-----
 verify.c         | 118 ++++++++++++-----------------------------
 verify.h         |  11 +++-
 12 files changed, 202 insertions(+), 179 deletions(-)

---

Diff of recent changes:

backend.c +117 -40
index 0cdee864..922309a0 100644
@@ -714,7 +714,7 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 				break;
 		}
 
-		if (verify_state_should_stop(td, io_u)) {
+		if (verify_state_should_stop(td, io_u->numberio)) {
 			put_io_u(td, io_u);
 			break;
 		}
@@ -948,6 +948,102 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir,
 	}
 }
 
+/*
+ * Add numberio from io_u to the inflight log.
+ */
+void log_inflight(struct thread_data *td, struct io_u *io_u)
+{
+	int idx, i;
+
+	if (!td->inflight_numberio || io_u->ddir != DDIR_WRITE)
+		return;
+
+	if (io_u->inflight_idx != -1) {
+		log_err("inflight_idx already set: inflight_idx=%d\n",
+			io_u->inflight_idx);
+		abort();
+	}
+
+	if (td->inflight_issued != io_u->numberio) {
+		log_err("inflight_issued does not match: numberio=%"PRIu64", inflight_issued=%"PRIu64"\n",
+			io_u->numberio, td->inflight_issued);
+		abort();
+	}
+
+	/* Walk the inflight list until we find a free slot. */
+	idx = td->next_inflight_numberio_idx;
+	for (i = 0; i < td->o.iodepth; i++) {
+		if (td->inflight_numberio[idx] == INVALID_NUMBERIO) {
+			/*
+			 * The order here is important - we must "protect" this write in the
+			 * inflight list before making it visible in inflight_issued.
+			 */
+			atomic_store_release(&td->inflight_numberio[idx], io_u->numberio);
+			td->next_inflight_numberio_idx = (idx + 1) % td->o.iodepth;
+			io_u->inflight_idx = idx;
+
+			atomic_store_release(&td->inflight_issued, io_u->numberio + 1);
+			dprint(FD_VERIFY, "log_inflight: numberio=%"PRIu64", inflight_idx=%d\n",
+				io_u->numberio, idx);
+			return;
+		}
+		idx = (idx + 1) % td->o.iodepth;
+	}
+
+	log_err("failed to allocate inflight slot: next_inflight_numberio_idx=%u\n",
+		td->next_inflight_numberio_idx);
+	abort();
+}
+
+/*
+ * Invalidate inflight log entry.
+ */
+void invalidate_inflight(struct thread_data *td, struct io_u *io_u)
+{
+	if (!td->inflight_numberio ||
+		io_u->ddir != DDIR_WRITE ||
+		io_u->inflight_idx == -1) {
+		return;
+	}
+
+	dprint(FD_VERIFY, "invalidate_inflight: numberio=%"PRIu64", inflight_idx=%d\n",
+		io_u->numberio, io_u->inflight_idx);
+
+	if (td->inflight_numberio[io_u->inflight_idx] == INVALID_NUMBERIO) {
+		log_err("inflight entry already invalid: numberio=%"PRIu64", inflight_idx=%d\n",
+			io_u->numberio, io_u->inflight_idx);
+		abort();
+	} else if (td->inflight_numberio[io_u->inflight_idx] != io_u->numberio) {
+		log_err("inflight entry numberio does not match: expected numberio=%"PRIu64", observed numberio=%"PRIu64", inflight_idx=%d\n",
+			io_u->numberio, td->inflight_numberio[io_u->inflight_idx], io_u->inflight_idx);
+		abort();
+	}
+
+	atomic_store_release(&td->inflight_numberio[io_u->inflight_idx], INVALID_NUMBERIO);
+	io_u->inflight_idx = -1;
+}
+
+/*
+ * Clear inflight log.
+ */
+void clear_inflight(struct thread_data *td)
+{
+	int i;
+
+	if (!td->inflight_numberio)
+		return;
+
+	for (i = 0; i < td->o.iodepth; i++)
+		td->inflight_numberio[i] = INVALID_NUMBERIO;
+
+	td->next_inflight_numberio_idx = 0;
+	/*
+	 * Experimental verify can increment io_issues for writes, so catch
+	 * inflight_issued up in between loops.
+	 */
+	td->inflight_issued = td->io_issues[DDIR_WRITE];
+}
+
 /*
  * Main IO worker function. It retrieves io_u's to process and queues
  * and reaps them, checking for rate and errors along the way.
@@ -1059,6 +1155,7 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 				io_u_set(td, io_u, IO_U_F_PATTERN_DONE);
 				io_u->numberio = td->io_issues[io_u->ddir];
 				populate_verify_io_u(td, io_u);
+				log_inflight(td, io_u);
 			}
 		}
 
@@ -1083,7 +1180,7 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 					io_u->rand_seed *= __rand(&td->verify_state);
 			}
 
-			if (verify_state_should_stop(td, io_u)) {
+			if (verify_state_should_stop(td, td->io_issues[io_u->ddir])) {
 				put_io_u(td, io_u);
 				break;
 			}
@@ -1227,49 +1324,29 @@ reap:
 		bytes_done[i] = td->bytes_done[i] - bytes_done[i];
 }
 
-static void free_file_completion_logging(struct thread_data *td)
+static int init_inflight_logging(struct thread_data *td)
 {
-	struct fio_file *f;
-	unsigned int i;
-
-	for_each_file(td, f, i) {
-		if (!f->last_write_comp)
-			break;
-		sfree(f->last_write_comp);
-	}
-}
-
-static int init_file_completion_logging(struct thread_data *td,
-					unsigned int depth)
-{
-	struct fio_file *f;
 	unsigned int i;
 
 	if (td->o.verify == VERIFY_NONE || !td->o.verify_state_save)
 		return 0;
 
-	/*
-	 * Async IO completion order may be different from issue order. Double
-	 * the number of write completions to cover the case the writes issued
-	 * earlier complete slowly and fall in the last write log entries.
-	 */
-	td->last_write_comp_depth = depth;
-	if (!td_ioengine_flagged(td, FIO_SYNCIO))
-		td->last_write_comp_depth += depth;
-
-	for_each_file(td, f, i) {
-		f->last_write_comp = scalloc(td->last_write_comp_depth,
-					     sizeof(uint64_t));
-		if (!f->last_write_comp)
-			goto cleanup;
+	td->inflight_numberio = scalloc(td->o.iodepth, sizeof(uint64_t));
+	if (!td->inflight_numberio) {
+		log_err("fio: failed to alloc inflight write data\n");
+		return 1;
 	}
 
+	for (i = 0; i < td->o.iodepth; i++)
+		td->inflight_numberio[i] = INVALID_NUMBERIO;
+
 	return 0;
+}
 
-cleanup:
-	free_file_completion_logging(td);
-	log_err("fio: failed to alloc write comp data\n");
-	return 1;
+static void free_inflight_logging(struct thread_data *td)
+{
+	if (td->inflight_numberio)
+		sfree(td->inflight_numberio);
 }
 
 static void cleanup_io_u(struct thread_data *td)
@@ -1294,16 +1371,15 @@ static void cleanup_io_u(struct thread_data *td)
 	io_u_qexit(&td->io_u_freelist, false);
 	io_u_qexit(&td->io_u_all, td_offload_overlap(td));
 
-	free_file_completion_logging(td);
+	free_inflight_logging(td);
 }
 
 static int init_io_u(struct thread_data *td)
 {
 	struct io_u *io_u;
-	int cl_align, i, max_units;
+	int cl_align, i;
 	int err;
 
-	max_units = td->o.iodepth;
 
 	err = 0;
 	err += !io_u_rinit(&td->io_u_requeues, td->o.iodepth);
@@ -1317,7 +1393,7 @@ static int init_io_u(struct thread_data *td)
 
 	cl_align = os_cache_line_size();
 
-	for (i = 0; i < max_units; i++) {
+	for (i = 0; i < td->o.iodepth; i++) {
 		void *ptr;
 
 		if (td->terminate)
@@ -1334,6 +1410,7 @@ static int init_io_u(struct thread_data *td)
 		INIT_FLIST_HEAD(&io_u->verify_list);
 		dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i);
 
+		io_u->inflight_idx = -1;
 		io_u->index = i;
 		io_u->flags = IO_U_F_FREE;
 		io_u_qpush(&td->io_u_freelist, io_u);
@@ -1357,7 +1434,7 @@ static int init_io_u(struct thread_data *td)
 	if (init_io_u_buffers(td))
 		return 1;
 
-	if (init_file_completion_logging(td, max_units))
+	if (init_inflight_logging(td))
 		return 1;
 
 	return 0;
file.h +0 -7
index e38ed2f1..8fd40cdf 100644
@@ -122,13 +122,6 @@ struct fio_file {
 	uint64_t first_write;
 	uint64_t last_write;
 
-	/*
-	 * Tracks the last iodepth number of completed writes, if data
-	 * verification is enabled
-	 */
-	uint64_t *last_write_comp;
-	unsigned int last_write_idx;
-
 	/*
 	 * For use by the io engine to store offset
 	 */
fio.h +16 -1
index 00f0f09b..e11b9261 100644
@@ -73,6 +73,8 @@ struct fio_sem;
 
 #define MAX_TRIM_RANGE	256
 
+#define INVALID_NUMBERIO UINT64_MAX
+
 /*
  * Range for trim command
  */
@@ -376,6 +378,13 @@ struct thread_data {
 	uint64_t io_issue_bytes[DDIR_RWDIR_CNT];
 	uint64_t loops;
 
+	/*
+	 * Keep track of inflight write sequence numbers (numberio) which are used to save verify state.
+	 */
+	uint64_t *inflight_numberio;
+	unsigned int next_inflight_numberio_idx;
+	uint64_t inflight_issued;
+
 	/*
 	 * Completions
 	 */
@@ -388,7 +397,6 @@ struct thread_data {
 	struct fio_sem *sem;
 	uint64_t bytes_done[DDIR_RWDIR_CNT];
 	uint64_t bytes_verified;
-	uint32_t last_write_comp_depth;
 
 	uint64_t *thinktime_blocks_counter;
 	struct timespec last_thinktime;
@@ -782,6 +790,13 @@ extern void lat_target_check(struct thread_data *);
 extern void lat_target_init(struct thread_data *);
 extern void lat_target_reset(struct thread_data *);
 
+/*
+ * Inflight log
+ */
+extern void log_inflight(struct thread_data *, struct io_u *);
+extern void invalidate_inflight(struct thread_data *, struct io_u *);
+extern void clear_inflight(struct thread_data *);
+
 /*
  * Iterates all threads/processes within all the defined jobs
  * Usage:
io_u.c +1 -10
index ca97f388..6d0f32a8 100644
@@ -2065,8 +2065,6 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,
 static void file_log_write_comp(const struct thread_data *td, struct fio_file *f,
 				uint64_t offset, unsigned int bytes)
 {
-	int idx;
-
 	if (!f)
 		return;
 
@@ -2074,14 +2072,6 @@ static void file_log_write_comp(const struct thread_data *td, struct fio_file *f
 		f->first_write = offset;
 	if (f->last_write == -1ULL || ((offset + bytes) > f->last_write))
 		f->last_write = offset + bytes;
-
-	if (!f->last_write_comp)
-		return;
-
-	idx = f->last_write_idx++;
-	f->last_write_comp[idx] = offset;
-	if (f->last_write_idx == td->last_write_comp_depth)
-		f->last_write_idx = 0;
 }
 
 static bool should_account(struct thread_data *td)
@@ -2101,6 +2091,7 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
 
 	assert(io_u->flags & IO_U_F_FLIGHT);
 	io_u_clear(td, io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK | IO_U_F_PATTERN_DONE);
+	invalidate_inflight(td, io_u);
 
 	if (td->o.zone_mode == ZONE_MODE_ZBD && td->o.recover_zbd_write_error &&
 	    io_u->error && io_u->ddir == DDIR_WRITE &&
io_u.h +3 -1
index 2d20a2b2..68771eba 100644
@@ -46,7 +46,7 @@ struct io_u {
 	/*
 	 * Write generation
 	 */
-	unsigned short numberio;
+	uint64_t numberio;
 
 	/*
 	 * IO priority.
@@ -90,6 +90,8 @@ struct io_u {
 	unsigned long long resid;
 	unsigned int error;
 
+	int inflight_idx;
+
 	/*
 	 * io engine private data
 	 */
ioengines.h +1 -1
index bd5d189c..afafeeb9 100644
@@ -9,7 +9,7 @@
 #include "zbd_types.h"
 #include "dataplacement.h"
 
-#define FIO_IOOPS_VERSION	37
+#define FIO_IOOPS_VERSION	38
 
 #ifndef CONFIG_DYNAMIC_ENGINES
 #define FIO_STATIC	static
iolog.h +1 -1
index e864d169..b52ae87d 100644
@@ -260,7 +260,7 @@ struct io_piece {
 		struct fio_file *file;
 	};
 	unsigned long long offset;
-	unsigned short numberio;
+	uint64_t numberio;
 	unsigned long len;
 	unsigned int flags;
 	enum fio_ddir ddir;
libfio.c +2 -0
index 2596ae5a..3503c65e 100644
@@ -128,6 +128,8 @@ void clear_io_state(struct thread_data *td, int all)
 	 */
 	if (td->o.rand_repeatable)
 		td_fill_rand_seeds(td);
+
+	clear_inflight(td);
 }
 
 void reset_all_stats(struct thread_data *td)
t/verify-state.c +9 -17
index f8787e9a..e1b92ee8 100644
@@ -21,19 +21,16 @@ static void show_s(struct thread_io_list *s, unsigned int no_s)
 
 	printf("Thread:\t\t%u\n", no_s);
 	printf("Name:\t\t%s\n", s->name);
-	printf("Completions:\t%llu\n", (unsigned long long) s->no_comps);
 	printf("Depth:\t\t%llu\n", (unsigned long long) s->depth);
-	printf("Max completions per file:\t\t%lu\n", (unsigned long) s->max_no_comps_per_file);
 	printf("Number IOs:\t%llu\n", (unsigned long long) s->numberio);
 	printf("Index:\t\t%llu\n", (unsigned long long) s->index);
 
-	printf("Completions:\n");
-	if (!s->no_comps)
+	printf("Inflight writes:\n");
+	if (!s->depth)
 		return;
-	for (i = s->no_comps - 1; i >= 0; i--) {
-		printf("\t(file=%2llu) %llu\n",
-				(unsigned long long) s->comps[i].fileno,
-				(unsigned long long) s->comps[i].offset);
+	for (i = s->depth - 1; i >= 0; i--) {
+		printf("\t%llu\n",
+				(unsigned long long) s->inflight[i].numberio);
 	}
 }
 
@@ -45,23 +42,18 @@ static void show(struct thread_io_list *s, size_t size)
 	do {
 		int i;
 
-		s->no_comps = le64_to_cpu(s->no_comps);
 		s->depth = le32_to_cpu(s->depth);
-		s->max_no_comps_per_file = le32_to_cpu(s->max_no_comps_per_file);
-		s->nofiles = le32_to_cpu(s->nofiles);
 		s->numberio = le64_to_cpu(s->numberio);
 		s->index = le64_to_cpu(s->index);
 
-		for (i = 0; i < s->no_comps; i++) {
-			s->comps[i].fileno = le64_to_cpu(s->comps[i].fileno);
-			s->comps[i].offset = le64_to_cpu(s->comps[i].offset);
-		}
+		for (i = 0; i < s->depth; i++)
+			s->inflight[i].numberio = le64_to_cpu(s->inflight[i].numberio);
 
 		show_s(s, no_s);
 		no_s++;
-		size -= __thread_io_list_sz(s->max_no_comps_per_file, s->nofiles);
+		size -= __thread_io_list_sz(s->depth);
 		s = (struct thread_io_list *)((char *) s +
-			__thread_io_list_sz(s->max_no_comps_per_file, s->nofiles));
+			__thread_io_list_sz(s->depth));
 	} while (size != 0);
 }
 
verify-state.h +9 -15
index 603af70d..c055008e 100644
@@ -22,24 +22,18 @@ struct thread_rand_state {
 	};
 };
 
-/*
- * For dumping current write state
- */
-struct file_comp {
-	uint64_t fileno;
-	uint64_t offset;
+/* a single inflight write */
+struct inflight_write {
+	uint64_t numberio;
 };
 
 struct thread_io_list {
-	uint64_t no_comps; /* Number of completions saved for the thread */
 	uint32_t depth; /* I/O depth of the job that saves the verify state */
-	uint32_t max_no_comps_per_file;
-	uint32_t nofiles;
-	uint64_t numberio;
+	uint64_t numberio; /* Number of issued writes */
 	uint64_t index;
 	struct thread_rand_state rand;
 	uint8_t name[64];
-	struct file_comp comps[0];
+	struct inflight_write inflight[0];
 };
 
 struct all_io_list {
@@ -63,18 +57,18 @@ extern void __verify_save_state(struct all_io_list *, const char *);
 extern void verify_save_state(int mask);
 extern int verify_load_state(struct thread_data *, const char *);
 extern void verify_free_state(struct thread_data *);
-extern int verify_state_should_stop(struct thread_data *, struct io_u *);
+extern int verify_state_should_stop(struct thread_data *, uint64_t);
 extern void verify_assign_state(struct thread_data *, void *);
 extern int verify_state_hdr(struct verify_state_hdr *, struct thread_io_list *);
 
-static inline size_t __thread_io_list_sz(uint32_t depth, uint32_t nofiles)
+static inline size_t __thread_io_list_sz(uint32_t depth)
 {
-	return sizeof(struct thread_io_list) + depth * nofiles * sizeof(struct file_comp);
+	return sizeof(struct thread_io_list) + depth * sizeof(struct inflight_write);
 }
 
 static inline size_t thread_io_list_sz(struct thread_io_list *s)
 {
-	return __thread_io_list_sz(le32_to_cpu(s->max_no_comps_per_file), le32_to_cpu(s->nofiles));
+	return __thread_io_list_sz(le32_to_cpu(s->depth));
 }
 
 static inline struct thread_io_list *io_list_next(struct thread_io_list *s)
verify.c +34 -84

192 lines not on this page · raw message

verify.h +9 -2
index 539e6f6c..e361337c 100644
@@ -32,6 +32,12 @@ enum {
 	VERIFY_NULL,			/* pretend to verify */
 };
 
+/*
+ * Set the high bit to distinguish versioned headers from older
+ * non-versioned headers.
+ */
+#define VERIFY_HEADER_VERSION 0x81
+
 /*
  * A header structure associated with each checksummed data block. It is
  * followed by a checksum specific header that contains the verification
@@ -39,14 +45,15 @@ enum {
  */
 struct verify_header {
 	uint16_t magic;
-	uint16_t verify_type;
+	uint8_t version;
+	uint8_t verify_type;
 	uint32_t len;
 	uint64_t rand_seed;
 	uint64_t offset;
 	uint32_t time_sec;
 	uint32_t time_nsec;
+	uint64_t numberio;
 	uint16_t thread;
-	uint16_t numberio;
 	uint32_t crc32;
 };