Recent changes (master)
The following changes since commit ef740f7cbe47d5552677fb018b15b4ab1f1de4f4:
t/io_uring_pi: test script for io_uring PI (2025-08-02 09:40:12 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to b1b07c8dfbb562a949afd127d693e9c0cb009827:
ioengines: bump version number (2025-08-05 15:56:45 -0600)
----------------------------------------------------------------
Jens Axboe (2):
Merge branch 'verify_inflight' of https://github.com/noclip-code/fio
ioengines: bump version number
Riley Thomasson (5):
verify: make numberio uint64_t
verify: rework write tracking for use with verify_save_state()
verify: plumb inflight write information through verify state
verify: clear inflight log in between loops
verify: add versioning to verify_header
backend.c | 157 +++++++++++++++++++++++++++++++++++++++++-------------- file.h | 7 --- fio.h | 17 +++++- io_u.c | 11 +--- io_u.h | 4 +- ioengines.h | 2 +- iolog.h | 2 +- libfio.c | 2 + t/verify-state.c | 26 ++++----- verify-state.h | 24 ++++----- verify.c | 118 ++++++++++++----------------------------- verify.h | 11 +++- 12 files changed, 202 insertions(+), 179 deletions(-)
---
Diff of recent changes:
backend.c +117 -40
@@ -714,7 +714,7 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)break ; } -if (verify_state_should_stop (td, io_u)) { +if (verify_state_should_stop (td, io_u->numberio)) {put_io_u (td, io_u);break ; }
@@ -948,6 +948,102 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir, } } +/* +* Add numberio from io_u to the inflight log. +*/ +void log_inflight (struct thread_data *td,struct io_u *io_u) +{ +int idx, i; + +if (!td->inflight_numberio || io_u->ddir !=DDIR_WRITE ) +return ; + +if (io_u->inflight_idx !=-1 ) { +log_err ("inflight_idx already set: inflight_idx=%d\n" , + io_u->inflight_idx); +abort (); + } + +if (td->inflight_issued != io_u->numberio) { +log_err ("inflight_issued does not match: numberio=%" PRIu64", inflight_issued=%" PRIu64"\n" , + io_u->numberio, td->inflight_issued); +abort (); + } + +/* Walk the inflight list until we find a free slot. */ + idx = td->next_inflight_numberio_idx; +for (i =0 ; i < td->o.iodepth; i++) { +if (td->inflight_numberio[idx] ==INVALID_NUMBERIO ) { +/* +* The order here is important - we must "protect" this write in the +* inflight list before making it visible in inflight_issued. +*/ +atomic_store_release (&td->inflight_numberio[idx], io_u->numberio); + td->next_inflight_numberio_idx = (idx +1 ) % td->o.iodepth; + io_u->inflight_idx = idx; + +atomic_store_release (&td->inflight_issued, io_u->numberio +1 ); +dprint (FD_VERIFY ,"log_inflight: numberio=%" PRIu64", inflight_idx=%d\n" , + io_u->numberio, idx); +return ; + } + idx = (idx +1 ) % td->o.iodepth; + } + +log_err ("failed to allocate inflight slot: next_inflight_numberio_idx=%u\n" , + td->next_inflight_numberio_idx); +abort (); +} + +/* +* Invalidate inflight log entry. +*/ +void invalidate_inflight (struct thread_data *td,struct io_u *io_u) +{ +if (!td->inflight_numberio || + io_u->ddir !=DDIR_WRITE || + io_u->inflight_idx ==-1 ) { +return ; + } + +dprint (FD_VERIFY ,"invalidate_inflight: numberio=%" PRIu64", inflight_idx=%d\n" , + io_u->numberio, io_u->inflight_idx); + +if (td->inflight_numberio[io_u->inflight_idx] ==INVALID_NUMBERIO ) { +log_err ("inflight entry already invalid: numberio=%" PRIu64", inflight_idx=%d\n" , + io_u->numberio, io_u->inflight_idx); +abort (); + }else if (td->inflight_numberio[io_u->inflight_idx] != io_u->numberio) { +log_err ("inflight entry numberio does not match: expected numberio=%" PRIu64", observed numberio=%" PRIu64", inflight_idx=%d\n" , + io_u->numberio, td->inflight_numberio[io_u->inflight_idx], io_u->inflight_idx); +abort (); + } + +atomic_store_release (&td->inflight_numberio[io_u->inflight_idx],INVALID_NUMBERIO ); + io_u->inflight_idx =-1 ; +} + +/* +* Clear inflight log. +*/ +void clear_inflight (struct thread_data *td) +{ +int i; + +if (!td->inflight_numberio) +return ; + +for (i =0 ; i < td->o.iodepth; i++) + td->inflight_numberio[i] =INVALID_NUMBERIO ; + + td->next_inflight_numberio_idx =0 ; +/* +* Experimental verify can increment io_issues for writes, so catch +* inflight_issued up in between loops. +*/ + td->inflight_issued = td->io_issues[DDIR_WRITE ]; +} +/* * Main IO worker function. It retrieves io_u's to process and queues * and reaps them, checking for rate and errors along the way.
@@ -1059,6 +1155,7 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)io_u_set (td, io_u,IO_U_F_PATTERN_DONE ); io_u->numberio = td->io_issues[io_u->ddir];populate_verify_io_u (td, io_u); +log_inflight (td, io_u); } }
@@ -1083,7 +1180,7 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done) io_u->rand_seed *=__rand (&td->verify_state); } -if (verify_state_should_stop (td, io_u)) { +if (verify_state_should_stop (td, td->io_issues[io_u->ddir])) {put_io_u (td, io_u);break ; }
@@ -1227,49 +1324,29 @@ reap: bytes_done[i] = td->bytes_done[i] - bytes_done[i]; } -static void free_file_completion_logging (struct thread_data *td) +static int init_inflight_logging (struct thread_data *td) { -struct fio_file *f; -unsigned int i; - -for_each_file (td, f, i) { -if (!f->last_write_comp) -break ; -sfree (f->last_write_comp); - } -} - -static int init_file_completion_logging (struct thread_data *td, -unsigned int depth) -{ -struct fio_file *f;unsigned int i;if (td->o.verify ==VERIFY_NONE || !td->o.verify_state_save)return 0 ; -/* -* Async IO completion order may be different from issue order. Double -* the number of write completions to cover the case the writes issued -* earlier complete slowly and fall in the last write log entries. -*/ - td->last_write_comp_depth = depth; -if (!td_ioengine_flagged (td,FIO_SYNCIO )) - td->last_write_comp_depth += depth; - -for_each_file (td, f, i) { - f->last_write_comp =scalloc (td->last_write_comp_depth, -sizeof (uint64_t )); -if (!f->last_write_comp) - goto cleanup; + td->inflight_numberio =scalloc (td->o.iodepth,sizeof (uint64_t )); +if (!td->inflight_numberio) { +log_err ("fio: failed to alloc inflight write data\n" ); +return 1 ; } +for (i =0 ; i < td->o.iodepth; i++) + td->inflight_numberio[i] =INVALID_NUMBERIO ; +return 0 ; +} -cleanup: -free_file_completion_logging (td); -log_err ("fio: failed to alloc write comp data\n" ); -return 1 ; +static void free_inflight_logging (struct thread_data *td) +{ +if (td->inflight_numberio) +sfree (td->inflight_numberio); }static void cleanup_io_u (struct thread_data *td)
@@ -1294,16 +1371,15 @@ static void cleanup_io_u(struct thread_data *td)io_u_qexit (&td->io_u_freelist, false);io_u_qexit (&td->io_u_all,td_offload_overlap (td)); -free_file_completion_logging (td); +free_inflight_logging (td); }static int init_io_u (struct thread_data *td) {struct io_u *io_u; -int cl_align, i, max_units; +int cl_align, i;int err; - max_units = td->o.iodepth; err =0 ; err += !io_u_rinit (&td->io_u_requeues, td->o.iodepth);
@@ -1317,7 +1393,7 @@ static int init_io_u(struct thread_data *td) cl_align =os_cache_line_size (); -for (i =0 ; i < max_units; i++) { +for (i =0 ; i < td->o.iodepth; i++) {void *ptr;if (td->terminate)
@@ -1334,6 +1410,7 @@ static int init_io_u(struct thread_data *td)INIT_FLIST_HEAD (&io_u->verify_list);dprint (FD_MEM ,"io_u alloc %p, index %u\n" , io_u, i); + io_u->inflight_idx =-1 ; io_u->index = i; io_u->flags =IO_U_F_FREE ;io_u_qpush (&td->io_u_freelist, io_u);
@@ -1357,7 +1434,7 @@ static int init_io_u(struct thread_data *td)if (init_io_u_buffers (td))return 1 ; -if (init_file_completion_logging (td, max_units)) +if (init_inflight_logging (td))return 1 ;return 0 ;
file.h +0 -7
@@ -122,13 +122,6 @@ struct fio_file {uint64_t first_write;uint64_t last_write; -/* -* Tracks the last iodepth number of completed writes, if data -* verification is enabled -*/ -uint64_t *last_write_comp; -unsigned int last_write_idx; -/* * For use by the io engine to store offset */
fio.h +16 -1
@@ -73,6 +73,8 @@ struct fio_sem;#define MAX_TRIM_RANGE 256 +#define INVALID_NUMBERIO UINT64_MAX +/* * Range for trim command */
@@ -376,6 +378,13 @@ struct thread_data {uint64_t io_issue_bytes[DDIR_RWDIR_CNT ];uint64_t loops; +/* +* Keep track of inflight write sequence numbers (numberio) which are used to save verify state. +*/ +uint64_t *inflight_numberio; +unsigned int next_inflight_numberio_idx; +uint64_t inflight_issued; +/* * Completions */
@@ -388,7 +397,6 @@ struct thread_data {struct fio_sem *sem;uint64_t bytes_done[DDIR_RWDIR_CNT ];uint64_t bytes_verified; -uint32_t last_write_comp_depth;uint64_t *thinktime_blocks_counter;struct timespec last_thinktime;
@@ -782,6 +790,13 @@ extern void lat_target_check(struct thread_data *);extern void lat_target_init (struct thread_data *);extern void lat_target_reset (struct thread_data *); +/* +* Inflight log +*/ +extern void log_inflight (struct thread_data *,struct io_u *); +extern void invalidate_inflight (struct thread_data *,struct io_u *); +extern void clear_inflight (struct thread_data *); +/* * Iterates all threads/processes within all the defined jobs * Usage:
io_u.c +1 -10
@@ -2065,8 +2065,6 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,static void file_log_write_comp (const struct thread_data *td,struct fio_file *f,uint64_t offset,unsigned int bytes) { -int idx; -if (!f)return ;
@@ -2074,14 +2072,6 @@ static void file_log_write_comp(const struct thread_data *td, struct fio_file *f f->first_write = offset;if (f->last_write ==-1ULL || ((offset + bytes) > f->last_write)) f->last_write = offset + bytes; - -if (!f->last_write_comp) -return ; - - idx = f->last_write_idx++; - f->last_write_comp[idx] = offset; -if (f->last_write_idx == td->last_write_comp_depth) - f->last_write_idx =0 ; }static bool should_account (struct thread_data *td)
@@ -2101,6 +2091,7 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,assert (io_u->flags &IO_U_F_FLIGHT );io_u_clear (td, io_u,IO_U_F_FLIGHT |IO_U_F_BUSY_OK |IO_U_F_PATTERN_DONE ); +invalidate_inflight (td, io_u); if (td->o.zone_mode == ZONE_MODE_ZBD && td->o.recover_zbd_write_error && io_u->error && io_u->ddir == DDIR_WRITE &&
io_u.h +3 -1
@@ -46,7 +46,7 @@ struct io_u {/* * Write generation */ -unsigned short numberio; +uint64_t numberio;/* * IO priority.
@@ -90,6 +90,8 @@ struct io_u {unsigned long long resid;unsigned int error; +int inflight_idx; +/* * io engine private data */
ioengines.h +1 -1
@@ -9,7 +9,7 @@#include "zbd_types.h" #include "dataplacement.h" -#define FIO_IOOPS_VERSION 37 +#define FIO_IOOPS_VERSION 38#ifndef CONFIG_DYNAMIC_ENGINES #define FIO_STATIC static
iolog.h +1 -1
@@ -260,7 +260,7 @@ struct io_piece {struct fio_file *file; };unsigned long long offset; -unsigned short numberio; +uint64_t numberio;unsigned long len;unsigned int flags;enum fio_ddir ddir;
libfio.c +2 -0
@@ -128,6 +128,8 @@ void clear_io_state(struct thread_data *td, int all)*/ if (td->o.rand_repeatable)td_fill_rand_seeds (td); + +clear_inflight (td); }void reset_all_stats (struct thread_data *td)
t/verify-state.c +9 -17
@@ -21,19 +21,16 @@ static void show_s(struct thread_io_list *s, unsigned int no_s)printf ("Thread:\t\t%u\n" , no_s);printf ("Name:\t\t%s\n" , s->name); -printf ("Completions:\t%llu\n" , (unsigned long long ) s->no_comps);printf ("Depth:\t\t%llu\n" , (unsigned long long ) s->depth); -printf ("Max completions per file:\t\t%lu\n" , (unsigned long ) s->max_no_comps_per_file);printf ("Number IOs:\t%llu\n" , (unsigned long long ) s->numberio);printf ("Index:\t\t%llu\n" , (unsigned long long ) s->index); -printf ("Completions:\n" ); -if (!s->no_comps) +printf ("Inflight writes:\n" ); +if (!s->depth)return ; -for (i = s->no_comps -1 ; i >=0 ; i--) { -printf ("\t(file=%2llu) %llu\n" , - (unsigned long long ) s->comps[i].fileno, - (unsigned long long ) s->comps[i].offset); +for (i = s->depth -1 ; i >=0 ; i--) { +printf ("\t%llu\n" , + (unsigned long long ) s->inflight[i].numberio); } }
@@ -45,23 +42,18 @@ static void show(struct thread_io_list *s, size_t size)do {int i; - s->no_comps =le64_to_cpu (s->no_comps); s->depth =le32_to_cpu (s->depth); - s->max_no_comps_per_file =le32_to_cpu (s->max_no_comps_per_file); - s->nofiles =le32_to_cpu (s->nofiles); s->numberio =le64_to_cpu (s->numberio); s->index =le64_to_cpu (s->index); -for (i =0 ; i < s->no_comps; i++) { - s->comps[i].fileno =le64_to_cpu (s->comps[i].fileno); - s->comps[i].offset =le64_to_cpu (s->comps[i].offset); - } +for (i =0 ; i < s->depth; i++) + s->inflight[i].numberio =le64_to_cpu (s->inflight[i].numberio);show_s (s, no_s); no_s++; - size -=__thread_io_list_sz (s->max_no_comps_per_file, s->nofiles); + size -=__thread_io_list_sz (s->depth); s = (struct thread_io_list *)((char *) s + -__thread_io_list_sz (s->max_no_comps_per_file, s->nofiles)); +__thread_io_list_sz (s->depth)); }while (size !=0 ); }
verify-state.h +9 -15
@@ -22,24 +22,18 @@ struct thread_rand_state { }; }; -/* -* For dumping current write state -*/ -struct file_comp { -uint64_t fileno; -uint64_t offset; +/* a single inflight write */ +struct inflight_write { +uint64_t numberio; };struct thread_io_list { -uint64_t no_comps;/* Number of completions saved for the thread */ uint32_t depth;/* I/O depth of the job that saves the verify state */ -uint32_t max_no_comps_per_file; -uint32_t nofiles; -uint64_t numberio; +uint64_t numberio;/* Number of issued writes */ uint64_t index;struct thread_rand_state rand;uint8_t name[64 ]; -struct file_comp comps[0 ]; +struct inflight_write inflight[0 ]; };struct all_io_list {
@@ -63,18 +57,18 @@ extern void __verify_save_state(struct all_io_list *, const char *);extern void verify_save_state (int mask);extern int verify_load_state (struct thread_data *,const char *);extern void verify_free_state (struct thread_data *); -extern int verify_state_should_stop (struct thread_data *,struct io_u *); +extern int verify_state_should_stop (struct thread_data *,uint64_t );extern void verify_assign_state (struct thread_data *,void *);extern int verify_state_hdr (struct verify_state_hdr *,struct thread_io_list *); -static inline size_t __thread_io_list_sz (uint32_t depth,uint32_t nofiles) +static inline size_t __thread_io_list_sz (uint32_t depth) { -return sizeof (struct thread_io_list ) + depth * nofiles *sizeof (struct file_comp ); +return sizeof (struct thread_io_list ) + depth *sizeof (struct inflight_write ); }static inline size_t thread_io_list_sz (struct thread_io_list *s) { -return __thread_io_list_sz (le32_to_cpu (s->max_no_comps_per_file),le32_to_cpu (s->nofiles)); +return __thread_io_list_sz (le32_to_cpu (s->depth)); }static inline struct thread_io_list *io_list_next (struct thread_io_list *s)
verify.c +34 -84
192 lines not on this page · raw message
verify.h +9 -2
@@ -32,6 +32,12 @@ enum { VERIFY_NULL, /* pretend to verify */ }; +/* +* Set the high bit to distinguish versioned headers from older +* non-versioned headers. +*/ +#define VERIFY_HEADER_VERSION 0x81 +/* * A header structure associated with each checksummed data block. It is * followed by a checksum specific header that contains the verification
@@ -39,14 +45,15 @@ enum {*/ struct verify_header {uint16_t magic; -uint16_t verify_type; +uint8_t version; +uint8_t verify_type;uint32_t len;uint64_t rand_seed;uint64_t offset;uint32_t time_sec;uint32_t time_nsec; +uint64_t numberio;uint16_t thread; -uint16_t numberio;uint32_t crc32; };