Recent changes (master)
The following changes since commit 52a0b9ed71c3e929461e64b39059281948107071:
Merge branch 'patch-1' of https://github.com/Nikratio/fio (2022-01-28 14:50:51 -0700)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 62e9ece4d540ff2af865e4b43811f3150b8b846b:
fio: use correct function declaration for set_epoch_time() (2022-02-03 16:06:59 -0700)
----------------------------------------------------------------
David Korczynski (1):
ci/Github actions: add CIFuzz integration
Jens Axboe (6):
Merge branch 'master' of https://github.com/blah325/fio
server: fix formatting issue
Merge branch 'freebsd-comment-update' of https://github.com/macdice/fio
Merge branch 'cifuzz-integration' of https://github.com/DavidKorczynski/fio
Merge branch 'fio_pr_alternate_epoch' of https://github.com/PCPartPicker/fio
fio: use correct function declaration for set_epoch_time()
Niklas Cassel (18):
init: verify option lat_percentiles consistency for all jobs in group
backend: do ioprio_set() before calling the ioengine init callback
stat: save the default ioprio in struct thread_stat client/server: convert ss_data to use an offset instead of fixed position
stat: add a new function to allocate a clat_prio_stat array
os: define min/max prio class and level for systems without ioprio
options: add a parsing function for an additional cmdprio_bssplit format
cmdprio: add support for a new cmdprio_bssplit entry format
examples: add new cmdprio_bssplit format examples
stat: use enum fio_ddir consistently
stat: increment members counter after call to sum_thread_stats()
stat: add helper for resetting the latency buckets
stat: disable per prio stats where not needed
stat: report clat stats on a per priority granularity
stat: convert json output to a new per priority granularity format
gfio: drop support for high/low priority latency results
stat: remove unused high/low prio struct members t/latency_percentiles.py: add tests for the new cmdprio_bssplit format
Thomas Munro (1):
Update comments about availability of fdatasync().
aggieNick02 (1):
Support for alternate epochs in fio log files
james rizzo (3):
Avoid client calls to recv() without prior poll()
Add Windows support for --server.
Added a new windows only IO engine option “no_completion_thread”.
.github/workflows/cifuzz.yml | 24 ++ HOWTO | 41 +++- backend.c | 27 ++- cconv.c | 4 + client.c | 48 ++-- engines/cmdprio.c | 440 +++++++++++++++++++++++++++++------ engines/cmdprio.h | 22 +- engines/filecreate.c | 2 +- engines/filedelete.c | 2 +- engines/filestat.c | 2 +- engines/windowsaio.c | 134 +++++++++-- examples/cmdprio-bssplit.fio | 39 +++- fio.1 | 45 +++- fio.h | 2 +- fio_time.h | 2 +- gclient.c | 55 +---- init.c | 37 +++ io_u.c | 7 +- io_u.h | 3 +- libfio.c | 2 +- optgroup.h | 2 + options.c | 140 ++++++++++++ os/os-windows.h | 2 + os/os.h | 4 + os/windows/posix.c | 182 ++++++++++++++- rate-submit.c | 11 +- server.c | 369 +++++++++++++++++++++++++++--- server.h | 7 +- stat.c | 531 ++++++++++++++++++++++++++++++++++--------- stat.h | 40 +++- t/latency_percentiles.py | 211 ++++++++++------- thread_options.h | 14 ++ time.c | 12 +- 33 files changed, 2019 insertions(+), 444 deletions(-) create mode 100644 .github/workflows/cifuzz.yml
---
Diff of recent changes:
.github/workflows/cifuzz.yml new file +24 -0
@@ -0,0 +1,24 @@ +name: CIFuzz +on: [pull_request] +jobs: + Fuzzing: + runs-on: ubuntu-latest + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'fio' + dry-run: false + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'fio' + fuzz-seconds: 600 + dry-run: false + - name: Upload Crash + uses: actions/upload-artifact@v1 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts
HOWTO +36 -5
59 lines not on this page · raw message
backend.c +16 -11
@@ -1777,6 +1777,18 @@ static void *thread_main(void *data)if (!init_iolog (td)) goto err; +/* ioprio_set() has to be done before td_io_init() */ +if (fio_option_is_set (o, ioprio) || +fio_option_is_set (o, ioprio_class)) { + ret =ioprio_set (IOPRIO_WHO_PROCESS ,0 , o->ioprio_class, o->ioprio); +if (ret ==-1 ) { +td_verror (td, errno,"ioprio_set" ); + goto err; + } + td->ioprio =ioprio_value (o->ioprio_class, o->ioprio); + td->ts.ioprio = td->ioprio; + } +if (td_io_init (td)) goto err;
@@ -1789,16 +1801,6 @@ static void *thread_main(void *data)if (o->verify_async &&verify_async_init (td)) goto err; -if (fio_option_is_set (o, ioprio) || -fio_option_is_set (o, ioprio_class)) { - ret =ioprio_set (IOPRIO_WHO_PROCESS ,0 , o->ioprio_class, o->ioprio); -if (ret ==-1 ) { -td_verror (td, errno,"ioprio_set" ); - goto err; - } - td->ioprio =ioprio_value (o->ioprio_class, o->ioprio); - } -if (o->cgroup &&cgroup_setup (td, cgroup_list, &cgroup_mnt)) goto err;
@@ -1828,7 +1830,7 @@ static void *thread_main(void *data)if (rate_submit_init (td, sk_out)) goto err; -set_epoch_time (td, o->log_unix_epoch); +set_epoch_time (td, o->log_unix_epoch | o->log_alternate_epoch, o->log_alternate_epoch_clock_id);fio_getrusage (&td->ru_start);memcpy (&td->bw_sample_time, &td->epoch,sizeof (td->epoch));memcpy (&td->iops_sample_time, &td->epoch,sizeof (td->epoch));
@@ -2611,6 +2613,9 @@ int fio_backend(struct sk_out *sk_out) }for_each_td (td, i) { +struct thread_stat *ts = &td->ts; + +free_clat_prio_stats (ts);steadystate_free (td);fio_options_free (td);fio_dump_options_free (td);
cconv.c +4 -0
@@ -197,6 +197,8 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->log_gz =le32_to_cpu (top->log_gz); o->log_gz_store =le32_to_cpu (top->log_gz_store); o->log_unix_epoch =le32_to_cpu (top->log_unix_epoch); + o->log_alternate_epoch =le32_to_cpu (top->log_alternate_epoch); + o->log_alternate_epoch_clock_id =le32_to_cpu (top->log_alternate_epoch_clock_id); o->norandommap =le32_to_cpu (top->norandommap); o->softrandommap =le32_to_cpu (top->softrandommap); o->bs_unaligned =le32_to_cpu (top->bs_unaligned);
@@ -425,6 +427,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->log_gz =cpu_to_le32 (o->log_gz); top->log_gz_store =cpu_to_le32 (o->log_gz_store); top->log_unix_epoch =cpu_to_le32 (o->log_unix_epoch); + top->log_alternate_epoch =cpu_to_le32 (o->log_alternate_epoch); + top->log_alternate_epoch_clock_id =cpu_to_le32 (o->log_alternate_epoch_clock_id); top->norandommap =cpu_to_le32 (o->norandommap); top->softrandommap =cpu_to_le32 (o->softrandommap); top->bs_unaligned =cpu_to_le32 (o->bs_unaligned);
client.c +34 -14
96 lines not on this page · raw message
engines/cmdprio.c +365 -75
534 lines not on this page · raw message
engines/cmdprio.h +20 -2
@@ -17,6 +17,24 @@ enum { CMDPRIO_MODE_BSSPLIT, }; +struct cmdprio_prio { +int32_t prio; +uint32_t perc; +uint16_t clat_prio_index; +}; + +struct cmdprio_bsprio { +uint64_t bs; +uint32_t tot_perc; +unsigned int nr_prios; +struct cmdprio_prio *prios; +}; + +struct cmdprio_bsprio_desc { +struct cmdprio_bsprio *bsprios; +unsigned int nr_bsprios; +}; +struct cmdprio_options {unsigned int percentage[CMDPRIO_RWDIR_CNT ];unsigned int class[CMDPRIO_RWDIR_CNT ];
@@ -26,8 +44,8 @@ struct cmdprio_options {struct cmdprio {struct cmdprio_options *options; -unsigned int bssplit_nr[CMDPRIO_RWDIR_CNT ]; -struct bssplit *bssplit[CMDPRIO_RWDIR_CNT ]; +struct cmdprio_prio perc_entry[CMDPRIO_RWDIR_CNT ]; +struct cmdprio_bsprio_desc bsprio_desc[CMDPRIO_RWDIR_CNT ];unsigned int mode; };
engines/filecreate.c +1 -1
@@ -49,7 +49,7 @@ static int open_file(struct thread_data *td, struct fio_file *f)uint64_t nsec; nsec =ntime_since_now (&start); -add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 , false); +add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 ,0 ); }return 0 ;
engines/filedelete.c +1 -1
@@ -51,7 +51,7 @@ static int delete_file(struct thread_data *td, struct fio_file *f)uint64_t nsec; nsec =ntime_since_now (&start); -add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 , false); +add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 ,0 ); }return 0 ;
engines/filestat.c +1 -1
@@ -125,7 +125,7 @@ static int stat_file(struct thread_data *td, struct fio_file *f)uint64_t nsec; nsec =ntime_since_now (&start); -add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 , false); +add_clat_sample (td, data->stat_ddir, nsec,0 ,0 ,0 ,0 ); }return 0 ;
engines/windowsaio.c +111 -23
186 lines not on this page · raw message
examples/cmdprio-bssplit.fio +33 -6
@@ -1,17 +1,44 @@ ; Randomly read/write a block device file at queue depth 16. -; 40 % of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB. -; 100% of the 64kB reads are executed at the highest priority and -; all other IOs executed without a priority set. [global] filename=/dev/sda direct=1 write_lat_log=prio-run.log log_prio=1 - -[randrw] rw=randrw -bssplit=64k/40:1024k/60,1024k/100 ioengine=libaio iodepth=16 + +; Simple cmdprio_bssplit format. All non-zero percentage entries will +; use the same prio class and prio level defined by the cmdprio_class +; and cmdprio options. +[cmdprio] +; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB. +; 100% of the 64kB reads are executed with prio class 1 and prio level 0. +; All other IOs are executed without a priority set. +bssplit=64k/40:1024k/60,1024k/100 cmdprio_bssplit=64k/100:1024k/0,1024k/0 cmdprio_class=1 +cmdprio=0 + +; Advanced cmdprio_bssplit format. Each non-zero percentage entry can +; use a different prio class and prio level (appended to each entry). +[cmdprio-adv] +; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB. +; 25% of the 64kB reads are executed with prio class 1 and prio level 1, +; 75% of the 64kB reads are executed with prio class 3 and prio level 2. +; All other IOs are executed without a priority set. +stonewall +bssplit=64k/40:1024k/60,1024k/100 +cmdprio_bssplit=64k/25/1/1:64k/75/3/2:1024k/0,1024k/0 + +; Identical to the previous example, but with a default priority defined. +[cmdprio-adv-def] +; 40% of read IOs are 64kB and 60% are 1MB. 100% of writes are 1MB. +; 25% of the 64kB reads are executed with prio class 1 and prio level 1, +; 75% of the 64kB reads are executed with prio class 3 and prio level 2. +; All other IOs are executed with prio class 2 and prio level 7. +stonewall +prioclass=2 +prio=7 +bssplit=64k/40:1024k/60,1024k/100 +cmdprio_bssplit=64k/25/1/1:64k/75/3/2:1024k/0,1024k/0
fio.1 +40 -5
63 lines not on this page · raw message
fio.h +1 -1
@@ -380,7 +380,7 @@ struct thread_data {struct timespec start;/* start of this loop */ struct timespec epoch;/* time job was started */ -unsigned long long unix_epoch;/* Time job was started, unix epoch based. */ +unsigned long long alternate_epoch;/* Time job was started, clock_gettime's clock_id epoch based. */ struct timespec last_issue;long time_offset;struct timespec ts_cache;
fio_time.h +1 -1
@@ -30,6 +30,6 @@ extern bool ramp_time_over(struct thread_data *);extern bool in_ramp_time (struct thread_data *);extern void fio_time_init (void );extern void timespec_add_msec (struct timespec *,unsigned int ); -extern void set_epoch_time (struct thread_data *,int ); +extern void set_epoch_time (struct thread_data *,int ,clockid_t );#endif
gclient.c +4 -51
103 lines not on this page · raw message
init.c +37 -0
@@ -224,6 +224,13 @@ static struct option l_opts[FIO_NR_OPTIONS] = { .has_arg = optional_argument, .val = 'S', }, +#ifdef WIN32 + { + .name = (char *) "server-internal", + .has_arg = required_argument, + .val = 'N', + }, +#endif { .name = (char *) "daemonize", .has_arg = required_argument, .val = 'D',
@@ -1445,6 +1452,26 @@ static bool wait_for_ok(const char *jobname, struct thread_options *o)return true; } +static int verify_per_group_options (struct thread_data *td,const char *jobname) +{ +struct thread_data *td2; +int i; + +for_each_td (td2, i) { +if (td->groupid != td2->groupid) +continue ; + +if (td->o.stats && + td->o.lat_percentiles != td2->o.lat_percentiles) { +log_err ("fio: lat_percentiles in job: %s differs from group\n" , + jobname); +return 1 ; + } + } + +return 0 ; +} +/* * Treat an empty log file name the same as a one not given */
@@ -1563,6 +1590,10 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, td->groupid = groupid; prev_group_jobs++; +if (td->o.group_reporting && prev_group_jobs >1 && +verify_per_group_options (td, jobname)) + goto err; +if (setup_rate (td)) goto err;
@@ -2795,6 +2826,12 @@ int parse_cmd_line(int argc, char *argv[], int client_type) exit_val =1 ;#endif break ; +#ifdef WIN32 +case 'N' : + did_arg = true; +fio_server_internal_set (optarg); +break ; +#endif case 'D' :if (pid_file)free (pid_file);
io_u.c +4 -3
@@ -1595,7 +1595,7 @@ again:assert (io_u->flags &IO_U_F_FREE );io_u_clear (td, io_u,IO_U_F_FREE |IO_U_F_NO_FILE_PUT |IO_U_F_TRIMMED |IO_U_F_BARRIER | -IO_U_F_VER_LIST |IO_U_F_HIGH_PRIO ); +IO_U_F_VER_LIST ); io_u->error =0 ; io_u->acct_ddir =-1 ;
@@ -1803,6 +1803,7 @@ struct io_u *get_io_u(struct thread_data *td)* Remember the issuing context priority. The IO engine may change this. */ io_u->ioprio = td->ioprio; + io_u->clat_prio_index =0 ; out:assert (io_u->file);if (!td_io_prep (td, io_u)) {
@@ -1889,7 +1890,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, tnsec =ntime_since (&io_u->start_time, &icd->time);add_lat_sample (td, idx, tnsec, bytes, io_u->offset, - io_u->ioprio,io_u_is_high_prio (io_u)); + io_u->ioprio, io_u->clat_prio_index);if (td->flags &TD_F_PROFILE_OPS ) {struct prof_io_ops *ops = &td->prof_io_ops;
@@ -1911,7 +1912,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,if (ddir_rw (idx)) {if (!td->o.disable_clat) {add_clat_sample (td, idx, llnsec, bytes, io_u->offset, - io_u->ioprio,io_u_is_high_prio (io_u)); + io_u->ioprio, io_u->clat_prio_index);io_u_mark_latency (td, llnsec); }
io_u.h +1 -2
@@ -21,7 +21,6 @@ enum {IO_U_F_TRIMMED =1 <<5 ,IO_U_F_BARRIER =1 <<6 , IO_U_F_VER_LIST = 1 << 7, - IO_U_F_HIGH_PRIO = 1 << 8, };/*
@@ -50,6 +49,7 @@ struct io_u {* IO priority. */ unsigned short ioprio; +unsigned short clat_prio_index;/* * Allocated/set buffer and length
@@ -193,6 +193,5 @@ static inline enum fio_ddir acct_ddir(struct io_u *io_u) td_flags_clear((td), &(io_u->flags), (val))#define io_u_set (td, io_u, val) \ td_flags_set((td), &(io_u)->flags, (val)) -#define io_u_is_high_prio (io_u) (io_u->flags & IO_U_F_HIGH_PRIO)#endif
libfio.c +1 -1
@@ -142,7 +142,7 @@ void reset_all_stats(struct thread_data *td) td->ts.runtime[i] =0 ; } -set_epoch_time (td, td->o.log_unix_epoch); +set_epoch_time (td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id);memcpy (&td->start, &td->epoch,sizeof (td->epoch));memcpy (&td->iops_sample_time, &td->epoch,sizeof (td->epoch));memcpy (&td->bw_sample_time, &td->epoch,sizeof (td->epoch));
optgroup.h +2 -0
@@ -71,6 +71,7 @@ enum opt_category_group { __FIO_OPT_G_LIBCUFILE, __FIO_OPT_G_DFS, __FIO_OPT_G_NFS, + __FIO_OPT_G_WINDOWSAIO,FIO_OPT_G_RATE = (1ULL << __FIO_OPT_G_RATE), FIO_OPT_G_ZONE = (1ULL << __FIO_OPT_G_ZONE),
@@ -116,6 +117,7 @@ enum opt_category_group {FIO_OPT_G_FILESTAT = (1ULL << __FIO_OPT_G_FILESTAT),FIO_OPT_G_LIBCUFILE = (1ULL << __FIO_OPT_G_LIBCUFILE),FIO_OPT_G_DFS = (1ULL << __FIO_OPT_G_DFS), + FIO_OPT_G_WINDOWSAIO = (1ULL << __FIO_OPT_G_WINDOWSAIO), };extern const struct opt_group *opt_group_from_mask (uint64_t *mask);
options.c +140 -0
152 lines not on this page · raw message
os/os-windows.h +2 -0
@@ -110,6 +110,8 @@ int nanosleep(const struct timespec *rqtp, struct timespec *rmtp);ssize_t pread (int fildes,void *buf,size_t nbyte,off_t offset);ssize_t pwrite (int fildes,const void *buf,size_t nbyte,off_t offset); +HANDLE windows_handle_connection (HANDLE hjob,int sk); +HANDLE windows_create_job (void );static inline int blockdev_size (struct fio_file *f,unsigned long long *bytes) {
os/os.h +4 -0
@@ -119,10 +119,14 @@ extern int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu);#ifndef FIO_HAVE_IOPRIO_CLASS #define ioprio_value_is_class_rt (prio) (false) +#define IOPRIO_MIN_PRIO_CLASS 0 +#define IOPRIO_MAX_PRIO_CLASS 0#endif #ifndef FIO_HAVE_IOPRIO #define ioprio_value (prioclass, prio) (0)#define ioprio_set (which, who, prioclass, prio) (0) +#define IOPRIO_MIN_PRIO 0 +#define IOPRIO_MAX_PRIO 0#endif #ifndef FIO_HAVE_ODIRECT
os/windows/posix.c +179 -3
199 lines not on this page · raw message
rate-submit.c +10 -1
@@ -173,7 +173,7 @@ static int io_workqueue_init_worker_fn(struct submit_worker *sw)if (td->io_ops->post_init && td->io_ops->post_init (td)) goto err_io_init; -set_epoch_time (td, td->o.log_unix_epoch); +set_epoch_time (td, td->o.log_unix_epoch | td->o.log_alternate_epoch, td->o.log_alternate_epoch_clock_id);fio_getrusage (&td->ru_start);clear_io_state (td,1 );
@@ -195,6 +195,15 @@ static void io_workqueue_exit_worker_fn(struct submit_worker *sw,struct thread_data *td = sw->priv; (*sum_cnt)++; + +/* +* io_workqueue_update_acct_fn() doesn't support per prio stats, and +* even if it did, offload can't be used with all async IO engines. +* If group reporting is set in the parent td, the group result +* generated by __show_run_stats() can still contain multiple prios +* from different offloaded jobs. +*/ + sw->wq->td->ts.disable_prio_stat =1 ;sum_thread_stats (&sw->wq->td->ts, &td->ts);fio_options_free (td);
server.c +332 -37
493 lines not on this page · raw message
server.h +6 -1
@@ -15,6 +15,9 @@ struct sk_out {unsigned int refs;/* frees sk_out when it drops to zero. * protected by below ->lock */ +#ifdef WIN32 +HANDLE hProcess;/* process handle of handle_connection_process*/ +#endif int sk;/* socket fd to talk to client */ struct fio_sem lock;/* protects ref and below list */ struct flist_head list;/* list of pending transmit work */
@@ -48,7 +51,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =95 , +FIO_SERVER_VER =96 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
@@ -212,6 +215,7 @@ extern int fio_server_text_output(int, const char *, size_t);extern int fio_net_send_cmd (int ,uint16_t ,const void *,off_t ,uint64_t *,struct flist_head *);extern int fio_net_send_simple_cmd (int ,uint16_t ,uint64_t ,struct flist_head *);extern void fio_server_set_arg (const char *); +extern void fio_server_internal_set (const char *);extern int fio_server_parse_string (const char *,char **,bool *,int *,struct in_addr *,struct in6_addr *,int *);extern int fio_server_parse_host (const char *,int ,struct in_addr *,struct in6_addr *);extern const char *fio_server_op (unsigned int );
@@ -222,6 +226,7 @@ extern void fio_server_send_gs(struct group_run_stats *);extern void fio_server_send_du (void );extern void fio_server_send_job_options (struct flist_head *,unsigned int );extern int fio_server_get_verify_state (const char *,int ,void **); +extern bool fio_server_poll_fd (int fd,short events,int timeout);extern struct fio_net_cmd *fio_net_recv_cmd (int sk,bool wait);
stat.c +421 -110
691 lines not on this page · raw message
t/latency_percentiles.py +134 -77
211 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Part of the diff is not shown here. View the raw message.