Recent changes (master)
The following changes since commit 72f397487788fc0b542870de5cc29ea8e8134346:
Make options mask a 64-bit type (2015-12-21 16:03:01 -0700)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 5c74fc767600f75cd6d53bdc5926962b75a614ae:
backend: continue on failed wait_for() check, not break (2015-12-22 13:40:14 -0700)
----------------------------------------------------------------
Andrey Kuzmin (1):
Add support for waiting for other jobs by name
Fabrice Bacchella (1):
rewrote libhdfs engine
Jens Axboe (3):
parse: ensure that option group/category hold 64-bit
gfio: update for 64-bit option category/group
backend: continue on failed wait_for() check, not break
HOWTO | 31 ++-- backend.c | 32 ++++ cconv.c | 3 + engines/libhdfs.c | 431 ++++++++++++++++++++++++++++++++++++--------------- examples/waitfor.fio | 35 +++++ file.h | 1 + filesetup.c | 2 +- fio.1 | 8 + fio.h | 1 + goptions.c | 8 +- init.c | 49 ++++++ libfio.c | 2 +- options.c | 9 ++ options.h | 2 + parse.h | 5 +- server.h | 2 +- thread_options.h | 2 + 17 files changed, 476 insertions(+), 147 deletions(-) create mode 100644 examples/waitfor.fio
---
Diff of recent changes:
HOWTO +20 -11
@@ -305,6 +305,16 @@ name=str ASCII name of the job. This may be used to override the special purpose of also signaling the start of a new job. +wait_for=str Specifies the name of the already defined job to wait + for. Single waitee name only may be specified. If set, the job + won't be started until all workers of the waitee job are done. + + Wait_for operates on the job name basis, so there are a few + limitations. First, the waitee must be defined prior to the + waiter job (meaning no forward references). Second, if a job + is being referenced as a waitee, it must have a unique name + (no duplicate waitees). + description=str Text description of the job. Doesn't do anything except dump this text description when this job is run. It's not parsed.
@@ -767,20 +777,14 @@ ioengine=str Defines how the job issues io to the file. The following defines engine specific options. libhdfs Read and write through Hadoop (HDFS). - The 'filename' option is used to specify host, - port of the hdfs name-node to connect. This - engine interprets offsets a little + This engine interprets offsets a little differently. In HDFS, files once created cannot be modified. So random writes are not possible. To imitate this, libhdfs engine - expects bunch of small files to be created - over HDFS, and engine will randomly pick a - file out of those files based on the offset - generated by fio backend. (see the example - job file to create such files, use rw=write - option). Please note, you might want to set - necessary environment variables to work with - hdfs/libhdfs properly. + creates bunch of small files, and engine will + pick a file out of those files based on the + offset enerated by fio backend. Each jobs uses + it's own connection to HDFS. mtd Read, write and erase an MTD character device (e.g., /dev/mtd0). Discards are treated as
@@ -1785,11 +1789,13 @@ that defines them is selected. If the job is a TCP listener or UDP reader, the hostname is not used and must be omitted unless it is a valid UDP multicast address. +[libhdfs] namenode=str The host name or IP address of a HDFS cluster namenode to contact. [netsplice] port=int [net] port=int The TCP or UDP port to bind to or connect to. If this is used with numjobs to spawn multiple instances of the same job type, then this will be the starting port number since fio will use a range of ports. +[libhdfs] port=int the listening port of the HFDS cluster namenode. [netsplice] interface=str [net] interface=str The IP address of the network interface used to send or
@@ -1848,6 +1854,9 @@ be the starting port number since fio will use a range of ports. [mtd] skip_bad=bool Skip operations against known bad blocks. +[libhdfs] hdfsdirectory libhdfs will create chunk in this HDFS directory +[libhdfs] chunck_size the size of the chunck to use for each file. + 6.0 Interpreting the output ---------------------------
backend.c +32 -0
@@ -1978,6 +1978,32 @@ mounted:return true; } +static bool waitee_running (struct thread_data *me) +{ +const char *waitee = me->o.wait_for; +const char *self = me->o.name; +struct thread_data *td; +int i; + +if (!waitee) +return false; + +for_each_td (td, i) { +if (!strcmp (td->o.name, self) ||strcmp (td->o.name, waitee)) +continue ; + +if (td->runstate <TD_EXITED ) { +dprint (FD_PROCESS ,"%s fenced by %s(%s)\n" , + self, td->o.name, +runstate_to_name (td->runstate)); +return true; + } + } + +dprint (FD_PROCESS ,"%s: %s completed, can run\n" , self, waitee); +return false; +} +/* * Main function for kicking off and reaping jobs, as needed. */
@@ -2101,6 +2127,12 @@ reap:break ; } +if (waitee_running (td)) { +dprint (FD_PROCESS ,"%s: waiting for %s\n" , + td->o.name, td->o.wait_for); +continue ; + } +init_disk_util (td); td->rusage_sem =fio_mutex_init (FIO_MUTEX_LOCKED );
cconv.c +3 -0
@@ -25,6 +25,7 @@ static void free_thread_options_to_cpu(struct thread_options *o) {free (o->description);free (o->name); +free (o->wait_for);free (o->directory);free (o->filename);free (o->filename_format);
@@ -54,6 +55,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,string_to_cpu (&o->description, top->description);string_to_cpu (&o->name, top->name); +string_to_cpu (&o->wait_for, top->wait_for);string_to_cpu (&o->directory, top->directory);string_to_cpu (&o->filename, top->filename);string_to_cpu (&o->filename_format, top->filename_format);
@@ -276,6 +278,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,string_to_net (top->description, o->description);string_to_net (top->name, o->name); +string_to_net (top->wait_for, o->wait_for);string_to_net (top->directory, o->directory);string_to_net (top->filename, o->filename);string_to_net (top->filename_format, o->filename_format);
engines/libhdfs.c +304 -127
526 lines not on this page · raw message
examples/waitfor.fio new file +35 -0
@@ -0,0 +1,35 @@ +[global] +threads=1 +group_reporting=1 +filename=/tmp/data +filesize=128m + +[writers] +rw=write +bs=128k +numjobs=4 +runtime=10 + +[readers] +new_group +wait_for=writers +rw=randread +bs=4k +numjobs=4 +runtime=10 + +[writers2] +new_group +wait_for=readers +rw=randwrite +bs=4k +numjobs=4 +runtime=10 + +[readers2] +new_group +wait_for=writers2 +rw=randread +bs=4k +numjobs=4 +runtime=10
file.h +1 -0
@@ -180,6 +180,7 @@ extern int __must_check generic_close_file(struct thread_data *, struct fio_fileextern int __must_check generic_get_file_size (struct thread_data *,struct fio_file *);extern int __must_check file_lookup_open (struct fio_file *f,int flags);extern int __must_check pre_read_files (struct thread_data *); +extern unsigned long long get_rand_file_size (struct thread_data *td);extern int add_file (struct thread_data *,const char *,int ,int );extern int add_file_exclusive (struct thread_data *,const char *);extern void get_file (struct fio_file *);
filesetup.c +1 -1
@@ -264,7 +264,7 @@ error:return ret; } -static unsigned long long get_rand_file_size (struct thread_data *td) +unsigned long long get_rand_file_size (struct thread_data *td) {unsigned long long ret, sized;uint64_t frand_max;
fio.1 +8 -0
@@ -180,6 +180,14 @@ a ':' character. May be used to override the job name. On the command line, this parameter has the special purpose of signalling the start of a new job. .TP +.BI wait_for \fR=\fPstr +Specifies the name of the already defined job to wait for. Single waitee name +only may be specified. If set, the job won't be started until all workers of +the waitee job are done. Wait_for operates on the job name basis, so there are +a few limitations. First, the waitee must be defined prior to the waiter job +(meaning no forward references). Second, if a job is being referenced as a +waitee, it must have a unique name (no duplicate waitees). +.TP .BI description \fR=\fPstr Human-readable description of the job. It is printed when the job is run, but otherwise has no special purpose.
fio.h +1 -0
@@ -542,6 +542,7 @@ enum {extern void td_set_runstate (struct thread_data *,int );extern int td_bump_runstate (struct thread_data *,int );extern void td_restore_runstate (struct thread_data *,int ); +extern const char *runstate_to_name (int runstate);/* * Allow 60 seconds for a job to quit on its own, otherwise reap with
goptions.c +4 -4
@@ -92,9 +92,9 @@ struct gopt_job_view {static GNode *gopt_dep_tree;static GtkWidget *gopt_get_group_frame (struct gopt_job_view *gjv, -GtkWidget *box,unsigned int groupmask) +GtkWidget *box,uint64_t groupmask) { -unsigned int mask, group; +uint64_t mask, group;struct opt_group *og;GtkWidget *frame, *hbox;struct gopt_frame_widget *gfw;
@@ -1135,7 +1135,7 @@ static void gopt_add_options(struct gopt_job_view *gjv,*/ for (i =0 ; fio_options[i].name; i++) {struct fio_option *o = &fio_options[i]; -unsigned int mask = o->category; +uint64_t mask = o->category;struct opt_group *og;while ((og =opt_group_from_mask (&mask)) !=NULL ) {
@@ -1189,7 +1189,7 @@ static void gopt_add_group_tabs(GtkWidget *notebook, struct gopt_job_view *gjv) i =0 ; do { - unsigned int mask = (1U << i); + uint64_t mask = (1ULL << i); og = opt_group_from_mask(&mask); if (!og)
init.c +49 -0
61 lines not on this page · raw message
libfio.c +1 -1
@@ -190,7 +190,7 @@ static const char *td_runstates[] = { "REAPED", }; -static const char *runstate_to_name (int runstate) +const char *runstate_to_name (int runstate) {compiletime_assert (TD_LAST ==12 ,"td runstate list" );if (runstate >=0 && runstate <TD_LAST )
options.c +9 -0
@@ -1229,6 +1229,15 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_DESC, }, { + .name = "wait_for", + .lname = "Waitee name", + .type = FIO_OPT_STR_STORE, + .off1 = td_var_offset(wait_for), + .help = "Name of the job this one wants to wait for before starting", + .category = FIO_OPT_C_GENERAL, + .group = FIO_OPT_G_DESC, + }, + { .name = "filename", .lname = "Filename(s)", .type = FIO_OPT_STR_STORE,
options.h +2 -0
@@ -116,6 +116,7 @@ enum opt_category_group { __FIO_OPT_G_RBD, __FIO_OPT_G_GFAPI, __FIO_OPT_G_MTD, + __FIO_OPT_G_HDFS, __FIO_OPT_G_NR, FIO_OPT_G_RATE = (1ULL << __FIO_OPT_G_RATE),
@@ -149,6 +150,7 @@ enum opt_category_group {FIO_OPT_G_RBD = (1ULL << __FIO_OPT_G_RBD),FIO_OPT_G_GFAPI = (1ULL << __FIO_OPT_G_GFAPI),FIO_OPT_G_MTD = (1ULL << __FIO_OPT_G_MTD), +FIO_OPT_G_HDFS = (1ULL << __FIO_OPT_G_HDFS), FIO_OPT_G_INVALID = (1ULL << __FIO_OPT_G_NR), };
parse.h +3 -2
@@ -1,6 +1,7 @@#ifndef FIO_PARSE_H #define FIO_PARSE_H +#include <inttypes.h> #include "flist.h" /*
@@ -69,8 +70,8 @@ struct fio_option {int (*verify)(struct fio_option *,void *);const char *prof_name;/* only valid for specific profile */ void *prof_opts; -unsigned int category;/* what type of option */ -unsigned int group;/* who to group with */ +uint64_t category;/* what type of option */ +uint64_t group;/* who to group with */ void *gui_data;int is_seconds;/* time value with seconds base */ int is_time;/* time based value */
server.h +1 -1
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =50 , +FIO_SERVER_VER =51 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +2 -0
@@ -40,6 +40,7 @@ struct thread_options {uint64_t set_options[NR_OPTS_SZ ];char *description;char *name; +char *wait_for;char *directory;char *filename;char *filename_format;
@@ -289,6 +290,7 @@ struct thread_options_pack {uint64_t set_options[NR_OPTS_SZ ];uint8_t description[FIO_TOP_STR_MAX ];uint8_t name[FIO_TOP_STR_MAX ]; +uint8_t wait_for[FIO_TOP_STR_MAX ];uint8_t directory[FIO_TOP_STR_MAX ];uint8_t filename[FIO_TOP_STR_MAX ];uint8_t filename_format[FIO_TOP_STR_MAX ];