Recent changes (master)
The following changes since commit 477bd5ab362b65fe9c7749735236e6ed0d4346f8:
server: bump version (2015-05-06 15:21:19 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 81f3d8e9b0c7e178fc308e6416fade2756898fed:
Fio 2.2.8 (2015-05-07 18:19:25 -0600)
----------------------------------------------------------------
Ben England (3):
allow --client to work with shared filesystem
allow --client parameter to be pathname containing client host IPs/names document changes to --client syntax and behavior
Bruce Cran (3):
Add Windows ctime_r implementation and add empty ioctl.h header Fix compiler warning and test progs linker errors on Windows Fix Windows installer build: LICENSE is now MORAL-LICENSE
Jens Axboe (8):
Merge branch 'multiclnt-sharedfs-try3' of git://github.com/bengland2/fio
Ensure that set_name_idx() doesn't exceed target bounds
posixaio: use 'errno' for inline error
Merge branch 'client-hostfile' of git://github.com/bengland2/fio
Fix keyword replacement leaks
README: add git:// for github as well
Use PATH_MAX instead of _POSIX_HOST_NAME_MAX
Fio 2.2.8
FIO-VERSION-GEN | 2 +- Makefile | 6 ++++++ README | 30 ++++++++++++++++++++++++++++-- engines/posixaio.c | 2 +- filesetup.c | 2 +- fio.1 | 31 +++++++++++++++++++++++++++++-- fio.c | 14 ++++++++++---- fio.h | 2 ++ init.c | 29 +++++++++++++++++++++++++++++ libfio.c | 5 +++++ options.c | 25 +++++++++++++++++++++++-- options.h | 4 +++- os/os-windows.h | 1 + os/windows/install.wxs | 6 +++--- os/windows/posix.c | 24 ++++++++++++++++++++++++ os/windows/posix/include/sys/ioctl.h | 7 +++++++ server.c | 1 + 17 files changed, 174 insertions(+), 17 deletions(-) create mode 100644 os/windows/posix/include/sys/ioctl.h
---
Diff of recent changes:
FIO-VERSION-GEN +1 -1
@@ -1,7 +1,7 @@ #!/bin/sh GVF=FIO-VERSION-FILE -DEF_VER=fio-2.2.7 +DEF_VER=fio-2.2.8 LF=' '
Makefile +6 -0
@@ -218,6 +218,12 @@ T_OBJS += $(T_LFSR_TEST_OBJS) T_OBJS += $(T_BTRACE_FIO_OBJS) T_OBJS += $(T_DEDUPE_OBJS) +ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS))) + T_DEDUPE_OBJS += os/windows/posix.o lib/hweight.o + T_SMALLOC_OBJS += os/windows/posix.o lib/hweight.o + T_LFSR_TEST_OBJS += os/windows/posix.o lib/hweight.o +endif + T_TEST_PROGS = $(T_SMALLOC_PROGS) T_TEST_PROGS += $(T_IEEE_PROGS) T_PROGS += $(T_ZIPF_PROGS)
README +28 -2
@@ -35,6 +35,7 @@ down for some reason, either one of those is safe to use: or + git://github.com/axboe/fio.git https://github.com/axboe/fio.git
@@ -172,7 +173,7 @@ $ fio --warnings-fatal Fio parser warnings are fatal --max-jobs Maximum number of threads/processes to support --server=args Start backend server. See Client/Server section. - --client=host Connect to specified backend. + --client=host Connect to specified backend(s). --remote-config=file Tell fio server to load this local file --idle-prof=option Report cpu idleness on a system or percpu basis (option=system,percpu) or run unit work
@@ -311,9 +312,34 @@ to load a local file as well. This is done by using --remote-config: fio --client=server --remote-config /path/to/file.fio -Then the fio server will open this local (to the server) job file instead +Then fio will open this local (to the server) job file instead of being passed one from the client. +If you have many servers (example: 100 VMs/containers), +you can input a pathname of a file containing host IPs/names as the parameter +value for the --client option. For example, here is an example "host.list" +file containing 2 hostnames: + +host1.your.dns.domain +host2.your.dns.domain + +The fio command would then be: + +fio --client=host.list <job file(s)> + +In this mode, you cannot input server-specific parameters or job files -- all +servers receive the same job file. + +In order to let fio --client runs use a shared filesystem +from multiple hosts, fio --client now prepends the IP address of the +server to the filename. For example, if fio is using directory /mnt/nfs/fio +and is writing filename fileio.tmp, with a --client hostfile containing +two hostnames h1 and h2 with IP addresses 192.168.10.120 and 192.168.10.121, +then fio will create two files: + + /mnt/nfs/fio/192.168.10.120.fileio.tmp + /mnt/nfs/fio/192.168.10.121.fileio.tmp + Platforms ---------
engines/posixaio.c +1 -1
@@ -198,7 +198,7 @@ static int fio_posixaio_queue(struct thread_data *td, }if (ret) { -int aio_err =aio_error (aiocb); +int aio_err = errno;/* * At least OSX has a very low limit on the number of pending
filesetup.c +1 -1
@@ -1262,7 +1262,7 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc)dprint (FD_FILE ,"add file %s\n" , fname);if (td->o.directory) - len =set_name_idx (file_name, td->o.directory, numjob); + len =set_name_idx (file_name,PATH_MAX , td->o.directory, numjob);sprintf (file_name + len,"%s" , fname);
fio.1 +29 -2
@@ -93,7 +93,7 @@ Start a backend server, with \fIargs\fP specifying what to listen to. See client Background a fio server, writing the pid to the given pid file. .TP .BI \-\-client \fR=\fPhost -Instead of running the jobs locally, send and run them on the given host. +Instead of running the jobs locally, send and run them on the given host or set of hosts. See client/server section. .TP .BI \-\-idle\-prof \fR=\fPoption Report cpu idleness on a system or percpu basis (\fIoption\fP=system,percpu) or run unit work calibration only (\fIoption\fP=calibrate).
@@ -1911,8 +1911,35 @@ to load a local file as well. This is done by using \-\-remote-config: fio \-\-client=server \-\-remote-config /path/to/file.fio -Then the fio serer will open this local (to the server) job file instead +Then fio will open this local (to the server) job file instead of being passed one from the client. + +If you have many servers (example: 100 VMs/containers), you can input a pathname +of a file containing host IPs/names as the parameter value for the \-\-client option. +For example, here is an example "host.list" file containing 2 hostnames: + +host1.your.dns.domain +.br +host2.your.dns.domain + +The fio command would then be: + +fio \-\-client=host.list <job file> + +In this mode, you cannot input server-specific parameters or job files, and all +servers receive the same job file. + +In order to enable fio \-\-client runs utilizing a shared filesystem from multiple hosts, +fio \-\-client now prepends the IP address of the server to the filename. For example, +if fio is using directory /mnt/nfs/fio and is writing filename fileio.tmp, +with a \-\-client hostfile +containing two hostnames h1 and h2 with IP addresses 192.168.10.120 and 192.168.10.121, then +fio will create two files: + +/mnt/nfs/fio/192.168.10.120.fileio.tmp +.br +/mnt/nfs/fio/192.168.10.121.fileio.tmp + .SH AUTHORS .B fio
fio.c +10 -4
@@ -30,6 +30,8 @@int main (int argc,char *argv[],char *envp[]) { +int ret =1 ; +if (initialize_fio (envp))return 1 ;
@@ -38,7 +40,7 @@ int main(int argc, char *argv[], char *envp[])#endif if (parse_options (argc, argv)) -return 1 ; + goto done;fio_time_init ();
@@ -46,8 +48,12 @@ int main(int argc, char *argv[], char *envp[])set_genesis_time ();if (fio_start_all_clients ()) -return 1 ; -return fio_handle_clients (&fio_client_ops); + goto done; + ret =fio_handle_clients (&fio_client_ops); }else -return fio_backend (); + ret =fio_backend (); + +done: +deinitialize_fio (); +return ret; }
fio.h +2 -0
@@ -464,6 +464,7 @@ extern void reset_fio_state(void);extern void clear_io_state (struct thread_data *);extern int fio_options_parse (struct thread_data *,char **,int ,int );extern void fio_keywords_init (void ); +extern void fio_keywords_exit (void );extern int fio_cmd_option_parse (struct thread_data *,const char *,char *);extern int fio_cmd_ioengine_option_parse (struct thread_data *,const char *,char *);extern void fio_fill_default_options (struct thread_data *);
@@ -483,6 +484,7 @@ extern int fio_set_fd_nonblocking(int, const char *);extern uintptr_t page_mask;extern uintptr_t page_size;extern int initialize_fio (char *envp[]); +extern void deinitialize_fio (void );#define FIO_GETOPT_JOB 0x89000000#define FIO_GETOPT_IOENGINE 0x98000000
init.c +29 -0
@@ -2242,6 +2242,35 @@ int parse_cmd_line(int argc, char *argv[], int client_type) exit_val =1 ;break ; } +/* if --client parameter contains a pathname */ +if (0 ==access (optarg,R_OK )) { +/* file contains a list of host addrs or names */ +char hostaddr[PATH_MAX ] = {0 }; +char formatstr[8 ]; +FILE * hostf =fopen (optarg,"r" ); +if (!hostf) { +log_err ("fio: could not open client list file %s for read\n" , optarg); + do_exit++; + exit_val =1 ; +break ; + } +sprintf (formatstr,"%%%ds" ,PATH_MAX -1 ); +/* +* read at most PATH_MAX-1 chars from each +* record in this file +*/ +while (fscanf (hostf, formatstr, hostaddr) ==1 ) { +/* expect EVERY host in file to be valid */ +if (fio_client_add (&fio_client_ops, hostaddr, &cur_client)) { +log_err ("fio: failed adding client %s from file %s\n" , hostaddr, optarg); + do_exit++; + exit_val =1 ; +break ; + } + } +fclose (hostf); +break ;/* no possibility of job file for "this client only" */ + }if (fio_client_add (&fio_client_ops, optarg, &cur_client)) {log_err ("fio: failed adding client %s\n" , optarg); do_exit++;
libfio.c +5 -0
@@ -353,3 +353,8 @@ int initialize_fio(char *envp[])fio_keywords_init ();return 0 ; } + +void deinitialize_fio (void ) +{ +fio_keywords_exit (); +}
options.c +23 -2
@@ -8,6 +8,7 @@#include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> +#include <netinet/in.h> #include "fio.h" #include "verify.h"
@@ -17,6 +18,8 @@#include "crc/crc32c.h" +char client_sockaddr_str[INET6_ADDRSTRLEN ] = {0 }; +/* * Check if mmap/mmaphuge has a :/foo/bar/file at the end. If so, return that. */
@@ -820,7 +823,7 @@ static int get_max_name_idx(char *input)* Returns the directory at the index, indexes > entires will be * assigned via modulo division of the index */ -int set_name_idx (char *target,char *input,int index) +int set_name_idx (char *target,size_t tlen,char *input,int index) {unsigned int cur_idx;int len;
@@ -832,7 +835,13 @@ int set_name_idx(char *target, char *input, int index)for (cur_idx =0 ; cur_idx <= index; cur_idx++) fname =get_next_name (&str); - len =sprintf (target,"%s/" , fname); +if (client_sockaddr_str[0 ]) { + len =snprintf (target, tlen,"%s/%s." , fname, + client_sockaddr_str); + }else + len =snprintf (target, tlen,"%s/" , fname); + + target[tlen -1 ] ='\0' ;free (p);return len;
@@ -3813,6 +3822,18 @@ static struct fio_keyword fio_keywords[] = { }, }; +void fio_keywords_exit (void ) +{ +struct fio_keyword *kw; + + kw = &fio_keywords[0 ]; +while (kw->word) { +free (kw->replace); + kw->replace =NULL ; + kw++; + } +} +void fio_keywords_init (void ) {unsigned long long mb_memory;
options.h +3 -1
@@ -18,7 +18,9 @@ void del_opt_posval(const char *, const char *);struct thread_data ;void fio_options_free (struct thread_data *);char *get_name_idx (char *,int ); -int set_name_idx (char *,char *,int ); +int set_name_idx (char *,size_t ,char *,int ); + +extern char client_sockaddr_str[];/* used with --client option */ extern struct fio_option fio_options[FIO_MAX_OPTS ];
os/os-windows.h +1 -0
@@ -105,6 +105,7 @@ int fcntl(int fildes, int cmd, ...);int fdatasync (int fildes);int lstat (const char * path,struct stat * buf);uid_t geteuid (void ); +char *ctime_r (const time_t *t,char *buf);int nanosleep (const struct timespec *rqtp,struct timespec *rmtp);ssize_t pread (int fildes,void *buf,size_t nbyte,off_t offset); ssize_t pwrite(int fildes, const void *buf, size_t nbyte,
os/windows/install.wxs +3 -3
@@ -10,7 +10,7 @@ <Product Id="*" Codepage="1252" Language="1033" Manufacturer="fio" Name="fio" - UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.2.7"> + UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.2.8"> <Package Description="Flexible IO Tester" InstallerVersion="301" Keywords="Installer,MSI,Database"
@@ -40,7 +40,7 @@ <File Id="COPYING" Name="COPYING.txt" Source="..\..\COPYING"/> </Component> <Component> - <File Id="LICENSE" Name="LICENSE.txt" Source="..\..\LICENSE"/> + <File Id="MORAL_LICENSE" Name="MORAL-LICENSE.txt" Source="..\..\MORAL-LICENSE"/> </Component> <Directory Id="examples" Name="examples"/> </Directory>
@@ -54,7 +54,7 @@ <ComponentRef Id="README"/> <ComponentRef Id="REPORTING_BUGS"/> <ComponentRef Id="COPYING"/> - <ComponentRef Id="LICENSE"/> + <ComponentRef Id="MORAL_LICENSE"/> <ComponentGroupRef Id="examples"/> </Feature>
os/windows/posix.c +24 -0
@@ -229,6 +229,30 @@ char *dlerror(void)return dl_error; } +/* Copied from http://blogs.msdn.com/b/joshpoley/archive/2007/12/19/date-time-formats-and-conversions.aspx */ +void Time_tToSystemTime (time_t dosTime,SYSTEMTIME *systemTime) +{ +LARGE_INTEGER jan1970FT; +LARGE_INTEGER utcFT; + jan1970FT.QuadPart =116444736000000000LL ;// january 1st 1970 + utcFT.QuadPart = ((unsigned __int64 )dosTime) *10000000 + jan1970FT.QuadPart; + +FileTimeToSystemTime ((FILETIME *)&utcFT, systemTime); +} + +char *ctime_r (const time_t *t,char *buf) +{ +SYSTEMTIME systime; +const char *const dayOfWeek[] = {"Mon" ,"Tue" ,"Wed" ,"Thu" ,"Fri" ,"Sat" ,"Sun" }; +const char *const monthOfYear[] = {"Jan" ,"Feb" ,"Mar" ,"Apr" ,"May" ,"Jun" ,"Jul" ,"Aug" ,"Sep" ,"Oct" ,"Nov" ,"Dec" }; + +Time_tToSystemTime (*t, &systime); +/* We don't know how long `buf` is, but assume it's rounded up from the minimum of 25 to 32 */ +StringCchPrintfA (buf,32 ,"%s %s %d %02d:%02d:%02d %04d" , dayOfWeek[systime.wDayOfWeek -1 ], monthOfYear[systime.wMonth -1 ], + systime.wDay, systime.wHour, systime.wMinute, systime.wSecond, systime.wYear); +return buf; +} +int gettimeofday (struct timeval *restrict tp,void *restrict tzp) {FILETIME fileTime;
os/windows/posix/include/sys/ioctl.h new file +7 -0
@@ -0,0 +1,7 @@ +#ifndef IOCTL_H +#define IOCTL_H + +/* This file is empty since it only needs to exist on Windows +but isn't otherwise used */ + +#endif /* IOCTL_H */
server.c +1 -0
@@ -1007,6 +1007,7 @@ static int accept_loop(int listen_sk) }/* exits */ +strncpy (client_sockaddr_str, from,INET6_ADDRSTRLEN );handle_connection (sk); }