fio: add multi directory support

PATCH In review [email protected] 1 participant 1 message Feb 20, 2014

Share & tools

[email protected]

author

2014-02-20 13:21 UTC #0 lore raw

PATCH 5/9 fio: add multi directory support

*Resend with hopefully non mangled patches*
References: <[email protected]>
Content-Disposition: inline; filename=fio-multidir-support-v2.diff

From: Christian Ehrhardt <[email protected]>

This patch adds support for ':' seperated multiple directories at the directory config statement in order to achieve an automatic distribution of job clones (numjob) across directories.

That way people can distribute a load across these directories (usually mount points of disks) automatically - changing numjob will be sufficient to get all job clones evenly (optimal if dirs % numjobs = 0, otherwise as good as possible) distributed at all times.

To avoid confused users old config Files will behave like they always did, old fio binaries using new config files won't abort but just use the first specified dir. If one specifies an explcit (non generated) filename the distribution to many directories is also deactivated.

It also fixes an issue of events seeming out of order like when running with
--debug=file seeing the "..." message meaning "I created the clones" prior
to the last clone activities.
Now the clones are called with index N-1 .. 1, zero being the base thread as before.

Signed-off-by Christian Ehrhardt <[email protected]>

---

[diffstat]

 engines/net.c |    2 -
 file.h        |    8 +++++
 filesetup.c   |   70 ++++++++++++++++++++++++++++++++++++++++++++++------
 fio.1         |    6 ++++
 init.c        |    9 ++----
 iolog.c       |    2 -
 options.c     |   78 ++++++++++++++++++++++++++++++++++++++++++++++------------
 options.h     |    2 +
 8 files changed, 146 insertions(+), 31 deletions(-)

[diff]

engines/net.c +1 -1

@@ -1194,7 +1194,7 @@ static int fio_netio_setup(struct thread
 	struct netio_data *nd;
 
 	if (!td->files_index) {
-		add_file(td, td->o.filename ?: "net");
+		add_file(td, td->o.filename ?: "net", 0);
 		td->o.nr_files = td->o.nr_files ?: 1;
 	}

file.h +7 -1

@@ -125,6 +125,11 @@ struct fio_file {
 	struct disk_util *du;
 };
 
+struct file_name {
+	struct flist_head list;
+	char *filename;
+};
+
 #define FILE_FLAG_FNS(name)						\
 static inline void fio_file_set_##name(struct fio_file *f)		\
 {									\

@@ -162,7 +167,7 @@ extern int __must_check generic_close_fi
 extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *);
 extern int __must_check file_lookup_open(struct fio_file *f, int flags);
 extern int __must_check pre_read_files(struct thread_data *);
-extern int add_file(struct thread_data *, const char *);
+extern int add_file(struct thread_data *, const char *, int);
 extern int add_file_exclusive(struct thread_data *, const char *);
 extern void get_file(struct fio_file *);
 extern int __must_check put_file(struct thread_data *, struct fio_file *);

@@ -175,6 +180,7 @@ extern int init_random_map(struct thread
 extern void dup_files(struct thread_data *, struct thread_data *);
 extern int get_fileno(struct thread_data *, const char *);
 extern void free_release_files(struct thread_data *);
+extern void filesetup_mem_free(void);
 void fio_file_reset(struct thread_data *, struct fio_file *);
 int fio_files_done(struct thread_data *);

filesetup.c +63 -7

121 lines not on this page · raw message

fio.1 +6 -0

@@ -158,6 +158,12 @@ otherwise has no special purpose.
 .BI directory \fR=\fPstr
 Prefix filenames with this directory.  Used to place files in a location other
 than `./'.
+You can specify a number of directories by separating the names with a ':'
+character. These directories will be assigned equally distributed to job clones
+creates with \fInumjobs\fR as long as they are using generated filenames.
+If specific \fIfilename(s)\fR are set fio will use the first listed directory,
+and thereby matching the  \fIfilename\fR semantic which generates a file each
+clone if not specified, but let all clones use the same if set.
 .TP
 .BI filename \fR=\fPstr
 .B fio

init.c +4 -5

@@ -1020,10 +1020,10 @@ static int add_job(struct thread_data *t
 		file_alloced = 1;
 
 		if (o->nr_files == 1 && exists_and_not_file(jobname))
-			add_file(td, jobname);
+			add_file(td, jobname, job_add_num);
 		else {
 			for (i = 0; i < o->nr_files; i++)
-				add_file(td, make_filename(fname, o, jobname, td->thread_number, i));
+				add_file(td, make_filename(fname, o, jobname, job_add_num, i), job_add_num);
 		}
 	}

@@ -1166,9 +1166,7 @@ static int add_job(struct thread_data *t
 			}
 		}
 
-		job_add_num = numjobs - 1;
-
-		if (add_job(td_new, jobname, job_add_num, 1, client_type))
+		if (add_job(td_new, jobname, numjobs, 1, client_type))
 			goto err;
 	}

@@ -2027,6 +2025,7 @@ int parse_options(int argc, char *argv[]
 
 	free(ini_file);
 	fio_options_free(&def_thread);
+	filesetup_mem_free();
 
 	if (!thread_number) {
 		if (parse_dryrun())

iolog.c +1 -1

@@ -324,7 +324,7 @@ static int read_iolog2(struct thread_dat
 			rw = DDIR_INVAL;
 			if (!strcmp(act, "add")) {
 				td->o.nr_files++;
-				fileno = add_file(td, fname);
+				fileno = add_file(td, fname, 0);
 				file_action = FIO_LOG_ADD_FILE;
 				continue;
 			} else if (!strcmp(act, "open")) {

options.c +62 ~~-16~~

116 lines not on this page · raw message

options.h +2 -0

@@ -17,6 +17,8 @@ void add_opt_posval(const char *, const
 void del_opt_posval(const char *, const char *);
 struct thread_data;
 void fio_options_free(struct thread_data *);
+char *get_name_idx(char *, int);
+int set_name_idx(char *, char *, int);
 
 extern struct fio_option fio_options[FIO_MAX_OPTS];