Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Mar 18, 2021 Kbuild

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 014ab48afcbcf442464acc7427fcd0f194f64bf4:

Merge branch 'dev_luye_github' of https://github.com/louisluSCU/fio (2021-03-11 11:50:49 -0700)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to dede9b9fae3ab670c1ca864ac66aea5e997e1f34:

Merge branch 'free-dump-options' of https://github.com/floatious/fio (2021-03-17 09:25:46 -0600)

----------------------------------------------------------------
Jens Axboe (3):
Merge branch 'dfs_engine' of https://github.com/johannlombardi/fio
Merge branch 'patch-1' of https://github.com/ihsinme/fio
Merge branch 'free-dump-options' of https://github.com/floatious/fio

Johann Lombardi (2):
Disable pthread_condattr_setclock on cygwin engines/dfs: add DAOS File System (dfs) engine

Niklas Cassel (1):
options: free dump options list on exit

ihsinme (1):
fix loop with unreachable exit condition

 HOWTO            |  21 ++
 Makefile         |   5 +
 backend.c        |   1 +
 configure        |  52 ++++-
 engines/dfs.c    | 583 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 examples/dfs.fio |  33 ++++
 fio.1            |  18 ++
 init.c           |  13 --
 optgroup.c       |   4 +
 optgroup.h       |   2 +
 options.c        |  18 ++
 options.h        |   1 +
 server.c         |   2 +-
 13 files changed, 730 insertions(+), 23 deletions(-)
 create mode 100644 engines/dfs.c
 create mode 100644 examples/dfs.fio

---

Diff of recent changes:

HOWTO +21 -0
index 1e5ebd5d..041b91fa 100644
@@ -2067,6 +2067,9 @@ I/O engine
 			unless :option:`verify` is set or :option:`cuda_io` is `posix`.
 			:option:`iomem` must not be `cudamalloc`. This ioengine defines
 			engine specific options.
+		**dfs**
+			I/O engine supporting asynchronous read and write operations to the
+			DAOS File System (DFS) via libdfs.
 
 I/O engine specific parameters
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2452,6 +2455,24 @@ with the caveat that when used on the command line, they must come after the
 		GPU to RAM before a write and copied from RAM to GPU after a
 		read. :option:`verify` does not affect use of cudaMemcpy.
 
+.. option:: pool=str : [dfs]
+
+	Specify the UUID of the DAOS pool to connect to.
+
+.. option:: cont=str : [dfs]
+
+	Specify the UUID of the DAOS container to open.
+
+.. option:: chunk_size=int : [dfs]
+
+	Specificy a different chunk size (in bytes) for the dfs file.
+	Use DAOS container's chunk size by default.
+
+.. option:: object_class=str : [dfs]
+
+	Specificy a different object class for the dfs file.
+	Use DAOS container's object class by default.
+
 I/O depth
 ~~~~~~~~~
 
Makefile +5 -0
index 612344d1..87a47b66 100644
@@ -130,6 +130,11 @@ ifdef CONFIG_HTTP
   http_LIBS = -lcurl -lssl -lcrypto
   ENGINES += http
 endif
+ifdef CONFIG_DFS
+  dfs_SRCS = engines/dfs.c
+  dfs_LIBS = -luuid -ldaos -ldfs
+  ENGINES += dfs
+endif
 SOURCE += oslib/asprintf.c
 ifndef CONFIG_STRSEP
   SOURCE += oslib/strsep.c
backend.c +1 -0
index f2efddd6..52b4ca7e 100644
@@ -2537,6 +2537,7 @@ int fio_backend(struct sk_out *sk_out)
 	for_each_td(td, i) {
 		steadystate_free(td);
 		fio_options_free(td);
+		fio_dump_options_free(td);
 		if (td->rusage_sem) {
 			fio_sem_remove(td->rusage_sem);
 			td->rusage_sem = NULL;
configure +43 -9

100 lines not on this page · raw message

engines/dfs.c new file +583 -0

583 lines not on this page · raw message

examples/dfs.fio new file +33 -0
new file mode 100644
index 00000000..5de887d7
@@ -0,0 +1,33 @@
+[global]
+ioengine=dfs
+pool=${POOL}
+cont=${CONT}
+filename_format=fio-test.$jobnum
+
+cpus_allowed_policy=split
+group_reporting=1
+time_based=0
+percentile_list=99.0:99.9:99.99:99.999:99.9999:100
+disable_slat=1
+disable_clat=1
+
+bs=1M
+size=100G
+iodepth=16
+numjobs=16
+
+[daos-seqwrite]
+rw=write
+stonewall
+
+[daos-seqread]
+rw=read
+stonewall
+
+[daos-randwrite]
+rw=randwrite
+stonewall
+
+[daos-randread]
+rw=randread
+stonewall
fio.1 +18 -0
index b95a67ab..27cf2f15 100644
@@ -1856,6 +1856,10 @@ GPUDirect Storage-supported filesystem. This engine performs
 I/O without transferring buffers between user-space and the kernel,
 unless \fBverify\fR is set or \fBcuda_io\fR is \fBposix\fR. \fBiomem\fR must
 not be \fBcudamalloc\fR. This ioengine defines engine specific options.
+.TP
+.B dfs
+I/O engine supporting asynchronous read and write operations to the DAOS File
+System (DFS) via libdfs.
 .SS "I/O engine specific parameters"
 In addition, there are some parameters which are only valid when a specific
 \fBioengine\fR is in use. These are used identically to normal parameters,
@@ -2209,6 +2213,20 @@ from RAM to GPU after a read. \fBverify\fR does not affect
 the use of cudaMemcpy.
 .RE
 .RE
+.TP
+.BI (dfs)pool
+Specify the UUID of the DAOS pool to connect to.
+.TP
+.BI (dfs)cont
+Specify the UUID of the DAOS DAOS container to open.
+.TP
+.BI (dfs)chunk_size
+Specificy a different chunk size (in bytes) for the dfs file.
+Use DAOS container's chunk size by default.
+.TP
+.BI (dfs)object_class
+Specificy a different object class for the dfs file.
+Use DAOS container's object class by default.
 .SS "I/O depth"
 .TP
 .BI iodepth \fR=\fPint
init.c +0 -13
index e8530fec..37bff876 100644
@@ -448,19 +448,6 @@ static void dump_opt_list(struct thread_data *td)
 	}
 }
 
-static void fio_dump_options_free(struct thread_data *td)
-{
-	while (!flist_empty(&td->opt_list)) {
-		struct print_option *p;
-
-		p = flist_first_entry(&td->opt_list, struct print_option, list);
-		flist_del_init(&p->list);
-		free(p->name);
-		free(p->value);
-		free(p);
-	}
-}
-
 static void copy_opt_list(struct thread_data *dst, struct thread_data *src)
 {
 	struct flist_head *entry;
optgroup.c +4 -0
index 64774896..4cdea71f 100644
@@ -177,6 +177,10 @@ static const struct opt_group fio_opt_cat_groups[] = {
 		.name	= "libcufile I/O engine", /* libcufile */
 		.mask	= FIO_OPT_G_LIBCUFILE,
 	},
+	{
+		.name	= "DAOS File System (dfs) I/O engine", /* dfs */
+		.mask	= FIO_OPT_G_DFS,
+	},
 	{
 		.name	= NULL,
 	},
optgroup.h +2 -0
index d2f1ceb3..25b7fec1 100644
@@ -68,6 +68,7 @@ enum opt_category_group {
 	__FIO_OPT_G_FILESTAT,
 	__FIO_OPT_G_NR,
 	__FIO_OPT_G_LIBCUFILE,
+	__FIO_OPT_G_DFS,
 
 	FIO_OPT_G_RATE		= (1ULL << __FIO_OPT_G_RATE),
 	FIO_OPT_G_ZONE		= (1ULL << __FIO_OPT_G_ZONE),
@@ -110,6 +111,7 @@ enum opt_category_group {
 	FIO_OPT_G_IOURING	= (1ULL << __FIO_OPT_G_IOURING),
 	FIO_OPT_G_FILESTAT	= (1ULL << __FIO_OPT_G_FILESTAT),
 	FIO_OPT_G_LIBCUFILE	= (1ULL << __FIO_OPT_G_LIBCUFILE),
+	FIO_OPT_G_DFS		= (1ULL << __FIO_OPT_G_DFS),
 };
 
 extern const struct opt_group *opt_group_from_mask(uint64_t *mask);
options.c +18 -0
index 0791101e..151e7a7e 100644
@@ -2011,6 +2011,11 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 			  { .ival = "nbd",
 			    .help = "Network Block Device (NBD) IO engine"
 			  },
+#ifdef CONFIG_DFS
+			  { .ival = "dfs",
+			    .help = "DAOS File System (dfs) IO engine",
+			  },
+#endif
 		},
 	},
 	{
@@ -5456,6 +5461,19 @@ void fio_options_free(struct thread_data *td)
 	}
 }
 
+void fio_dump_options_free(struct thread_data *td)
+{
+	while (!flist_empty(&td->opt_list)) {
+		struct print_option *p;
+
+		p = flist_first_entry(&td->opt_list, struct print_option, list);
+		flist_del_init(&p->list);
+		free(p->name);
+		free(p->value);
+		free(p);
+	}
+}
+
 struct fio_option *fio_option_find(const char *name)
 {
 	return find_option(fio_options, name);
options.h +1 -0
index 5276f31e..df80fd98 100644
@@ -16,6 +16,7 @@ void add_opt_posval(const char *, const char *, const char *);
 void del_opt_posval(const char *, const char *);
 struct thread_data;
 void fio_options_free(struct thread_data *);
+void fio_dump_options_free(struct thread_data *);
 char *get_next_str(char **ptr);
 int get_max_str_idx(char *input);
 char* get_name_by_idx(char *input, int index);
server.c +1 -1
index 1b65297e..8daefbab 100644
@@ -1909,7 +1909,7 @@ static int fio_append_iolog_gz(struct sk_entry *first, struct io_log *log)
 			break;
 		}
 		flist_add_tail(&entry->list, &first->next);
-	} while (ret != Z_STREAM_END);
+	}
 
 	ret = deflateEnd(&stream);
 	if (ret == Z_OK)