Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Jul 4, 2020 Kbuild

lore

Share & tools

lore

Jens Axboe

author

2020-07-04 12:00 UTC #0 lore raw

The following changes since commit 5144df064b2fc011615868e59083cfc1786bd073:

Merge branch 'num2str' of https://github.com/bvanassche/fio (2020-07-02 18:08:14 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to a504a3902e00b6db0183872b0ff62e35abae7119:

configure: fix bad indentation (2020-07-03 08:34:56 -0600)

----------------------------------------------------------------
Jens Axboe (1):
configure: fix bad indentation

Yigal Korman (4):
fio: don't retry engine search on failure configure/Makefile: engine LIBS consistency
configure: new --dynamic-libengines build option
fio: better info when engine not found

 .gitignore         |  1 +
 Makefile           | 86 +++++++++++++++++++++++++++++++++++++++++++-----------
 configure          | 20 +++++--------
 engines/dev-dax.c  |  2 +-
 engines/guasi.c    |  2 +-
 engines/http.c     |  2 +-
 engines/libaio.c   |  2 +-
 engines/libhdfs.c  |  6 ++--
 engines/libiscsi.c |  6 ++--
 engines/libpmem.c  |  2 +-
 engines/libzbc.c   |  2 +-
 engines/nbd.c      |  2 +-
 engines/pmemblk.c  |  2 +-
 engines/rados.c    |  2 +-
 engines/rbd.c      |  2 +-
 engines/rdma.c     |  9 +++---
 init.c             | 14 ++++++---
 ioengines.c        | 26 +++++++++++++++--
 ioengines.h        |  6 ++++
 os/os-linux.h      |  2 ++
 20 files changed, 141 insertions(+), 55 deletions(-)

---

Diff of recent changes:

.gitignore +1 -0

index b84b0fda..0aa4a361 100644
@@ -1,5 +1,6 @@
 *.d
 *.o
+*.so
 *.exe
 /.depend
 /FIO-VERSION-FILE

Makefile +69 ~~-17~~

164 lines not on this page · raw message

configure +8 ~~-12~~

index 63b30555..6991393b 100755
@@ -151,6 +151,7 @@ march_set="no"
 libiscsi="no"
 libnbd="no"
 libaio_uring="no"
+dynamic_engines="no"
 prefix=/usr/local
 
 # parse options

@@ -215,6 +216,8 @@ for opt do
   ;;
   --enable-libaio-uring) libaio_uring="yes"
   ;;
+  --dynamic-libengines) dynamic_engines="yes"
+  ;;
   --help)
     show_help="yes"
     ;;

@@ -254,6 +257,7 @@ if test "$show_help" = "yes" ; then
   echo "--enable-libnbd         Enable libnbd (NBD engine) support"
   echo "--disable-tcmalloc	Disable tcmalloc support"
   echo "--enable-libaio-uring   Enable libaio emulated over io_uring"
+  echo "--dynamic-libengines	Lib-based ioengines as dynamic libraries"
   exit $exit_val
 fi

@@ -605,11 +609,9 @@ int main(void)
 EOF
   if test "$libaio_uring" = "yes" && compile_prog "" "-luring" "libaio io_uring" ; then
     libaio=yes
-    LIBS="-luring $LIBS"
   elif compile_prog "" "-laio" "libaio" ; then
     libaio=yes
     libaio_uring=no
-    LIBS="-laio $LIBS"
   else
     if test "$libaio" = "yes" ; then
       feature_not_found "linux AIO" "libaio-dev or libaio-devel"

@@ -859,7 +861,6 @@ int main(int argc, char **argv)
 EOF
 if test "$disable_rdma" != "yes" && compile_prog "" "-libverbs" "libverbs" ; then
     libverbs="yes"
-    LIBS="-libverbs $LIBS"
 fi
 print_config "libverbs" "$libverbs"

@@ -879,7 +880,6 @@ int main(int argc, char **argv)
 EOF
 if test "$disable_rdma" != "yes" && compile_prog "" "-lrdmacm" "rdma"; then
     rdmacm="yes"
-    LIBS="-lrdmacm $LIBS"
 fi
 print_config "rdmacm" "$rdmacm"

@@ -1770,10 +1770,8 @@ if test "$disable_http" != "yes"; then
   if compile_prog "" "$HTTP_LIBS" "curl-new-ssl"; then
     output_sym "CONFIG_HAVE_OPAQUE_HMAC_CTX"
     http="yes"
-    LIBS="$HTTP_LIBS $LIBS"
   elif mv $TMPC2 $TMPC && compile_prog "" "$HTTP_LIBS" "curl-old-ssl"; then
     http="yes"
-    LIBS="$HTTP_LIBS $LIBS"
   fi
 fi
 print_config "http engine" "$http"

@@ -1802,7 +1800,6 @@ int main(int argc, char **argv)
 }
 EOF
 if test "$disable_rados" != "yes"  && compile_prog "" "-lrados" "rados"; then
-  LIBS="-lrados $LIBS"
   rados="yes"
 fi
 print_config "Rados engine" "$rados"

@@ -1833,7 +1830,6 @@ int main(int argc, char **argv)
 }
 EOF
 if test "$disable_rbd" != "yes"  && compile_prog "" "-lrbd -lrados" "rbd"; then
-  LIBS="-lrbd -lrados $LIBS"
   rbd="yes"
 fi
 print_config "Rados Block Device engine" "$rbd"

@@ -1924,7 +1920,6 @@ int main(int argc, char **argv)
 }
 EOF
 if test "$disable_gfapi" != "yes"  && compile_prog "" "-lgfapi -lglusterfs" "gfapi"; then
-  LIBS="-lgfapi -lglusterfs $LIBS"
   gfapi="yes"
 fi
 print_config "Gluster API engine" "$gfapi"

@@ -2086,7 +2081,6 @@ int main(int argc, char **argv)
 EOF
 if compile_prog "" "-lpmem" "libpmem"; then
   libpmem="yes"
-  LIBS="-lpmem $LIBS"
 fi
 print_config "libpmem" "$libpmem"

@@ -2108,7 +2102,6 @@ int main(int argc, char **argv)
 EOF
   if compile_prog "" "-lpmemblk" "libpmemblk"; then
     libpmemblk="yes"
-    LIBS="-lpmemblk $LIBS"
   fi
 fi
 print_config "libpmemblk" "$libpmemblk"

@@ -2432,7 +2425,6 @@ if compile_prog "" "-lzbc" "libzbc"; then
   libzbcvermaj=$(pkg-config --modversion libzbc | sed 's/\.[0-9]*\.[0-9]*//')
   if test "$libzbcvermaj" -ge "5" ; then
     libzbc="yes"
-    LIBS="-lzbc $LIBS"
   else
     print_config "libzbc engine" "Unsupported libzbc version (version 5 or above required)"
     libzbc="no"

@@ -2966,6 +2958,10 @@ if test "$libnbd" = "yes" ; then
   echo "LIBNBD_CFLAGS=$libnbd_cflags" >> $config_host_mak
   echo "LIBNBD_LIBS=$libnbd_libs" >> $config_host_mak
 fi
+if test "$dynamic_engines" = "yes" ; then
+  output_sym "CONFIG_DYNAMIC_ENGINES"
+fi
+print_config "Lib-based ioengines dynamic" "$dynamic_engines"
 cat > $TMPC << EOF
 int main(int argc, char **argv)
 {

engines/dev-dax.c +1 -1

index 422ea634..1d0f66cb 100644
@@ -328,7 +328,7 @@ fio_devdax_get_file_size(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name		= "dev-dax",
 	.version	= FIO_IOOPS_VERSION,
 	.init		= fio_devdax_init,

engines/guasi.c +1 -1

index cb26802c..d4121757 100644
@@ -242,7 +242,7 @@ static int fio_guasi_init(struct thread_data *td)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name		= "guasi",
 	.version	= FIO_IOOPS_VERSION,
 	.init		= fio_guasi_init,

engines/http.c +1 -1

index 275fcab5..7a61b132 100644
@@ -639,7 +639,7 @@ static int fio_http_invalidate(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name = "http",
 	.version		= FIO_IOOPS_VERSION,
 	.flags			= FIO_DISKLESSIO | FIO_SYNCIO,

engines/libaio.c +1 -1

index 398fdf91..b909b79e 100644
@@ -445,7 +445,7 @@ static int fio_libaio_init(struct thread_data *td)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name			= "libaio",
 	.version		= FIO_IOOPS_VERSION,
 	.flags			= FIO_ASYNCIO_SYNC_TRIM,

engines/libhdfs.c +3 -3

index c57fcea6..9ca82f78 100644
@@ -393,7 +393,7 @@ static void fio_hdfsio_io_u_free(struct thread_data *td, struct io_u *io_u)
 	}
 }
 
-static struct ioengine_ops ioengine_hdfs = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name = "libhdfs",
 	.version = FIO_IOOPS_VERSION,
 	.flags = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NODISKUTIL,

@@ -412,10 +412,10 @@ static struct ioengine_ops ioengine_hdfs = {
 
 static void fio_init fio_hdfsio_register(void)
 {
-	register_ioengine(&ioengine_hdfs);
+	register_ioengine(&ioengine);
 }
 
 static void fio_exit fio_hdfsio_unregister(void)
 {
-	unregister_ioengine(&ioengine_hdfs);
+	unregister_ioengine(&ioengine);
 }

engines/libiscsi.c +3 -3

index 35761a61..c97b5709 100644
@@ -383,7 +383,7 @@ static struct io_u *fio_iscsi_event(struct thread_data *td, int event)
 	return io_u;
 }
 
-static struct ioengine_ops ioengine_iscsi = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name               = "libiscsi",
 	.version            = FIO_IOOPS_VERSION,
 	.flags              = FIO_SYNCIO | FIO_DISKLESSIO | FIO_NODISKUTIL,

@@ -402,10 +402,10 @@ static struct ioengine_ops ioengine_iscsi = {
 
 static void fio_init fio_iscsi_register(void)
 {
-	register_ioengine(&ioengine_iscsi);
+	register_ioengine(&ioengine);
 }
 
 static void fio_exit fio_iscsi_unregister(void)
 {
-	unregister_ioengine(&ioengine_iscsi);
+	unregister_ioengine(&ioengine);
 }

engines/libpmem.c +1 -1

index 99c7b50d..3f63055c 100644
@@ -558,7 +558,7 @@ static int fio_libpmem_close_file(struct thread_data *td, struct fio_file *f)
 	return generic_close_file(td, f);
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name		= "libpmem",
 	.version	= FIO_IOOPS_VERSION,
 	.init		= fio_libpmem_init,

engines/libzbc.c +1 -1

index 9e568334..fdde8ca6 100644
@@ -397,7 +397,7 @@ static enum fio_q_status libzbc_queue(struct thread_data *td, struct io_u *io_u)
 	return FIO_Q_COMPLETED;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name			= "libzbc",
 	.version		= FIO_IOOPS_VERSION,
 	.open_file		= libzbc_open_file,

engines/nbd.c +1 -1

index 53237929..b0ba75e6 100644
@@ -328,7 +328,7 @@ static int nbd_invalidate(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name			= "nbd",
 	.version		= FIO_IOOPS_VERSION,
 	.options		= options,

engines/pmemblk.c +1 -1

index e2eaa15e..fc6358e8 100644
@@ -426,7 +426,7 @@ static int fio_pmemblk_unlink_file(struct thread_data *td, struct fio_file *f)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name = "pmemblk",
 	.version = FIO_IOOPS_VERSION,
 	.queue = fio_pmemblk_queue,

engines/rados.c +1 -1

index d4413427..42ee48ff 100644
@@ -444,7 +444,7 @@ static int fio_rados_io_u_init(struct thread_data *td, struct io_u *io_u)
 }
 
 /* ioengine_ops for get_ioengine() */
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name = "rados",
 	.version		= FIO_IOOPS_VERSION,
 	.flags			= FIO_DISKLESSIO,

engines/rbd.c +1 -1

index a08f4775..268b6ebd 100644
@@ -668,7 +668,7 @@ static int fio_rbd_io_u_init(struct thread_data *td, struct io_u *io_u)
 	return 0;
 }
 
-static struct ioengine_ops ioengine = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name			= "rbd",
 	.version		= FIO_IOOPS_VERSION,
 	.setup			= fio_rbd_setup,

engines/rdma.c +5 -4

index f192f432..f4471869 100644
@@ -226,7 +226,8 @@ static int client_recv(struct thread_data *td, struct ibv_wc *wc)
 		rd->rmt_nr = ntohl(rd->recv_buf.nr);
 
 		for (i = 0; i < rd->rmt_nr; i++) {
-			rd->rmt_us[i].buf = be64_to_cpu(rd->recv_buf.rmt_us[i].buf);
+			rd->rmt_us[i].buf = __be64_to_cpu(
+						rd->recv_buf.rmt_us[i].buf);
 			rd->rmt_us[i].rkey = ntohl(rd->recv_buf.rmt_us[i].rkey);
 			rd->rmt_us[i].size = ntohl(rd->recv_buf.rmt_us[i].size);

@@ -1389,7 +1390,7 @@ static int fio_rdmaio_setup(struct thread_data *td)
 	return 0;
 }
 
-static struct ioengine_ops ioengine_rw = {
+FIO_STATIC struct ioengine_ops ioengine = {
 	.name			= "rdma",
 	.version		= FIO_IOOPS_VERSION,
 	.setup			= fio_rdmaio_setup,

@@ -1410,10 +1411,10 @@ static struct ioengine_ops ioengine_rw = {
 
 static void fio_init fio_rdmaio_register(void)
 {
-	register_ioengine(&ioengine_rw);
+	register_ioengine(&ioengine);
 }
 
 static void fio_exit fio_rdmaio_unregister(void)
 {
-	unregister_ioengine(&ioengine_rw);
+	unregister_ioengine(&ioengine);
 }

init.c +10 -4

index e53be35a..3710e3d4 100644
@@ -1099,6 +1099,9 @@ int ioengine_load(struct thread_data *td)
 		 */
 		dlhandle = td->io_ops_dlhandle;
 		ops = load_ioengine(td);
+		if (!ops)
+			goto fail;
+
 		if (ops == td->io_ops && dlhandle == td->io_ops_dlhandle) {
 			if (dlhandle)
 				dlclose(dlhandle);

@@ -1113,10 +1116,8 @@ int ioengine_load(struct thread_data *td)
 	}
 
 	td->io_ops = load_ioengine(td);
-	if (!td->io_ops) {
-		log_err("fio: failed to load engine\n");
-		return 1;
-	}
+	if (!td->io_ops)
+		goto fail;
 
 	if (td->io_ops->option_struct_size && td->io_ops->options) {
 		/*

@@ -1155,6 +1156,11 @@ int ioengine_load(struct thread_data *td)
 
 	td_set_ioengine_flags(td);
 	return 0;
+
+fail:
+	log_err("fio: failed to load engine\n");
+	return 1;
+
 }
 
 static void init_flags(struct thread_data *td)

ioengines.c +24 -2

index 2c7a0df9..c1b430a1 100644
@@ -75,6 +75,25 @@ static struct ioengine_ops *find_ioengine(const char *name)
 	return NULL;
 }
 
+#ifdef CONFIG_DYNAMIC_ENGINES
+static void *dlopen_external(struct thread_data *td, const char *engine)
+{
+	char engine_path[PATH_MAX];
+	void *dlhandle;
+
+	sprintf(engine_path, "%s/lib%s.so", FIO_EXT_ENG_DIR, engine);
+
+	dlhandle = dlopen(engine_path, RTLD_LAZY);
+	if (!dlhandle)
+		log_info("Engine %s not found; Either name is invalid, was not built, or fio-engine-%s package is missing.\n",
+			 engine, engine);
+
+	return dlhandle;
+}
+#else
+#define dlopen_external(td, engine) (NULL)
+#endif
+
 static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
 					    const char *engine_lib)
 {

@@ -86,8 +105,11 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
 	dlerror();
 	dlhandle = dlopen(engine_lib, RTLD_LAZY);
 	if (!dlhandle) {
-		td_vmsg(td, -1, dlerror(), "dlopen");
-		return NULL;
+		dlhandle = dlopen_external(td, engine_lib);
+		if (!dlhandle) {
+			td_vmsg(td, -1, dlerror(), "dlopen");
+			return NULL;
+		}
 	}
 
 	/*

ioengines.h +6 -0

index f48b4db9..54dadba2 100644
@@ -10,6 +10,12 @@
 
 #define FIO_IOOPS_VERSION	26
 
+#ifndef CONFIG_DYNAMIC_ENGINES
+#define FIO_STATIC	static
+#else
+#define FIO_STATIC
+#endif
+
 /*
  * io_ops->queue() return values
  */

os/os-linux.h +2 -0

index 6ec7243d..65d3b429 100644
@@ -58,6 +58,8 @@
 
 #define OS_MAP_ANON		MAP_ANONYMOUS
 
+#define FIO_EXT_ENG_DIR	"/usr/lib/fio"
+
 typedef cpu_set_t os_cpu_mask_t;
 
 #ifdef CONFIG_3ARG_AFFINITY