Skip to contentSkip to conversation

virtiofs daemon (base)

PATCH In review Dr. David Alan Gilbert (git) 5 participants 61 messages Oct 21, 2019 – Nov 14, 2019 1 message not in the archive Page 2 of 2

Subsystems (2)
lore
Share & tools
lore
#15 lore raw

PATCH 9/30 virtiofsd: Fix fuse_daemonize ignored return values

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

QEMU's compiler enables warnings/errors for ignored values and the (void) trick used in the fuse code isn't enough. Turn all the return values into a return value on the function.

---
 contrib/virtiofsd/helper.c | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)
contrib/virtiofsd/helper.c +14 -11
index f7b00db258..958a42b6f4 100644
@@ -10,12 +10,10 @@
   See the file COPYING.LIB.
 */
 
-#include "config.h"
 #include "fuse_i.h"
 #include "fuse_misc.h"
 #include "fuse_opt.h"
 #include "fuse_lowlevel.h"
-#include "mount_util.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -173,6 +171,7 @@ int fuse_parse_cmdline(struct fuse_args *args,
 
 int fuse_daemonize(int foreground)
 {
+	int ret = 0, rett;
 	if (!foreground) {
 		int nullfd;
 		int waiter[2];
@@ -194,8 +193,8 @@ int fuse_daemonize(int foreground)
 		case 0:
 			break;
 		default:
-			(void) read(waiter[0], &completed, sizeof(completed));
-			_exit(0);
+			_exit( read(waiter[0], &completed, sizeof(completed) !=
+                               sizeof(completed)));
 		}
 
 		if (setsid() == -1) {
@@ -203,26 +202,30 @@ int fuse_daemonize(int foreground)
 			return -1;
 		}
 
-		(void) chdir("/");
+		ret = chdir("/");
 
 		nullfd = open("/dev/null", O_RDWR, 0);
 		if (nullfd != -1) {
-			(void) dup2(nullfd, 0);
-			(void) dup2(nullfd, 1);
-			(void) dup2(nullfd, 2);
+			rett = dup2(nullfd, 0);
+			if (!ret) ret = rett;
+			rett = dup2(nullfd, 1);
+			if (!ret) ret = rett;
+			rett = dup2(nullfd, 2);
+			if (!ret) ret = rett;
 			if (nullfd > 2)
 				close(nullfd);
 		}
 
 		/* Propagate completion of daemon initialization */
 		completed = 1;
-		(void) write(waiter[1], &completed, sizeof(completed));
+		rett = write(waiter[1], &completed, sizeof(completed));
+		if (!ret) ret = rett;
 		close(waiter[0]);
 		close(waiter[1]);
 	} else {
-		(void) chdir("/");
+		ret = chdir("/");
 	}
-	return 0;
+	return ret;
 }
 
 void fuse_apply_conn_info_opts(struct fuse_conn_info_opts *opts,
Signature · 1 line
2.23.0
#16 lore raw

PATCH 10/30 virtiofsd: Fix common header and define for QEMU builds

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

All of the fuse files include config.h and define GNU_SOURCE where we don't have either under our build - remove them.

---
 contrib/virtiofsd/buffer.c         |  3 ---
 contrib/virtiofsd/fuse_i.h         |  3 +++
 contrib/virtiofsd/fuse_loop_mt.c   |  3 +--
 contrib/virtiofsd/fuse_lowlevel.c  |  9 ---------
 contrib/virtiofsd/fuse_opt.c       |  1 -
 contrib/virtiofsd/fuse_signals.c   |  3 +--
 contrib/virtiofsd/passthrough_ll.c | 10 +++-------
 7 files changed, 8 insertions(+), 24 deletions(-)
contrib/virtiofsd/buffer.c +0 -3
index 5ab9b87455..3c9b443fa0 100644
@@ -9,9 +9,6 @@
   See the file COPYING.LIB
 */
 
-#define _GNU_SOURCE
-
-#include "config.h"
 #include "fuse_i.h"
 #include "fuse_lowlevel.h"
 #include <string.h>
contrib/virtiofsd/fuse_i.h +3 -0
index bcd6a140fc..6c3f432d0e 100644
@@ -6,6 +6,9 @@
   See the file COPYING.LIB
 */
 
+#define FUSE_USE_VERSION 31
+
+
 #include "fuse.h"
 #include "fuse_lowlevel.h"
 
contrib/virtiofsd/fuse_loop_mt.c +1 -2
index 6fcd47e42c..e45b2337a0 100644
@@ -8,11 +8,10 @@
   See the file COPYING.LIB.
 */
 
-#include "config.h"
+#include "fuse_i.h"
 #include "fuse_lowlevel.h"
 #include "fuse_misc.h"
 #include "fuse_kernel.h"
-#include "fuse_i.h"
 
 #include <stdio.h>
 #include <stdlib.h>
contrib/virtiofsd/fuse_lowlevel.c +0 -9
index a36a00e793..5ea113ab49 100644
@@ -9,14 +9,10 @@
   See the file COPYING.LIB
 */
 
-#define _GNU_SOURCE
-
-#include "config.h"
 #include "fuse_i.h"
 #include "fuse_kernel.h"
 #include "fuse_opt.h"
 #include "fuse_misc.h"
-#include "mount_util.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -1954,7 +1950,6 @@ static struct {
 	[FUSE_READDIRPLUS] = { do_readdirplus,	"READDIRPLUS"},
 	[FUSE_RENAME2]     = { do_rename2,      "RENAME2"    },
 	[FUSE_COPY_FILE_RANGE] = { do_copy_file_range, "COPY_FILE_RANGE" },
-	[CUSE_INIT]	   = { cuse_lowlevel_init, "CUSE_INIT"   },
 };
 
 #define FUSE_MAXOP (sizeof(fuse_ll_ops) / sizeof(fuse_ll_ops[0]))
@@ -2073,7 +2068,6 @@ void fuse_lowlevel_version(void)
 {
 	printf("using FUSE kernel interface version %i.%i\n",
 	       FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION);
-	fuse_mount_version();
 }
 
 void fuse_lowlevel_help(void)
@@ -2151,9 +2145,6 @@ struct fuse_session *fuse_session_new(struct fuse_args *args,
 		goto out4;
 	}
 
-	if (se->debug)
-		fuse_log(FUSE_LOG_DEBUG, "FUSE library version: %s\n", PACKAGE_VERSION);
-
 	se->bufsize = FUSE_MAX_MAX_PAGES * getpagesize() +
 		FUSE_BUFFER_HEADER_SIZE;
 
contrib/virtiofsd/fuse_opt.c +0 -1
index 93066b926e..b81d4bc4c9 100644
@@ -9,7 +9,6 @@
   See the file COPYING.LIB
 */
 
-#include "config.h"
 #include "fuse_i.h"
 #include "fuse_opt.h"
 #include "fuse_misc.h"
contrib/virtiofsd/fuse_signals.c +1 -2
index 4271947bd4..e8f9b1e9a6 100644
@@ -8,9 +8,8 @@
   See the file COPYING.LIB
 */
 
-#include "config.h"
-#include "fuse_lowlevel.h"
 #include "fuse_i.h"
+#include "fuse_lowlevel.h"
 
 #include <stdio.h>
 #include <string.h>
contrib/virtiofsd/passthrough_ll.c +3 -7
index 8f94dcbc92..72fae9d10b 100644
@@ -34,12 +34,7 @@
  * \include passthrough_ll.c
  */
 
-#define _GNU_SOURCE
-#define FUSE_USE_VERSION 31
-
-#include "config.h"
-
-#include <fuse_lowlevel.h>
+#include "fuse_lowlevel.h"
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -57,6 +52,8 @@
 
 #include "passthrough_helpers.h"
 
+#define HAVE_POSIX_FALLOCATE 1
+
 /* We are re-using pointers to our `struct lo_inode` and `struct
    lo_dirp` elements as inodes. This means that we must be able to
    store uintptr_t values in a fuse_ino_t variable. The following
@@ -1224,7 +1221,6 @@ int main(int argc, char *argv[])
 		ret = 0;
 		goto err_out1;
 	} else if (opts.show_version) {
-		printf("FUSE library version %s\n", fuse_pkgversion());
 		fuse_lowlevel_version();
 		ret = 0;
 		goto err_out1;
Signature · 1 line
2.23.0
#17 lore raw

PATCH 11/30 virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

---
 contrib/virtiofsd/fuse_lowlevel.c | 2 +-
 contrib/virtiofsd/fuse_lowlevel.h | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_lowlevel.c +1 -1
index 5ea113ab49..ff68ec0c9d 100644
@@ -74,7 +74,7 @@ static void convert_attr(const struct fuse_setattr_in *attr, struct stat *stbuf)
 	ST_CTIM_NSEC_SET(stbuf, attr->ctimensec);
 }
 
-static	size_t iov_length(const struct iovec *iov, size_t count)
+size_t iov_length(const struct iovec *iov, size_t count)
 {
 	size_t seg;
 	size_t ret = 0;
contrib/virtiofsd/fuse_lowlevel.h +9 -0
index 018e7c0be0..84379a4761 100644
@@ -2000,6 +2000,15 @@ void fuse_session_process_buf(struct fuse_session *se,
  */
 int fuse_session_receive_buf(struct fuse_session *se, struct fuse_buf *buf);
 
+/**
+ * General utility to get the length of an iov, not Fuse specific
+ *
+ * @param iov The iovec to get the length of
+ * @param count Number of entries in the iovec
+ * @return the total length of iov in bytes
+ */
+size_t iov_length(const struct iovec *iov, size_t count);
+
 #ifdef __cplusplus
 }
 #endif
Signature · 1 line
2.23.0
#18 lore raw

Re: [PATCH 11/30] virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c

Hi

Quoted 4 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Similar function already exists in util/iov.c, iov_size()

Perhaps you could link with util/iov.o and others to avoid code duplication.

Quoted 42 lines
---
 contrib/virtiofsd/fuse_lowlevel.c | 2 +-
 contrib/virtiofsd/fuse_lowlevel.h | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_lowlevel.c +1 -1
index 5ea113ab49..ff68ec0c9d 100644
@@ -74,7 +74,7 @@ static void convert_attr(const struct fuse_setattr_in *attr, struct stat *stbuf)
        ST_CTIM_NSEC_SET(stbuf, attr->ctimensec);
 }
 
-static size_t iov_length(const struct iovec *iov, size_t count)
+size_t iov_length(const struct iovec *iov, size_t count)
 {
        size_t seg;
        size_t ret = 0;
contrib/virtiofsd/fuse_lowlevel.h +9 -1
index 018e7c0be0..84379a4761 100644
@@ -2000,6 +2000,15 @@ void fuse_session_process_buf(struct fuse_session *se,
  */
 int fuse_session_receive_buf(struct fuse_session *se, struct fuse_buf *buf);
 
+/**
+ * General utility to get the length of an iov, not Fuse specific
+ *
+ * @param iov The iovec to get the length of
+ * @param count Number of entries in the iovec
+ * @return the total length of iov in bytes
+ */
+size_t iov_length(const struct iovec *iov, size_t count);
+
 #ifdef __cplusplus
 }
 #endif
--

2.23.0

Signature · 1 line
Marc-André Lureau
#19 lore raw

Re: [PATCH 11/30] virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c

Quoted 5 lines · * Marc-André Lureau ([email protected]) wrote:

Hi

Quoted 1 line · Dr. David Alan Gilbert (git)
Quoted 7 lines
Quoted 3 lines

From: "Dr. David Alan Gilbert" <[email protected]>

Similar function already exists in util/iov.c, iov_size()

Perhaps you could link with util/iov.o and others to avoid code duplication.

Done.
(It's a shame there's no iov_* functions in glib; every project reinvents iov_ functions).

Dave

Quoted 27 lines
Quoted 27 lines
---
 contrib/virtiofsd/fuse_lowlevel.c | 2 +-
 contrib/virtiofsd/fuse_lowlevel.h | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_lowlevel.c +1 -1
index 5ea113ab49..ff68ec0c9d 100644
@@ -74,7 +74,7 @@ static void convert_attr(const struct fuse_setattr_in *attr, struct stat *stbuf)
        ST_CTIM_NSEC_SET(stbuf, attr->ctimensec);
 }
 
-static size_t iov_length(const struct iovec *iov, size_t count)
+size_t iov_length(const struct iovec *iov, size_t count)
 {
        size_t seg;
        size_t ret = 0;
contrib/virtiofsd/fuse_lowlevel.h +0 -0
index 018e7c0be0..84379a4761 100644
@@ -2000,6 +2000,15 @@ void fuse_session_process_buf(struct fuse_session *se,
  */
 int fuse_session_receive_buf(struct fuse_session *se, struct fuse_buf *buf);
 
Quoted 20 lines
Quoted 16 lines
contrib/virtiofsd/fuse_lowlevel.h +9 -1
+/**
+ * General utility to get the length of an iov, not Fuse specific
+ *
+ * @param iov The iovec to get the length of
+ * @param count Number of entries in the iovec
+ * @return the total length of iov in bytes
+ */
+size_t iov_length(const struct iovec *iov, size_t count);
+
 #ifdef __cplusplus
 }
 #endif
--

2.23.0

Signature · 1 line
Marc-André Lureau

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#20 lore raw

PATCH 12/30 virtiofsd: Make fsync work even if only inode is passed in

In reply to Dr. David Alan Gilbert (git)

From: Vivek Goyal <[email protected]>

If caller has not sent file handle in request, then using inode, retrieve the fd opened using O_PATH and use that to open file again and issue fsync. This will be needed when dax_flush() calls fsync. At that time we only have inode information (and not file).

---
 contrib/virtiofsd/fuse_lowlevel.c  |  9 ++++++---
 contrib/virtiofsd/passthrough_ll.c | 24 ++++++++++++++++++++++--
 2 files changed, 28 insertions(+), 5 deletions(-)
contrib/virtiofsd/fuse_lowlevel.c +6 -3
index ff68ec0c9d..6fe7506ead 100644
@@ -1083,9 +1083,12 @@ static void do_fsync(fuse_req_t req, fuse_ino_t nodeid, const void *inarg)
 	memset(&fi, 0, sizeof(fi));
 	fi.fh = arg->fh;
 
-	if (req->se->op.fsync)
-		req->se->op.fsync(req, nodeid, datasync, &fi);
-	else
+	if (req->se->op.fsync) {
+		if (fi.fh == (uint64_t)-1)
+			req->se->op.fsync(req, nodeid, datasync, NULL);
+		else
+			req->se->op.fsync(req, nodeid, datasync, &fi);
+	} else
 		fuse_reply_err(req, ENOSYS);
 }
 
contrib/virtiofsd/passthrough_ll.c +22 -2
index 72fae9d10b..63a32c87c3 100644
@@ -863,10 +863,30 @@ static void lo_fsync(fuse_req_t req, fuse_ino_t ino, int datasync,
 {
 	int res;
 	(void) ino;
+	int fd;
+	char *buf;
+
+	fuse_log(FUSE_LOG_DEBUG, "lo_fsync(ino=%" PRIu64 ", fi=0x%p)\n", ino,
+		 (void *)fi);
+
+	if (!fi) {
+		res = asprintf(&buf, "/proc/self/fd/%i", lo_fd(req, ino));
+		if (res == -1)
+			return (void) fuse_reply_err(req, errno);
+
+		fd = open(buf, O_RDWR);
+		free(buf);
+		if (fd == -1)
+			return (void) fuse_reply_err(req, errno);
+	} else
+		fd = fi->fh;
+
 	if (datasync)
-		res = fdatasync(fi->fh);
+		res = fdatasync(fd);
 	else
-		res = fsync(fi->fh);
+		res = fsync(fd);
+	if (!fi)
+		close(fd);
 	fuse_reply_err(req, res == -1 ? errno : 0);
 }
 
Signature · 1 line
2.23.0
#21 lore raw

PATCH 13/30 virtiofsd: Add options for virtio

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Add options to specify parameters for virtio-fs paths, i.e.

./virtiofsd -o vhost_user_socket=/tmp/vhostqemu

---
 contrib/virtiofsd/fuse_i.h        |  1 +
 contrib/virtiofsd/fuse_lowlevel.c | 10 +++++++---
 contrib/virtiofsd/helper.c        | 18 +++++++++---------
 3 files changed, 17 insertions(+), 12 deletions(-)
contrib/virtiofsd/fuse_i.h +1 -0
index 6c3f432d0e..d65af06ff0 100644
@@ -63,6 +63,7 @@ struct fuse_session {
 	struct fuse_notify_req notify_list;
 	size_t bufsize;
 	int error;
+	char *vu_socket_path;
 };
 
 struct fuse_chan {
contrib/virtiofsd/fuse_lowlevel.c +7 -3
index 6fe7506ead..d47e4438f3 100644
@@ -2064,6 +2064,8 @@ static const struct fuse_opt fuse_ll_opts[] = {
 	LL_OPTION("-d", debug, 1),
 	LL_OPTION("--debug", debug, 1),
 	LL_OPTION("allow_root", deny_others, 1),
+	LL_OPTION("--socket-path=%s", vu_socket_path, 0),
+	LL_OPTION("vhost_user_socket=%s", vu_socket_path, 0),
 	FUSE_OPT_END
 };
 
@@ -2078,9 +2080,11 @@ void fuse_lowlevel_help(void)
 	/* These are not all options, but the ones that are
 	   potentially of interest to an end-user */
 	printf(
-"    -o allow_other         allow access by all users\n"
-"    -o allow_root          allow access by root\n"
-"    -o auto_unmount        auto unmount on process termination\n");
+"    -o allow_other             allow access by all users\n"
+"    -o allow_root              allow access by root\n"
+"    --socket-path=PATH         path for the vhost-user socket\n"
+"    -o vhost_user_socket=PATH  path for the vhost-user socket\n"
+"    -o auto_unmount            auto unmount on process termination\n");
 }
 
 void fuse_session_destroy(struct fuse_session *se)
contrib/virtiofsd/helper.c +9 -9
index 958a42b6f4..f67533f053 100644
@@ -126,15 +126,15 @@ static const struct fuse_opt conn_info_opt_spec[] = {
 
 void fuse_cmdline_help(void)
 {
-	printf("    -h   --help            print help\n"
-	       "    -V   --version         print version\n"
-	       "    -d   -o debug          enable debug output (implies -f)\n"
-	       "    -f                     foreground operation\n"
-	       "    -s                     disable multi-threaded operation\n"
-	       "    -o clone_fd            use separate fuse device fd for each thread\n"
-	       "                           (may improve performance)\n"
-	       "    -o max_idle_threads    the maximum number of idle worker threads\n"
-	       "                           allowed (default: 10)\n");
+	printf("    -h   --help                print help\n"
+	       "    -V   --version             print version\n"
+	       "    -d   -o debug              enable debug output (implies -f)\n"
+	       "    -f                         foreground operation\n"
+	       "    -s                         disable multi-threaded operation\n"
+	       "    -o clone_fd                use separate fuse device fd for each thread\n"
+	       "                               (may improve performance)\n"
+	       "    -o max_idle_threads        the maximum number of idle worker threads\n"
+	       "                               allowed (default: 10)\n");
 }
 
 static int fuse_helper_opt_proc(void *data, const char *arg, int key,
Signature · 1 line
2.23.0
#22 lore raw

PATCH 14/30 virtiofsd: add -o source=PATH to help output

In reply to Dr. David Alan Gilbert (git)

From: Stefan Hajnoczi <[email protected]>

The -o source=PATH option will be used by most command-line invocations. Let's document it!

---
 contrib/virtiofsd/passthrough_ll.c | 1 +
 1 file changed, 1 insertion(+)
contrib/virtiofsd/passthrough_ll.c +1 -0
index 63a32c87c3..847814b793 100644
@@ -1237,6 +1237,7 @@ int main(int argc, char *argv[])
 	if (opts.show_help) {
 		printf("usage: %s [options]\n\n", argv[0]);
 		fuse_cmdline_help();
+		printf("    -o source=PATH             shared directory tree\n");
 		fuse_lowlevel_help();
 		ret = 0;
 		goto err_out1;
Signature · 1 line
2.23.0
#23 lore raw

PATCH 15/30 virtiofsd: Open vhost connection instead of mounting

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

When run with vhost-user options we conect to the QEMU instead via a socket. Start this off by creating the socket.

---
 contrib/virtiofsd/fuse_i.h        |  7 ++-
 contrib/virtiofsd/fuse_lowlevel.c | 53 +++-------------------
 contrib/virtiofsd/fuse_virtio.c   | 75 +++++++++++++++++++++++++++++++
 contrib/virtiofsd/fuse_virtio.h   | 23 ++++++++++
 4 files changed, 110 insertions(+), 48 deletions(-)
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
contrib/virtiofsd/fuse_i.h +5 -2
index d65af06ff0..27fa50274d 100644
@@ -6,9 +6,10 @@
   See the file COPYING.LIB
 */
 
-#define FUSE_USE_VERSION 31
-
+#ifndef FUSE_I_H
+#define FUSE_I_H
 
+#define FUSE_USE_VERSION 31
 #include "fuse.h"
 #include "fuse_lowlevel.h"
 
@@ -119,3 +120,5 @@ void fuse_session_process_buf_int(struct fuse_session *se,
 
 /* room needed in buffer to accommodate header */
 #define FUSE_BUFFER_HEADER_SIZE 0x1000
+
+#endif
contrib/virtiofsd/fuse_lowlevel.c +7 -46

71 lines not on this page · raw message

contrib/virtiofsd/fuse_virtio.c new file +75 -0

75 lines not on this page · raw message

contrib/virtiofsd/fuse_virtio.h new file +23 -0
new file mode 100644
index 0000000000..6652e609e0
@@ -0,0 +1,23 @@
+/*
+  virtio-fs glue for FUSE
+  Copyright (C) 2018 Red Hat, Inc. and/or its affiliates
+
+  Authors:
+    Dave Gilbert  <[email protected]>
+
+  Implements the glue between libfuse and libvhost-user
+
+  This program can be distributed under the terms of the GNU LGPLv2.
+  See the file COPYING.LIB
+*/
+
+#ifndef FUSE_VIRTIO_H
+#define FUSE_VIRTIO_H
+
+#include "fuse_i.h"
+
+struct fuse_session;
+
+int virtio_session_mount(struct fuse_session *se);
+
+#endif
Signature · 1 line
2.23.0
#24 lore raw

PATCH 16/30 virtiofsd: Start wiring up vhost-user

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Listen on our unix socket for the connection from QEMU, when we get it initialise vhost-user and dive into our own loop variant (currently dummy).

---
 contrib/virtiofsd/fuse_i.h         |  4 ++
 contrib/virtiofsd/fuse_loop_mt.c   |  1 +
 contrib/virtiofsd/fuse_lowlevel.c  |  5 ++
 contrib/virtiofsd/fuse_lowlevel.h  |  7 +++
 contrib/virtiofsd/fuse_virtio.c    | 81 +++++++++++++++++++++++++++++-
 contrib/virtiofsd/fuse_virtio.h    |  2 +
 contrib/virtiofsd/passthrough_ll.c |  6 +--
 7 files changed, 101 insertions(+), 5 deletions(-)
contrib/virtiofsd/fuse_i.h +4 -0
index 27fa50274d..ea55f618a5 100644
@@ -13,6 +13,8 @@
 #include "fuse.h"
 #include "fuse_lowlevel.h"
 
+struct fv_VuDev;
+
 struct fuse_req {
 	struct fuse_session *se;
 	uint64_t unique;
@@ -65,6 +67,8 @@ struct fuse_session {
 	size_t bufsize;
 	int error;
 	char *vu_socket_path;
+	int   vu_socketfd;
+	struct fv_VuDev *virtio_dev;
 };
 
 struct fuse_chan {
contrib/virtiofsd/fuse_loop_mt.c +1 -0
index e45b2337a0..2000a8902a 100644
@@ -12,6 +12,7 @@
 #include "fuse_lowlevel.h"
 #include "fuse_misc.h"
 #include "fuse_kernel.h"
+#include "fuse_virtio.h"
 
 #include <stdio.h>
 #include <stdlib.h>
contrib/virtiofsd/fuse_lowlevel.c +5 -0
index 247667e3f4..328a9aa001 100644
@@ -2193,6 +2193,11 @@ void fuse_session_unmount(struct fuse_session *se)
 {
 }
 
+int fuse_lowlevel_is_virtio(struct fuse_session *se)
+{
+	return se->vu_socket_path != NULL;
+}
+
 #ifdef linux
 int fuse_req_getgroups(fuse_req_t req, int size, gid_t list[])
 {
contrib/virtiofsd/fuse_lowlevel.h +7 -0
index 84379a4761..b06ab6915a 100644
@@ -1756,6 +1756,13 @@ void fuse_req_interrupt_func(fuse_req_t req, fuse_interrupt_func_t func,
  */
 int fuse_req_interrupted(fuse_req_t req);
 
+/**
+ * Check if the session is connected via virtio
+ *
+ * @param se session object
+ * @return 1 if the session is a virtio session
+ */
+int fuse_lowlevel_is_virtio(struct fuse_session *se);
 
 /* ----------------------------------------------------------- *
  * Inquiry functions                                           *
contrib/virtiofsd/fuse_virtio.c +80 -1

103 lines not on this page · raw message

contrib/virtiofsd/fuse_virtio.h +2 -0
index 6652e609e0..025a4d30ae 100644
@@ -20,4 +20,6 @@ struct fuse_session;
 
 int virtio_session_mount(struct fuse_session *se);
 
+int virtio_loop(struct fuse_session *se);
+
 #endif
contrib/virtiofsd/passthrough_ll.c +2 -4
index 847814b793..c09fc4a65c 100644
@@ -34,6 +34,7 @@
  * \include passthrough_ll.c
  */
 
+#include "fuse_virtio.h"
 #include "fuse_lowlevel.h"
 #include <unistd.h>
 #include <stdlib.h>
@@ -1311,10 +1312,7 @@ int main(int argc, char *argv[])
 	fuse_daemonize(opts.foreground);
 
 	/* Block until ctrl+c or fusermount -u */
-	if (opts.singlethread)
-		ret = fuse_session_loop(se);
-	else
-		ret = fuse_session_loop_mt(se, opts.clone_fd);
+	ret = virtio_loop(se);
 
 	fuse_session_unmount(se);
 err_out3:
Signature · 1 line
2.23.0
#25 lore raw

PATCH 17/30 virtiofsd: Add main virtio loop

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Processes incoming requests on the vhost-user fd.

---
 contrib/virtiofsd/fuse_virtio.c | 42 ++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_virtio.c +39 -3
index 22f71d260f..9c58f11634 100644
@@ -11,12 +11,14 @@
  * See the file COPYING.LIB
  */
 
+#include "fuse_virtio.h"
 #include "fuse_i.h"
 #include "fuse_kernel.h"
 #include "fuse_misc.h"
 #include "fuse_opt.h"
-#include "fuse_virtio.h"
 
+#include <assert.h>
+#include <errno.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -80,15 +82,49 @@ static const VuDevIface fv_iface = {
     .queue_is_processed_in_order = fv_queue_order,
 };
 
+/*
+ * Main loop; this mostly deals with events on the vhost-user
+ * socket itself, and not actual fuse data.
+ */
 int virtio_loop(struct fuse_session *se)
 {
     fuse_log(FUSE_LOG_INFO, "%s: Entry\n", __func__);
 
-    while (1) {
-        /* TODO: Add stuffing */
+    while (!fuse_session_exited(se)) {
+        struct pollfd pf[1];
+        pf[0].fd = se->vu_socketfd;
+        pf[0].events = POLLIN;
+        pf[0].revents = 0;
+
+        fuse_log(FUSE_LOG_DEBUG, "%s: Waiting for VU event\n", __func__);
+        int poll_res = ppoll(pf, 1, NULL, NULL);
+
+        if (poll_res == -1) {
+            if (errno == EINTR) {
+                fuse_log(FUSE_LOG_INFO, "%s: ppoll interrupted, going around\n",
+                         __func__);
+                continue;
+            }
+            fuse_log(FUSE_LOG_ERR, "virtio_loop ppoll: %m\n");
+            break;
+        }
+        assert(poll_res == 1);
+        if (pf[0].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+            fuse_log(FUSE_LOG_ERR, "%s: Unexpected poll revents %x\n", __func__,
+                     pf[0].revents);
+            break;
+        }
+        assert(pf[0].revents & POLLIN);
+        fuse_log(FUSE_LOG_DEBUG, "%s: Got VU event\n", __func__);
+        if (!vu_dispatch(&se->virtio_dev->dev)) {
+            fuse_log(FUSE_LOG_ERR, "%s: vu_dispatch failed\n", __func__);
+            break;
+        }
     }
 
     fuse_log(FUSE_LOG_INFO, "%s: Exit\n", __func__);
+
+    return 0;
 }
 
 int virtio_session_mount(struct fuse_session *se)
Signature · 1 line
2.23.0
#26 lore raw

Re: [PATCH 17/30] virtiofsd: Add main virtio loop

Quoted 4 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Processes incoming requests on the vhost-user fd.

Is there a reason to avoid using glib & its main loop?

Quoted 82 lines
---
 contrib/virtiofsd/fuse_virtio.c | 42 ++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_virtio.c +39 -4
index 22f71d260f..9c58f11634 100644
@@ -11,12 +11,14 @@
  * See the file COPYING.LIB
  */
 
+#include "fuse_virtio.h"
 #include "fuse_i.h"
 #include "fuse_kernel.h"
 #include "fuse_misc.h"
 #include "fuse_opt.h"
-#include "fuse_virtio.h"
 
+#include <assert.h>
+#include <errno.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -80,15 +82,49 @@ static const VuDevIface fv_iface = {
     .queue_is_processed_in_order = fv_queue_order,
 };
 
+/*
+ * Main loop; this mostly deals with events on the vhost-user
+ * socket itself, and not actual fuse data.
+ */
 int virtio_loop(struct fuse_session *se)
 {
     fuse_log(FUSE_LOG_INFO, "%s: Entry\n", __func__);
 
-    while (1) {
-        /* TODO: Add stuffing */
+    while (!fuse_session_exited(se)) {
+        struct pollfd pf[1];
+        pf[0].fd = se->vu_socketfd;
+        pf[0].events = POLLIN;
+        pf[0].revents = 0;
+
+        fuse_log(FUSE_LOG_DEBUG, "%s: Waiting for VU event\n", __func__);
+        int poll_res = ppoll(pf, 1, NULL, NULL);
+
+        if (poll_res == -1) {
+            if (errno == EINTR) {
+                fuse_log(FUSE_LOG_INFO, "%s: ppoll interrupted, going around\n",
+                         __func__);
+                continue;
+            }
+            fuse_log(FUSE_LOG_ERR, "virtio_loop ppoll: %m\n");
+            break;
+        }
+        assert(poll_res == 1);
+        if (pf[0].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+            fuse_log(FUSE_LOG_ERR, "%s: Unexpected poll revents %x\n", __func__,
+                     pf[0].revents);
+            break;
+        }
+        assert(pf[0].revents & POLLIN);
+        fuse_log(FUSE_LOG_DEBUG, "%s: Got VU event\n", __func__);
+        if (!vu_dispatch(&se->virtio_dev->dev)) {
+            fuse_log(FUSE_LOG_ERR, "%s: vu_dispatch failed\n", __func__);
+            break;
+        }
     }
 
     fuse_log(FUSE_LOG_INFO, "%s: Exit\n", __func__);
+
+    return 0;
 }
 
 int virtio_session_mount(struct fuse_session *se)
--

2.23.0

Signature · 1 line
Marc-André Lureau
#27 lore raw

Re: [PATCH 17/30] virtiofsd: Add main virtio loop

Quoted 8 lines · * Marc-André Lureau ([email protected]) wrote:
Quoted 4 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Processes incoming requests on the vhost-user fd.

Is there a reason to avoid using glib & its main loop?

Not particularly; would it actually work out any easier? Is there an easy way to get the glib loop to check something like the fuse_session_exited at each iteration.

I probably didn't originally because I wasn't sure if this code was going back into libfuse or staying here; and libfuse doesn't use glib.

Dave

Quoted 77 lines
Quoted 77 lines
---
 contrib/virtiofsd/fuse_virtio.c | 42 ++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_virtio.c +39 -3
index 22f71d260f..9c58f11634 100644
@@ -11,12 +11,14 @@
  * See the file COPYING.LIB
  */
 
+#include "fuse_virtio.h"
 #include "fuse_i.h"
 #include "fuse_kernel.h"
 #include "fuse_misc.h"
 #include "fuse_opt.h"
-#include "fuse_virtio.h"
 
+#include <assert.h>
+#include <errno.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -80,15 +82,49 @@ static const VuDevIface fv_iface = {
     .queue_is_processed_in_order = fv_queue_order,
 };
 
+/*
+ * Main loop; this mostly deals with events on the vhost-user
+ * socket itself, and not actual fuse data.
+ */
 int virtio_loop(struct fuse_session *se)
 {
     fuse_log(FUSE_LOG_INFO, "%s: Entry\n", __func__);
 
-    while (1) {
-        /* TODO: Add stuffing */
+    while (!fuse_session_exited(se)) {
+        struct pollfd pf[1];
+        pf[0].fd = se->vu_socketfd;
+        pf[0].events = POLLIN;
+        pf[0].revents = 0;
+
+        fuse_log(FUSE_LOG_DEBUG, "%s: Waiting for VU event\n", __func__);
+        int poll_res = ppoll(pf, 1, NULL, NULL);
+
+        if (poll_res == -1) {
+            if (errno == EINTR) {
+                fuse_log(FUSE_LOG_INFO, "%s: ppoll interrupted, going around\n",
+                         __func__);
+                continue;
+            }
+            fuse_log(FUSE_LOG_ERR, "virtio_loop ppoll: %m\n");
+            break;
+        }
+        assert(poll_res == 1);
+        if (pf[0].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+            fuse_log(FUSE_LOG_ERR, "%s: Unexpected poll revents %x\n", __func__,
+                     pf[0].revents);
+            break;
+        }
+        assert(pf[0].revents & POLLIN);
+        fuse_log(FUSE_LOG_DEBUG, "%s: Got VU event\n", __func__);
+        if (!vu_dispatch(&se->virtio_dev->dev)) {
+            fuse_log(FUSE_LOG_ERR, "%s: vu_dispatch failed\n", __func__);
+            break;
+        }
     }
 
     fuse_log(FUSE_LOG_INFO, "%s: Exit\n", __func__);
+
+    return 0;
 }
 
Quoted 9 lines
Quoted 5 lines
contrib/virtiofsd/fuse_virtio.c +0 -1
 int virtio_session_mount(struct fuse_session *se)
--

2.23.0

Signature · 1 line
Marc-André Lureau

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#28 lore raw

Re: [PATCH 17/30] virtiofsd: Add main virtio loop

Quoted 10 lines · Dr. David Alan Gilbert
Quoted 8 lines · * Marc-André Lureau ([email protected]) wrote:
Quoted 4 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Processes incoming requests on the vhost-user fd.

Is there a reason to avoid using glib & its main loop?

Quoted 7 lines

Not particularly; would it actually work out any easier? Is there an easy way to get the glib loop to check something like the fuse_session_exited at each iteration.

I probably didn't originally because I wasn't sure if this code was going back into libfuse or staying here; and libfuse doesn't use glib.

I tried to convert this to glib a few months ago, but stopped because libvhost-user-glib.c is difficult to use from multiple threads. If I remember correctly we'd need to change it quite a bit to make multi-threaded vhost-user device backends possible and I gave up.

Stefan

#29 lore raw

PATCH 18/30 virtiofsd: get/set features callbacks

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Add the get/set features callbacks.

---
 contrib/virtiofsd/fuse_virtio.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_virtio.c +14 -1
index 9c58f11634..ee9b3af3a4 100644
@@ -46,6 +46,17 @@ struct virtio_fs_config {
     uint32_t num_queues;
 };
 
+/* Callback from libvhost-user */
+static uint64_t fv_get_features(VuDev *dev)
+{
+    return 1ULL << VIRTIO_F_VERSION_1;
+}
+
+/* Callback from libvhost-user */
+static void fv_set_features(VuDev *dev, uint64_t features)
+{
+}
+
 /*
  * Callback from libvhost-user if there's a new fd we're supposed to listen
  * to, typically a queue kick?
@@ -78,7 +89,9 @@ static bool fv_queue_order(VuDev *dev, int qidx)
 }
 
 static const VuDevIface fv_iface = {
-    /* TODO: Add other callbacks */
+    .get_features = fv_get_features,
+    .set_features = fv_set_features,
+
     .queue_is_processed_in_order = fv_queue_order,
 };
 
Signature · 1 line
2.23.0
#30 lore raw

PATCH 19/30 virtiofsd: Start queue threads

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Start a thread for each queue when we get notified it's been started.

fix by:

---
 contrib/virtiofsd/fuse_virtio.c | 89 +++++++++++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
contrib/virtiofsd/fuse_virtio.c +89 -0
index ee9b3af3a4..66b8421899 100644
@@ -11,6 +11,7 @@
  * See the file COPYING.LIB
  */
 
+#include "qemu/osdep.h"
 #include "fuse_virtio.h"
 #include "fuse_i.h"
 #include "fuse_kernel.h"
@@ -30,6 +31,15 @@
 
 #include "contrib/libvhost-user/libvhost-user.h"
 
+struct fv_QueueInfo {
+    pthread_t thread;
+    struct fv_VuDev *virtio_dev;
+
+    /* Our queue index, corresponds to array position */
+    int qidx;
+    int kick_fd;
+};
+
 /*
  * We pass the dev element into libvhost-user
  * and then use it to get back to the outer
@@ -38,6 +48,13 @@
 struct fv_VuDev {
     VuDev dev;
     struct fuse_session *se;
+
+    /*
+     * The following pair of fields are only accessed in the main
+     * virtio_loop
+     */
+    size_t nqueues;
+    struct fv_QueueInfo **qi;
 };
 
 /* From spec */
@@ -83,6 +100,75 @@ static void fv_panic(VuDev *dev, const char *err)
     exit(EXIT_FAILURE);
 }
 
+static void *fv_queue_thread(void *opaque)
+{
+    struct fv_QueueInfo *qi = opaque;
+    fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
+             qi->qidx, qi->kick_fd);
+    while (1) {
+        /* TODO */
+    }
+
+    return NULL;
+}
+
+/* Callback from libvhost-user on start or stop of a queue */
+static void fv_queue_set_started(VuDev *dev, int qidx, bool started)
+{
+    struct fv_VuDev *vud = container_of(dev, struct fv_VuDev, dev);
+    struct fv_QueueInfo *ourqi;
+
+    fuse_log(FUSE_LOG_INFO, "%s: qidx=%d started=%d\n", __func__, qidx,
+             started);
+    assert(qidx >= 0);
+
+    /*
+     * Ignore additional request queues for now.  passthrough_ll.c must be
+     * audited for thread-safety issues first.  It was written with a
+     * well-behaved client in mind and may not protect against all types of
+     * races yet.
+     */
+    if (qidx > 1) {
+        fuse_log(FUSE_LOG_ERR,
+                 "%s: multiple request queues not yet implemented, please only "
+                 "configure 1 request queue\n",
+                 __func__);
+        exit(EXIT_FAILURE);
+    }
+
+    if (started) {
+        /* Fire up a thread to watch this queue */
+        if (qidx >= vud->nqueues) {
+            vud->qi = realloc(vud->qi, (qidx + 1) * sizeof(vud->qi[0]));
+            assert(vud->qi);
+            memset(vud->qi + vud->nqueues, 0,
+                   sizeof(vud->qi[0]) * (1 + (qidx - vud->nqueues)));
+            vud->nqueues = qidx + 1;
+        }
+        if (!vud->qi[qidx]) {
+            vud->qi[qidx] = calloc(sizeof(struct fv_QueueInfo), 1);
+            assert(vud->qi[qidx]);
+            vud->qi[qidx]->virtio_dev = vud;
+            vud->qi[qidx]->qidx = qidx;
+        } else {
+            /* Shouldn't have been started */
+            assert(vud->qi[qidx]->kick_fd == -1);
+        }
+        ourqi = vud->qi[qidx];
+        ourqi->kick_fd = dev->vq[qidx].kick_fd;
+        if (pthread_create(&ourqi->thread, NULL, fv_queue_thread, ourqi)) {
+            fuse_log(FUSE_LOG_ERR, "%s: Failed to create thread for queue %d\n",
+                     __func__, qidx);
+            assert(0);
+        }
+    } else {
+        /* TODO: Kill the thread */
+        assert(qidx < vud->nqueues);
+        ourqi = vud->qi[qidx];
+        ourqi->kick_fd = -1;
+    }
+}
+
 static bool fv_queue_order(VuDev *dev, int qidx)
 {
     return false;
@@ -92,6 +178,9 @@ static const VuDevIface fv_iface = {
     .get_features = fv_get_features,
     .set_features = fv_set_features,
 
+    /* Don't need process message, we've not got any at vhost-user level */
+    .queue_set_started = fv_queue_set_started,
+
     .queue_is_processed_in_order = fv_queue_order,
 };
 
Signature · 1 line
2.23.0
#31 lore raw

PATCH 20/30 virtiofsd: Poll kick_fd for queue

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

In the queue thread poll the kick_fd we're passed.

---
 contrib/virtiofsd/fuse_virtio.c | 40 ++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_virtio.c +39 -1
index 66b8421899..0513531c13 100644
@@ -24,6 +24,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/eventfd.h>
 #include <sys/socket.h>
 #include <sys/types.h>
 #include <sys/un.h>
@@ -100,13 +101,50 @@ static void fv_panic(VuDev *dev, const char *err)
     exit(EXIT_FAILURE);
 }
 
+/* Thread function for individual queues, created when a queue is 'started' */
 static void *fv_queue_thread(void *opaque)
 {
     struct fv_QueueInfo *qi = opaque;
     fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
              qi->qidx, qi->kick_fd);
     while (1) {
-        /* TODO */
+        struct pollfd pf[1];
+        pf[0].fd = qi->kick_fd;
+        pf[0].events = POLLIN;
+        pf[0].revents = 0;
+
+        fuse_log(FUSE_LOG_DEBUG, "%s: Waiting for Queue %d event\n", __func__,
+                 qi->qidx);
+        int poll_res = ppoll(pf, 1, NULL, NULL);
+
+        if (poll_res == -1) {
+            if (errno == EINTR) {
+                fuse_log(FUSE_LOG_INFO, "%s: ppoll interrupted, going around\n",
+                         __func__);
+                continue;
+            }
+            fuse_log(FUSE_LOG_ERR, "fv_queue_thread ppoll: %m\n");
+            break;
+        }
+        assert(poll_res == 1);
+        if (pf[0].revents & (POLLERR | POLLHUP | POLLNVAL)) {
+            fuse_log(FUSE_LOG_ERR, "%s: Unexpected poll revents %x Queue %d\n",
+                     __func__, pf[0].revents, qi->qidx);
+            break;
+        }
+        assert(pf[0].revents & POLLIN);
+        fuse_log(FUSE_LOG_DEBUG, "%s: Got queue event on Queue %d\n", __func__,
+                 qi->qidx);
+
+        eventfd_t evalue;
+        if (eventfd_read(qi->kick_fd, &evalue)) {
+            fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
+            break;
+        }
+        if (qi->virtio_dev->se->debug) {
+            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
+                    qi->qidx, (size_t)evalue);
+        }
     }
 
     return NULL;
Signature · 1 line
2.23.0
#32 lore raw

PATCH 21/30 virtiofsd: Start reading commands from queue

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Pop queue elements off queues, copy the data from them and pass that to fuse.

Note: 'out' in a VuVirtqElement is from QEMU
'in' in libfuse is into the daemon

So we read from the out iov's to get a fuse_in_header

When we get a kick we've got to read all the elements until the queue is empty.

---
 contrib/virtiofsd/fuse_i.h      |  2 +
 contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
 2 files changed, 98 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_i.h +2 -0
index ea55f618a5..5b6ef09df5 100644
@@ -14,6 +14,7 @@
 #include "fuse_lowlevel.h"
 
 struct fv_VuDev;
+struct fv_QueueInfo;
 
 struct fuse_req {
 	struct fuse_session *se;
@@ -75,6 +76,7 @@ struct fuse_chan {
 	pthread_mutex_t lock;
 	int ctr;
 	int fd;
+	struct fv_QueueInfo *qi;
 };
 
 /**
contrib/virtiofsd/fuse_virtio.c +96 -3

123 lines not on this page · raw message

Signature · 1 line
2.23.0
#33 lore raw

Re: [PATCH 21/30] virtiofsd: Start reading commands from queue

Quoted 63 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Pop queue elements off queues, copy the data from them and pass that to fuse.

Note: 'out' in a VuVirtqElement is from QEMU
'in' in libfuse is into the daemon

So we read from the out iov's to get a fuse_in_header

When we get a kick we've got to read all the elements until the queue is empty.

---
 contrib/virtiofsd/fuse_i.h      |  2 +
 contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
 2 files changed, 98 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_i.h +2 -0
index ea55f618a5..5b6ef09df5 100644
@@ -14,6 +14,7 @@
 #include "fuse_lowlevel.h"
 
 struct fv_VuDev;
+struct fv_QueueInfo;
 
 struct fuse_req {
        struct fuse_session *se;
@@ -75,6 +76,7 @@ struct fuse_chan {
        pthread_mutex_t lock;
        int ctr;
        int fd;
+       struct fv_QueueInfo *qi;
 };
 
 /**
contrib/virtiofsd/fuse_virtio.c +8 -0
index 0513531c13..e189511729 100644
@@ -32,6 +32,7 @@
 
 #include "contrib/libvhost-user/libvhost-user.h"
 
+struct fv_VuDev;
 struct fv_QueueInfo {
     pthread_t thread;
     struct fv_VuDev *virtio_dev;
@@ -101,10 +102,42 @@ static void fv_panic(VuDev *dev, const char *err)
     exit(EXIT_FAILURE);
 }
 
+/*
+ * Copy from an iovec into a fuse_buf (memory only)
+ * Caller must ensure there is space
+ */
+static void copy_from_iov(struct fuse_buf *buf, size_t out_num,
+                          const struct iovec *out_sg)
+{

contrib/virtiofsd/fuse_virtio.c static void fv_panic(VuDev *dev, const char *err) · line 121

Quoted 10 lines
contrib/virtiofsd/fuse_virtio.c +10 -0
+    void *dest = buf->mem;
+
+    while (out_num) {
+        size_t onelen = out_sg->iov_len;
+        memcpy(dest, out_sg->iov_base, onelen);
+        dest += onelen;
+        out_sg++;
+        out_num--;
+    }
+}

Or iov_to_buf()

Just found out that vhost-user-blk.c should probably link too with util/iov.o, instead of copying the code.

Quoted 101 lines
contrib/virtiofsd/fuse_virtio.c +78 -4
+
 /* Thread function for individual queues, created when a queue is 'started' */
 static void *fv_queue_thread(void *opaque)
 {
     struct fv_QueueInfo *qi = opaque;
+    struct VuDev *dev = &qi->virtio_dev->dev;
+    struct VuVirtq *q = vu_get_queue(dev, qi->qidx);
+    struct fuse_session *se = qi->virtio_dev->se;
+    struct fuse_chan ch;
+    struct fuse_buf fbuf;
+
+    fbuf.mem = NULL;
+    fbuf.flags = 0;
+
+    fuse_mutex_init(&ch.lock);
+    ch.fd = (int)0xdaff0d111;
+    ch.ctr = 1;
+    ch.qi = qi;
+
     fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
              qi->qidx, qi->kick_fd);
     while (1) {
@@ -141,11 +174,71 @@ static void *fv_queue_thread(void *opaque)
             fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
             break;
         }
-        if (qi->virtio_dev->se->debug) {
-            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
-                    qi->qidx, (size_t)evalue);
+        /* out is from guest, in is too guest */
+        unsigned int in_bytes, out_bytes;
+        vu_queue_get_avail_bytes(dev, q, &in_bytes, &out_bytes, ~0, ~0);
+
+        fuse_log(FUSE_LOG_DEBUG,
+                 "%s: Queue %d gave evalue: %zx available: in: %u out: %u\n",
+                 __func__, qi->qidx, (size_t)evalue, in_bytes, out_bytes);
+
+        while (1) {
+            /*
+             * An element contains one request and the space to send our
+             * response They're spread over multiple descriptors in a
+             * scatter/gather set and we can't trust the guest to keep them
+             * still; so copy in/out.
+             */
+            VuVirtqElement *elem = vu_queue_pop(dev, q, sizeof(VuVirtqElement));
+            if (!elem) {
+                break;
+            }
+
+            if (!fbuf.mem) {
+                fbuf.mem = malloc(se->bufsize);
+                assert(fbuf.mem);
+                assert(se->bufsize > sizeof(struct fuse_in_header));
+            }
+            /* The 'out' part of the elem is from qemu */
+            unsigned int out_num = elem->out_num;
+            struct iovec *out_sg = elem->out_sg;
+            size_t out_len = iov_length(out_sg, out_num);
+            fuse_log(FUSE_LOG_DEBUG,
+                     "%s: elem %d: with %d out desc of length %zd\n", __func__,
+                     elem->index, out_num, out_len);
+
+            /*
+             * The elem should contain a 'fuse_in_header' (in to fuse)
+             * plus the data based on the len in the header.
+             */
+            if (out_len < sizeof(struct fuse_in_header)) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for in_header\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            if (out_len > se->bufsize) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too large for buffer\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            copy_from_iov(&fbuf, out_num, out_sg);
+            fbuf.size = out_len;
+
+            /* TODO! Endianness of header */
+
+            /* TODO: Fixup fuse_send_msg */
+            /* TODO: Add checks for fuse_session_exited */
+            fuse_session_process_buf_int(se, &fbuf, &ch);
+
+            /* TODO: vu_queue_push(dev, q, elem, qi->write_count); */
+            vu_queue_notify(dev, q);
+
+            free(elem);
+            elem = NULL;
         }
     }
+    pthread_mutex_destroy(&ch.lock);
+    free(fbuf.mem);
 
     return NULL;
 }
--

2.23.0

Signature · 1 line
Marc-André Lureau
#34 lore raw

Re: [PATCH 21/30] virtiofsd: Start reading commands from queue

Quoted 69 lines · * Marc-André Lureau ([email protected]) wrote:
Quoted 67 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Pop queue elements off queues, copy the data from them and pass that to fuse.

Note: 'out' in a VuVirtqElement is from QEMU
'in' in libfuse is into the daemon

So we read from the out iov's to get a fuse_in_header

When we get a kick we've got to read all the elements until the queue is empty.

---
 contrib/virtiofsd/fuse_i.h      |  2 +
 contrib/virtiofsd/fuse_virtio.c | 99 ++++++++++++++++++++++++++++++++-
 2 files changed, 98 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_i.h +2 -0
index ea55f618a5..5b6ef09df5 100644
@@ -14,6 +14,7 @@
 #include "fuse_lowlevel.h"
 
 struct fv_VuDev;
+struct fv_QueueInfo;
 
 struct fuse_req {
        struct fuse_session *se;
@@ -75,6 +76,7 @@ struct fuse_chan {
        pthread_mutex_t lock;
        int ctr;
        int fd;
+       struct fv_QueueInfo *qi;
 };
 
 /**
contrib/virtiofsd/fuse_virtio.c +12 -0
index 0513531c13..e189511729 100644
@@ -32,6 +32,7 @@
 
 #include "contrib/libvhost-user/libvhost-user.h"
 
+struct fv_VuDev;
 struct fv_QueueInfo {
     pthread_t thread;
     struct fv_VuDev *virtio_dev;
@@ -101,10 +102,42 @@ static void fv_panic(VuDev *dev, const char *err)
     exit(EXIT_FAILURE);
 }
 
+/*
+ * Copy from an iovec into a fuse_buf (memory only)
+ * Caller must ensure there is space
+ */
+static void copy_from_iov(struct fuse_buf *buf, size_t out_num,
+                          const struct iovec *out_sg)
+{
+    void *dest = buf->mem;
+
+    while (out_num) {
+        size_t onelen = out_sg->iov_len;
Quoted 8 lines

contrib/virtiofsd/fuse_virtio.c static void fv_panic(VuDev *dev, const char *err) · line 121

Quoted 6 lines
contrib/virtiofsd/fuse_virtio.c +6 -0
+        memcpy(dest, out_sg->iov_base, onelen);
+        dest += onelen;
+        out_sg++;
+        out_num--;
+    }
+}

Or iov_to_buf()

Is there a variant there that will copy the whole of an iovec? I just tried swapping the copy_from_iov out, but it's messier to use because it wants the bytes length; where in most cases I'm taking a single element from the iovec, or the 'rest of the iovec' by element count.

Dave

Quoted 98 lines

Just found out that vhost-user-blk.c should probably link too with util/iov.o, instead of copying the code.

Quoted 95 lines
contrib/virtiofsd/fuse_virtio.c +78 -3
+
 /* Thread function for individual queues, created when a queue is 'started' */
 static void *fv_queue_thread(void *opaque)
 {
     struct fv_QueueInfo *qi = opaque;
+    struct VuDev *dev = &qi->virtio_dev->dev;
+    struct VuVirtq *q = vu_get_queue(dev, qi->qidx);
+    struct fuse_session *se = qi->virtio_dev->se;
+    struct fuse_chan ch;
+    struct fuse_buf fbuf;
+
+    fbuf.mem = NULL;
+    fbuf.flags = 0;
+
+    fuse_mutex_init(&ch.lock);
+    ch.fd = (int)0xdaff0d111;
+    ch.ctr = 1;
+    ch.qi = qi;
+
     fuse_log(FUSE_LOG_INFO, "%s: Start for queue %d kick_fd %d\n", __func__,
              qi->qidx, qi->kick_fd);
     while (1) {
@@ -141,11 +174,71 @@ static void *fv_queue_thread(void *opaque)
             fuse_log(FUSE_LOG_ERR, "Eventfd_read for queue: %m\n");
             break;
         }
-        if (qi->virtio_dev->se->debug) {
-            fprintf(stderr, "%s: Queue %d gave evalue: %zx\n", __func__,
-                    qi->qidx, (size_t)evalue);
+        /* out is from guest, in is too guest */
+        unsigned int in_bytes, out_bytes;
+        vu_queue_get_avail_bytes(dev, q, &in_bytes, &out_bytes, ~0, ~0);
+
+        fuse_log(FUSE_LOG_DEBUG,
+                 "%s: Queue %d gave evalue: %zx available: in: %u out: %u\n",
+                 __func__, qi->qidx, (size_t)evalue, in_bytes, out_bytes);
+
+        while (1) {
+            /*
+             * An element contains one request and the space to send our
+             * response They're spread over multiple descriptors in a
+             * scatter/gather set and we can't trust the guest to keep them
+             * still; so copy in/out.
+             */
+            VuVirtqElement *elem = vu_queue_pop(dev, q, sizeof(VuVirtqElement));
+            if (!elem) {
+                break;
+            }
+
+            if (!fbuf.mem) {
+                fbuf.mem = malloc(se->bufsize);
+                assert(fbuf.mem);
+                assert(se->bufsize > sizeof(struct fuse_in_header));
+            }
+            /* The 'out' part of the elem is from qemu */
+            unsigned int out_num = elem->out_num;
+            struct iovec *out_sg = elem->out_sg;
+            size_t out_len = iov_length(out_sg, out_num);
+            fuse_log(FUSE_LOG_DEBUG,
+                     "%s: elem %d: with %d out desc of length %zd\n", __func__,
+                     elem->index, out_num, out_len);
+
+            /*
+             * The elem should contain a 'fuse_in_header' (in to fuse)
+             * plus the data based on the len in the header.
+             */
+            if (out_len < sizeof(struct fuse_in_header)) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for in_header\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            if (out_len > se->bufsize) {
+                fuse_log(FUSE_LOG_ERR, "%s: elem %d too large for buffer\n",
+                         __func__, elem->index);
+                assert(0); /* TODO */
+            }
+            copy_from_iov(&fbuf, out_num, out_sg);
+            fbuf.size = out_len;
+
+            /* TODO! Endianness of header */
+
+            /* TODO: Fixup fuse_send_msg */
+            /* TODO: Add checks for fuse_session_exited */
+            fuse_session_process_buf_int(se, &fbuf, &ch);
+
+            /* TODO: vu_queue_push(dev, q, elem, qi->write_count); */
+            vu_queue_notify(dev, q);
+
+            free(elem);
+            elem = NULL;
         }
     }
+    pthread_mutex_destroy(&ch.lock);
+    free(fbuf.mem);
 
Quoted 10 lines
Quoted 6 lines
contrib/virtiofsd/fuse_virtio.c +0 -1
     return NULL;
 }
--

2.23.0

Signature · 1 line
Marc-André Lureau

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#35 lore raw

PATCH 22/30 virtiofsd: Send replies to messages

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Route fuse out messages back through the same queue elements that had the command that triggered the request.

---
 contrib/virtiofsd/fuse_lowlevel.c |   4 ++
 contrib/virtiofsd/fuse_virtio.c   | 107 ++++++++++++++++++++++++++++--
 contrib/virtiofsd/fuse_virtio.h   |   4 ++
 3 files changed, 111 insertions(+), 4 deletions(-)
contrib/virtiofsd/fuse_lowlevel.c +4 -0
index 328a9aa001..fc7707b330 100644
@@ -171,6 +171,10 @@ static int fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch,
 		}
 	}
 
+	if (fuse_lowlevel_is_virtio(se)) {
+		return virtio_send_msg(se, ch, iov, count);
+	}
+
 	abort(); /* virtio should have taken it before here */
 	return 0;
 }
contrib/virtiofsd/fuse_virtio.c +103 -4

128 lines not on this page · raw message

contrib/virtiofsd/fuse_virtio.h +4 -0
index 025a4d30ae..f1e7391e44 100644
@@ -22,4 +22,8 @@ int virtio_session_mount(struct fuse_session *se);
 
 int virtio_loop(struct fuse_session *se);
 
+
+int virtio_send_msg(struct fuse_session *se, struct fuse_chan *ch,
+                    struct iovec *iov, int count);
+
 #endif
Signature · 1 line
2.23.0
#36 lore raw

PATCH 23/30 virtiofsd: Keep track of replies

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Keep track of whether we sent a reply to a request; this is a bit paranoid but it means:
a) We should always recycle an element even if there was an error in the request
b) Never try and send two replies on one queue element

---
 contrib/virtiofsd/fuse_virtio.c | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)
contrib/virtiofsd/fuse_virtio.c +20 -3
index 2dae08c26f..8dac47d42a 100644
@@ -43,6 +43,7 @@ struct fv_QueueInfo {
 
     /* The element for the command currently being processed */
     VuVirtqElement *qe;
+    bool reply_sent;
 };
 
 /*
@@ -177,6 +178,7 @@ int virtio_send_msg(struct fuse_session *se, struct fuse_chan *ch,
 {
     VuVirtqElement *elem;
     VuVirtq *q;
+    int ret = 0;
 
     assert(count >= 1);
     assert(iov[0].iov_len >= sizeof(struct fuse_out_header));
@@ -190,6 +192,7 @@ int virtio_send_msg(struct fuse_session *se, struct fuse_chan *ch,
     assert(out->unique);
     /* For virtio we always have ch */
     assert(ch);
+    assert(!ch->qi->reply_sent);
     elem = ch->qi->qe;
     q = &ch->qi->virtio_dev->dev.vq[ch->qi->qidx];
 
@@ -207,19 +210,23 @@ int virtio_send_msg(struct fuse_session *se, struct fuse_chan *ch,
     if (in_len < sizeof(struct fuse_out_header)) {
         fuse_log(FUSE_LOG_ERR, "%s: elem %d too short for out_header\n",
                  __func__, elem->index);
-        return -E2BIG;
+        ret = -E2BIG;
+        goto err;
     }
     if (in_len < tosend_len) {
         fuse_log(FUSE_LOG_ERR, "%s: elem %d too small for data len %zd\n",
                  __func__, elem->index, tosend_len);
-        return -E2BIG;
+        ret = -E2BIG;
+        goto err;
     }
 
     copy_iov(iov, count, in_sg, in_num, tosend_len);
     vu_queue_push(&se->virtio_dev->dev, q, elem, tosend_len);
     vu_queue_notify(&se->virtio_dev->dev, q);
+    ch->qi->reply_sent = true;
 
-    return 0;
+err:
+    return ret;
 }
 
 /* Thread function for individual queues, created when a queue is 'started' */
@@ -296,6 +303,9 @@ static void *fv_queue_thread(void *opaque)
                 break;
             }
 
+            qi->qe = elem;
+            qi->reply_sent = false;
+
             if (!fbuf.mem) {
                 fbuf.mem = malloc(se->bufsize);
                 assert(fbuf.mem);
@@ -331,6 +341,13 @@ static void *fv_queue_thread(void *opaque)
             /* TODO: Add checks for fuse_session_exited */
             fuse_session_process_buf_int(se, &fbuf, &ch);
 
+            if (!qi->reply_sent) {
+                fuse_log(FUSE_LOG_DEBUG, "%s: elem %d no reply sent\n",
+                         __func__, elem->index);
+                /* I think we've still got to recycle the element */
+                vu_queue_push(dev, q, elem, 0);
+                vu_queue_notify(dev, q);
+            }
             qi->qe = NULL;
             free(elem);
             elem = NULL;
Signature · 1 line
2.23.0
#37 lore raw

PATCH 24/30 virtiofsd: Add Makefile wiring for virtiofsd contrib

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Wire up the building of the virtiofsd in contrib.

virtiofsd relies on Linux-specific system calls and seccomp. Anyone wishing to port it to other host operating systems should do so carefully and without reducing security.

Only allow building on Linux hosts.

---
 Makefile                        |  8 ++++++++
 Makefile.objs                   |  1 +
 contrib/virtiofsd/Makefile.objs | 10 ++++++++++
 3 files changed, 19 insertions(+)
 create mode 100644 contrib/virtiofsd/Makefile.objs
Makefile +8 -0
index d20e7ffce3..694a871384 100644
@@ -316,6 +316,8 @@ HELPERS-y =
 HELPERS-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_LINUX)) = qemu-bridge-helper$(EXESUF)
 
 ifdef CONFIG_LINUX
+HELPERS-y += virtiofsd$(EXESUF)
+
 ifdef CONFIG_VIRGL
 ifdef CONFIG_GBM
 HELPERS-y += vhost-user-gpu$(EXESUF)
@@ -423,6 +425,7 @@ dummy := $(call unnest-vars,, \
                 elf2dmp-obj-y \
                 ivshmem-client-obj-y \
                 ivshmem-server-obj-y \
+                virtiofsd-obj-y \
                 rdmacm-mux-obj-y \
                 libvhost-user-obj-y \
                 vhost-user-scsi-obj-y \
@@ -658,6 +661,11 @@ rdmacm-mux$(EXESUF): LIBS += "-libumad"
 rdmacm-mux$(EXESUF): $(rdmacm-mux-obj-y) $(COMMON_LDADDS)
 	$(call LINK, $^)
 
+ifdef CONFIG_LINUX # relies on Linux-specific syscalls
+virtiofsd$(EXESUF): $(virtiofsd-obj-y) libvhost-user.a $(COMMON_LDADDS)
+	$(call LINK, $^)
+endif
+
 vhost-user-gpu$(EXESUF): $(vhost-user-gpu-obj-y) $(libvhost-user-obj-y) libqemuutil.a libqemustub.a
 	$(call LINK, $^)
 
Makefile.objs +1 -0
index abcbd89654..24d0b56f1f 100644
@@ -125,6 +125,7 @@ vhost-user-blk-obj-y = contrib/vhost-user-blk/
 rdmacm-mux-obj-y = contrib/rdmacm-mux/
 vhost-user-input-obj-y = contrib/vhost-user-input/
 vhost-user-gpu-obj-y = contrib/vhost-user-gpu/
+virtiofsd-obj-y = contrib/virtiofsd/
 
 ######################################################################
 trace-events-subdirs =
contrib/virtiofsd/Makefile.objs new file +10 -0
new file mode 100644
index 0000000000..67be16332c
@@ -0,0 +1,10 @@
+virtiofsd-obj-y = buffer.o \
+                  fuse_opt.o \
+                  fuse_log.o \
+                  fuse_loop_mt.o \
+                  fuse_lowlevel.o \
+                  fuse_signals.o \
+                  fuse_virtio.o \
+                  helper.o \
+                  passthrough_ll.o
+
Signature · 1 line
2.23.0
#38 lore raw

PATCH 25/30 virtiofsd: Fast path for virtio read

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Readv the data straight into the guests buffer.

With fix by:

---
 contrib/virtiofsd/fuse_lowlevel.c |   6 ++
 contrib/virtiofsd/fuse_virtio.c   | 159 ++++++++++++++++++++++++++++++
 contrib/virtiofsd/fuse_virtio.h   |   4 +
 3 files changed, 169 insertions(+)
contrib/virtiofsd/fuse_lowlevel.c +6 -0
index fc7707b330..b1f44ca17e 100644
@@ -471,6 +471,12 @@ static int fuse_send_data_iov_fallback(struct fuse_session *se,
 		return fuse_send_msg(se, ch, iov, iov_count);
 	}
 
+	if (fuse_lowlevel_is_virtio(se) && buf->count==1 &&
+	    buf->buf[0].flags == (FUSE_BUF_IS_FD | FUSE_BUF_FD_SEEK)) {
+		return virtio_send_data_iov(se, ch, iov, iov_count,
+					    buf, len);
+	}
+
 	abort(); /* Will have taken vhost path */
 	return 0;
 }
contrib/virtiofsd/fuse_virtio.c +159 -0

165 lines not on this page · raw message

contrib/virtiofsd/fuse_virtio.h +4 -0
index f1e7391e44..19c468ee31 100644
@@ -26,4 +26,8 @@ int virtio_loop(struct fuse_session *se);
 int virtio_send_msg(struct fuse_session *se, struct fuse_chan *ch,
                     struct iovec *iov, int count);
 
+int virtio_send_data_iov(struct fuse_session *se, struct fuse_chan *ch,
+                         struct iovec *iov, int count,
+                         struct fuse_bufvec *buf, size_t len);
+
 #endif
Signature · 1 line
2.23.0
#39 lore raw

PATCH 26/30 virtiofsd: add --fd=FDNUM fd passing option

In reply to Dr. David Alan Gilbert (git)

From: Stefan Hajnoczi <[email protected]>

Although --socket-path=PATH is useful for manual invocations, management tools typically create the UNIX domain socket themselves and pass it to the vhost-user device backend. This way QEMU can be launched immediately with a valid socket. No waiting for the vhost-user device backend is required when fd passing is used.

---
 contrib/virtiofsd/fuse_i.h        |  1 +
 contrib/virtiofsd/fuse_lowlevel.c | 13 ++++++++---
 contrib/virtiofsd/fuse_virtio.c   | 39 +++++++++++++++++++++++--------
 3 files changed, 40 insertions(+), 13 deletions(-)
contrib/virtiofsd/fuse_i.h +1 -0
index 5b6ef09df5..4b718f1aec 100644
@@ -68,6 +68,7 @@ struct fuse_session {
 	size_t bufsize;
 	int error;
 	char *vu_socket_path;
+	int   vu_listen_fd;
 	int   vu_socketfd;
 	struct fv_VuDev *virtio_dev;
 };
contrib/virtiofsd/fuse_lowlevel.c +10 -3
index b1f44ca17e..da2f151927 100644
@@ -2077,6 +2077,7 @@ static const struct fuse_opt fuse_ll_opts[] = {
 	LL_OPTION("allow_root", deny_others, 1),
 	LL_OPTION("--socket-path=%s", vu_socket_path, 0),
 	LL_OPTION("vhost_user_socket=%s", vu_socket_path, 0),
+	LL_OPTION("--fd=%d", vu_listen_fd, 0),
 	FUSE_OPT_END
 };
 
@@ -2095,6 +2096,7 @@ void fuse_lowlevel_help(void)
 "    -o allow_root              allow access by root\n"
 "    --socket-path=PATH         path for the vhost-user socket\n"
 "    -o vhost_user_socket=PATH  path for the vhost-user socket\n"
+"    --fd=FDNUM                 fd number of vhost-user socket\n"
 "    -o auto_unmount            auto unmount on process termination\n");
 }
 
@@ -2134,6 +2136,7 @@ struct fuse_session *fuse_session_new(struct fuse_args *args,
 		goto out1;
 	}
 	se->fd = -1;
+	se->vu_listen_fd = -1;
 	se->conn.max_write = UINT_MAX;
 	se->conn.max_readahead = UINT_MAX;
 
@@ -2163,8 +2166,12 @@ struct fuse_session *fuse_session_new(struct fuse_args *args,
 		goto out4;
 	}
 
-	if (!se->vu_socket_path) {
-		fprintf(stderr, "fuse: missing -o vhost_user_socket option\n");
+	if (!se->vu_socket_path && se->vu_listen_fd < 0) {
+		fuse_log(FUSE_LOG_ERR, "fuse: missing --socket-path or --fd option\n");
+		goto out4;
+	}
+	if (se->vu_socket_path && se->vu_listen_fd >= 0) {
+		fuse_log(FUSE_LOG_ERR, "fuse: --socket-path and --fd cannot be given together\n");
 		goto out4;
 	}
 
@@ -2205,7 +2212,7 @@ void fuse_session_unmount(struct fuse_session *se)
 
 int fuse_lowlevel_is_virtio(struct fuse_session *se)
 {
-	return se->vu_socket_path != NULL;
+	return !!se->virtio_dev;
 }
 
 #ifdef linux
contrib/virtiofsd/fuse_virtio.c +29 -10
index f52d815596..cc393c3cac 100644
@@ -635,21 +635,20 @@ int virtio_loop(struct fuse_session *se)
     return 0;
 }
 
-int virtio_session_mount(struct fuse_session *se)
+static int fv_create_listen_socket(struct fuse_session *se)
 {
     struct sockaddr_un un;
 
+    /* Nothing to do if fd is already initialized */
+    if (se->vu_listen_fd >= 0) {
+        return 0;
+    }
+
     if (strlen(se->vu_socket_path) >= sizeof(un.sun_path)) {
         fuse_log(FUSE_LOG_ERR, "Socket path too long\n");
         return -1;
     }
 
-    /*
-     * Poison the fuse FD so we spot if we accidentally use it;
-     * DO NOT check for this value, check for fuse_lowlevel_is_virtio()
-     */
-    se->fd = 0xdaff0d11;
-
     /*
      * Create the Unix socket to communicate with qemu
      * based on QEMU's vhost-user-bridge
@@ -675,15 +674,35 @@ int virtio_session_mount(struct fuse_session *se)
         return -1;
     }
 
+    se->vu_listen_fd = listen_sock;
+    return 0;
+}
+
+int virtio_session_mount(struct fuse_session *se)
+{
+    int ret;
+
+    ret = fv_create_listen_socket(se);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /*
+     * Poison the fuse FD so we spot if we accidentally use it;
+     * DO NOT check for this value, check fuse_lowlevel_is_virtio()
+     */
+    se->fd = 0xdaff0d11;
+
     fuse_log(FUSE_LOG_INFO, "%s: Waiting for vhost-user socket connection...\n",
              __func__);
-    int data_sock = accept(listen_sock, NULL, NULL);
+    int data_sock = accept(se->vu_listen_fd, NULL, NULL);
     if (data_sock == -1) {
         fuse_log(FUSE_LOG_ERR, "vhost socket accept: %m\n");
-        close(listen_sock);
+        close(se->vu_listen_fd);
         return -1;
     }
-    close(listen_sock);
+    close(se->vu_listen_fd);
+    se->vu_listen_fd = -1;
     fuse_log(FUSE_LOG_INFO, "%s: Received vhost-user socket connection\n",
              __func__);
     se->vu_socketfd = data_sock;
Signature · 1 line
2.23.0
#40 lore raw

PATCH 27/30 virtiofsd: make -f (foreground) the default

In reply to Dr. David Alan Gilbert (git)

From: Stefan Hajnoczi <[email protected]>

According to vhost-user.rst "Backend program conventions", backend programs should run in the foregound by default. Follow the conventions so libvirt and other management tools can control virtiofsd in a standard way.

---
 contrib/virtiofsd/helper.c | 6 ++++++
 1 file changed, 6 insertions(+)
contrib/virtiofsd/helper.c +6 -0
index f67533f053..ab20810c28 100644
@@ -26,6 +26,9 @@
 
 #define FUSE_HELPER_OPT(t, p) \
 	{ t, offsetof(struct fuse_cmdline_opts, p), 1 }
+#define FUSE_HELPER_OPT_VALUE(t, p, v) \
+	{ t, offsetof(struct fuse_cmdline_opts, p), v }
+
 
 static const struct fuse_opt fuse_helper_opts[] = {
 	FUSE_HELPER_OPT("-h",		show_help),
@@ -39,6 +42,7 @@ static const struct fuse_opt fuse_helper_opts[] = {
 	FUSE_OPT_KEY("-d",		FUSE_OPT_KEY_KEEP),
 	FUSE_OPT_KEY("debug",		FUSE_OPT_KEY_KEEP),
 	FUSE_HELPER_OPT("-f",		foreground),
+	FUSE_HELPER_OPT_VALUE("--daemonize", foreground, 0),
 	FUSE_HELPER_OPT("-s",		singlethread),
 	FUSE_HELPER_OPT("fsname=",	nodefault_subtype),
 	FUSE_OPT_KEY("fsname=",		FUSE_OPT_KEY_KEEP),
@@ -130,6 +134,7 @@ void fuse_cmdline_help(void)
 	       "    -V   --version             print version\n"
 	       "    -d   -o debug              enable debug output (implies -f)\n"
 	       "    -f                         foreground operation\n"
+	       "    --daemonize                run in background\n"
 	       "    -s                         disable multi-threaded operation\n"
 	       "    -o clone_fd                use separate fuse device fd for each thread\n"
 	       "                               (may improve performance)\n"
@@ -160,6 +165,7 @@ int fuse_parse_cmdline(struct fuse_args *args,
 	memset(opts, 0, sizeof(struct fuse_cmdline_opts));
 
 	opts->max_idle_threads = 10;
+	opts->foreground = 1;
 
 	if (fuse_opt_parse(args, opts, fuse_helper_opts,
 			   fuse_helper_opt_proc) == -1)
Signature · 1 line
2.23.0
#41 lore raw

PATCH 28/30 virtiofsd: add vhost-user.json file

In reply to Dr. David Alan Gilbert (git)

From: Stefan Hajnoczi <[email protected]>

Install a vhost-user.json file describing virtiofsd. This allows libvirt and other management tools to enumerate vhost-user backend programs.

---
 .gitignore                                  | 1 +
 Makefile                                    | 1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in | 5 +++++
 3 files changed, 7 insertions(+)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
.gitignore +1 -0
index 7de868d1ea..e9f621be06 100644
@@ -6,6 +6,7 @@
 /config-target.*
 /config.status
 /config-temp
+/contrib/virtiofsd/50-qemu-virtiofsd.json
 /elf2dmp
 /trace-events-all
 /trace/generated-events.h
Makefile +1 -0
index 694a871384..20531a32ae 100644
@@ -317,6 +317,7 @@ HELPERS-$(call land,$(CONFIG_SOFTMMU),$(CONFIG_LINUX)) = qemu-bridge-helper$(EXE
 
 ifdef CONFIG_LINUX
 HELPERS-y += virtiofsd$(EXESUF)
+vhost-user-json-y += contrib/virtiofsd/50-qemu-virtiofsd.json
 
 ifdef CONFIG_VIRGL
 ifdef CONFIG_GBM
contrib/virtiofsd/50-qemu-virtiofsd.json.in new file +5 -0
new file mode 100644
index 0000000000..9bcd86f8dc
@@ -0,0 +1,5 @@
+{
+  "description": "QEMU virtiofsd vhost-user-fs",
+  "type": "fs",
+  "binary": "@libexecdir@/virtiofsd"
+}
Signature · 1 line
2.23.0
#42 lore raw

PATCH 29/30 virtiofsd: add --print-capabilities option

In reply to Dr. David Alan Gilbert (git)

From: Stefan Hajnoczi <[email protected]>

Add the --print-capabilities option as per vhost-user.rst "Backend programs conventions". Currently there are no advertised features.

---
 contrib/virtiofsd/fuse_lowlevel.h  |  1 +
 contrib/virtiofsd/helper.c         |  2 ++
 contrib/virtiofsd/passthrough_ll.c | 12 ++++++++++++
 docs/interop/vhost-user.json       |  4 +++-
 4 files changed, 18 insertions(+), 1 deletion(-)
contrib/virtiofsd/fuse_lowlevel.h +1 -0
index b06ab6915a..ab964a6fb3 100644
@@ -1796,6 +1796,7 @@ struct fuse_cmdline_opts {
 	int nodefault_subtype;
 	int show_version;
 	int show_help;
+	int print_capabilities;
 	int clone_fd;
 	unsigned int max_idle_threads;
 };
contrib/virtiofsd/helper.c +2 -0
index ab20810c28..08e00c0d13 100644
@@ -35,6 +35,7 @@ static const struct fuse_opt fuse_helper_opts[] = {
 	FUSE_HELPER_OPT("--help",	show_help),
 	FUSE_HELPER_OPT("-V",		show_version),
 	FUSE_HELPER_OPT("--version",	show_version),
+	FUSE_HELPER_OPT("--print-capabilities", print_capabilities),
 	FUSE_HELPER_OPT("-d",		debug),
 	FUSE_HELPER_OPT("debug",	debug),
 	FUSE_HELPER_OPT("-d",		foreground),
@@ -132,6 +133,7 @@ void fuse_cmdline_help(void)
 {
 	printf("    -h   --help                print help\n"
 	       "    -V   --version             print version\n"
+	       "    --print-capabilities       print vhost-user.json\n"
 	       "    -d   -o debug              enable debug output (implies -f)\n"
 	       "    -f                         foreground operation\n"
 	       "    --daemonize                run in background\n"
contrib/virtiofsd/passthrough_ll.c +12 -0
index c09fc4a65c..2a1a880224 100644
@@ -1216,6 +1216,14 @@ static struct fuse_lowlevel_ops lo_oper = {
 #endif
 };
 
+/* Print vhost-user.json backend program capabilities */
+static void print_capabilities(void)
+{
+	printf("{\n");
+	printf("  \"type\": \"fs\"\n");
+	printf("}\n");
+}
+
 int main(int argc, char *argv[])
 {
 	struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
@@ -1246,6 +1254,10 @@ int main(int argc, char *argv[])
 		fuse_lowlevel_version();
 		ret = 0;
 		goto err_out1;
+	} else if (opts.print_capabilities) {
+		print_capabilities();
+		ret = 0;
+		goto err_out1;
 	}
 
 	if (fuse_opt_parse(&args, &lo, lo_opts, NULL)== -1)
docs/interop/vhost-user.json +3 -1
index da6aaf51c8..d4ea1f7ac5 100644
@@ -31,6 +31,7 @@
 # @rproc-serial: virtio remoteproc serial link
 # @scsi: virtio scsi
 # @vsock: virtio vsock transport
+# @fs: virtio fs (since 4.2)
 #
 # Since: 4.0
 ##
@@ -50,7 +51,8 @@
       'rpmsg',
       'rproc-serial',
       'scsi',
-      'vsock'
+      'vsock',
+      'fs'
   ]
 }
 
Signature · 1 line
2.23.0
#43 lore raw

PATCH 30/30 virtiofs: Add maintainers entry

In reply to Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

---
 MAINTAINERS | 8 ++++++++
 1 file changed, 8 insertions(+)
MAINTAINERS +8 -0
index fe4dc51b08..1e33932db3 100644
@@ -1543,6 +1543,14 @@ T: git https://github.com/cohuck/qemu.git s390-next
 T: git https://github.com/borntraeger/qemu.git s390-next
 L: [email protected]
 
+virtiofs
+M: Dr. David Alan Gilbert <[email protected]>
+M: Stefan Hajnoczi <[email protected]>
+S: Supported
+F: contrib/virtiofsd/*
+F: hw/virtio/vhost-user-fs*
+F: include/hw/virtio/vhost-user-fs.h
+
 virtio-input
 M: Gerd Hoffmann <[email protected]>
 S: Maintained
Signature · 1 line
2.23.0

A message that is not in the archive (sent off-list or not yet received): 157166664425.24734.3489596262271351160@37313f22b938

#45 lore raw
Quoted 13 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

Quoted 7 lines

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#46 lore raw
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 13 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

Quoted 7 lines

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

Quoted 2 lines

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#47 lore raw
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

The source files though, we appear to then be modifying locally, which suggests they'll live in our repo forever. In this case I'd expect to have compliance with QEMU coding standards.

I'm surprised we need to copy so much in from fuse though. Is there a case to be made for fuse to provide a library of APIs for people who are building fuse daemons to link against, instead of copy & fork ?

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#48 lore raw
Quoted 36 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

Quoted 8 lines

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

The headers are really two types; one are external definitions, the other are internal parts of libfuse. I'd expect to keep the internal parts long term; teh external parts hmm; where would we pull them in externally from?

Quoted 3 lines

The source files though, we appear to then be modifying locally, which suggests they'll live in our repo forever. In this case I'd expect to have compliance with QEMU coding standards.

OK.

Quoted 3 lines

I'm surprised we need to copy so much in from fuse though. Is there a case to be made for fuse to provide a library of APIs for people who are building fuse daemons to link against, instead of copy & fork ?

libfuse *is* such a library; but it preserves ABI compliance; it's intention is to be used to build filesystem implementations on top of - and that's got a fairly good separation; however changing the fuse transport, and security models is much more invasive than it was designed for.

Dave

Quoted 6 lines

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#49 lore raw
Quoted 61 lines · Dr. David Alan Gilbert
Quoted 36 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

Quoted 8 lines

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

The headers are really two types; one are external definitions, the other are internal parts of libfuse. I'd expect to keep the internal parts long term; teh external parts hmm; where would we pull them in externally from?

Quoted 3 lines

The source files though, we appear to then be modifying locally, which suggests they'll live in our repo forever. In this case I'd expect to have compliance with QEMU coding standards.

OK.

Quoted 3 lines

I'm surprised we need to copy so much in from fuse though. Is there a case to be made for fuse to provide a library of APIs for people who are building fuse daemons to link against, instead of copy & fork ?

Quoted 7 lines

libfuse *is* such a library; but it preserves ABI compliance; it's intention is to be used to build filesystem implementations on top of - and that's got a fairly good separation; however changing the fuse transport, and security models is much more invasive than it was designed for.

Dave

I guess you did try to propose adding the functionality to the libfuse maintainer and got rejected? If not it's worth asking.

Quoted 8 lines
Quoted 6 lines

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#50 lore raw
Quoted 68 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 67 lines · Dr. David Alan Gilbert
Quoted 36 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

Quoted 8 lines

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

The headers are really two types; one are external definitions, the other are internal parts of libfuse. I'd expect to keep the internal parts long term; teh external parts hmm; where would we pull them in externally from?

Quoted 3 lines

The source files though, we appear to then be modifying locally, which suggests they'll live in our repo forever. In this case I'd expect to have compliance with QEMU coding standards.

OK.

Quoted 3 lines

I'm surprised we need to copy so much in from fuse though. Is there a case to be made for fuse to provide a library of APIs for people who are building fuse daemons to link against, instead of copy & fork ?

libfuse *is* such a library; but it preserves ABI compliance; it's intention is to be used to build filesystem implementations on top of - and that's got a fairly good separation; however changing the fuse transport, and security models is much more invasive than it was designed for.

Quoted 5 lines
Quoted 1 line

Dave

I guess you did try to propose adding the functionality to the libfuse maintainer and got rejected? If not it's worth asking.

I looked at it; we started off thinking we'd merge much of it up there - but there's at least two problems:

a) Our libvhost-user isn't a separable library either - so we'd need
to copy it into libfuse which is just as big a problem.
b) It's not additional interfaces, it's changes to existing APIs that
break A*B*I compatibility that are very hard to avoid when you're changing the transport.

Also, we're not just changing the transport - if you look at the security patchset we're inverting the security model and there's a bunch of threading changes that you haven't got yet.

Dave

Quoted 10 lines
Quoted 10 lines
Quoted 8 lines

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#51 lore raw
In reply to Dr. David Alan Gilbert
Quoted 46 lines · Dr. David Alan Gilbert
Quoted 36 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

Quoted 8 lines

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

Quoted 4 lines

The headers are really two types; one are external definitions, the other are internal parts of libfuse. I'd expect to keep the internal parts long term; teh external parts hmm; where would we pull them in externally from?

The fuse3-devel RPM on Fedora has some, but not all, of the fuse headers the patches copy in. Not sure if that's enough though.

The kernel-devel RPM has fuse.h which seems to match fuse_kernel.h header being imported.

Obviously that would mean a configure check to see if the required headers exist or not & are new enough for VSOCK

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#52 lore raw
Quoted 47 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 46 lines · Dr. David Alan Gilbert
Quoted 44 lines · * Daniel P. Berrangé ([email protected]) wrote:
Quoted 23 lines · Michael S. Tsirkin
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

IIUC, most of this code is simpy copied as-is from the fuse or linux git repos. I'm wondering what the intention is for it long term ?

For header files, I would have expected us to be able to compile against the -devel package provided by the kernel or fuse packages. I can understand if we want to import the headers if the VSOCK additions to them are not yet widely available in distros though. If this is the case we should put a time limit on how long we'd keep these copied headers around for before dropping them. It would be fine to violate QEMU coding style in this case as its not code QEMU would "maintain" long term - just a read-only import.

Quoted 7 lines
Quoted 4 lines

The headers are really two types; one are external definitions, the other are internal parts of libfuse. I'd expect to keep the internal parts long term; teh external parts hmm; where would we pull them in externally from?

The fuse3-devel RPM on Fedora has some, but not all, of the fuse headers the patches copy in. Not sure if that's enough though.

No, that's the public API, not the internals which we're poking at.

Quoted 2 lines

The kernel-devel RPM has fuse.h which seems to match fuse_kernel.h header being imported.

We do require that we have one that's new enough, so in future it might be possible; but we're currently using very new parts of fuse. (It's also not obvious whether the kernel or libfuse owns the 'newest' version of that header; we think it's probably the kernel).

Quoted 2 lines

Obviously that would mean a configure check to see if the required headers exist or not & are new enough for VSOCK

Dave

Quoted 7 lines

Regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#53 lore raw
In reply to Michael S. Tsirkin
Quoted 23 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 22 lines · Dr. David Alan Gilbert
Quoted 20 lines · * [email protected] ([email protected]) wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series seems to have some coding style problems. See output below for more information:

Subject: [PATCH 00/30] virtiofs daemon (base)
Type: series
Message-id: [email protected]

=== TEST SCRIPT BEGIN === #!/bin/bash
git rev-parse base > /dev/null || exit 0 git config --local diff.renamelimit 0 git config --local diff.renames True git config --local diff.algorithm histogram ./scripts/checkpatch.pl --mailback base..

Quoted 8 lines
Quoted 4 lines

Expecting checkpatch to be broken here; most of the files follow FUSE's formatting.

Dave

I wonder what do others think about this.
One problem with such inconsistencies is that people tend to copy code around, which tends to result in a mess.

Converting them is not entirely trivial; so shout now if you'd like it! I've got an evil combination of indent and clang-tidy that does a reasonablish job of getting close to qemu's coding standard - but it does need some hand holding; so is a few days work.
(I applied it to fuse_virtio.c which had a mix of styles).

Dave

Quoted 2 lines
Quoted 2 lines

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#54 lore raw
In reply to Dr. David Alan Gilbert (git)
Quoted 23 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

Quoted 8 lines

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

Thinking of future updates, have you given any thought to automating them? Something along the lines of update-linux-headers maybe?

Quoted 105 lines

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Signature · 1 line
2.23.0
#55 lore raw
Quoted 29 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 28 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

Quoted 7 lines
Quoted 3 lines

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

Thinking of future updates, have you given any thought to automating them? Something along the lines of update-linux-headers maybe?

The problem is we're changing the code in some of them;
libfuse changes slowly enough that rebasing on a new libfuse isn't too bad; but once committed, just copying new versions in like update-linux-headers does isn't going to work because we have to merge in our changes. I expect to have to cherry-pick changes. Libfuse has had 75 patches in the last year; so it doesn't worry me too much.

Dave

Quoted 104 lines
Quoted 104 lines

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Quoted 2 lines
Quoted 2 lines
Signature · 1 line
2.23.0

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#56 lore raw
In reply to Dr. David Alan Gilbert (git)
Quoted 6 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

Quoted 6 lines

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

So based on this I'm guessing we don't want to rush this in before the soft freeze.
If you have a different opinion, let me know pls.

Signature · 1 line
MST
#57 lore raw
Quoted 11 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 10 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

Quoted 6 lines
Quoted 2 lines

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

So based on this I'm guessing we don't want to rush this in before the soft freeze.
If you have a different opinion, let me know pls.

With the 2nd set that I posted on thursday, the '(security)' set it becomes safe - with full sandboxing and path verification. Now, we do have a pile of separate small fixes, and threading code stuff; but we cna leave those off.

Dave

Quoted 2 lines
Signature · 1 line
MST

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#58 lore raw
In reply to Dr. David Alan Gilbert (git)
Quoted 9 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

So I went back and forth on this but this is huge
and there's not a lot of time for review.
So I parked it + the security patches on a next branch in my tree. I will rebase once after rc1 is out, and then stop.

Quoted 127 lines

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Signature · 1 line
2.23.0
#59 lore raw
Quoted 12 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 9 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

Quoted 4 lines

So I went back and forth on this but this is huge
and there's not a lot of time for review.
So I parked it + the security patches on a next branch in my tree. I will rebase once after rc1 is out, and then stop.

Thanks; I'll work on the extra sets that can go later (the threading and cleanups+fixes).

Dave

Quoted 126 lines
Quoted 126 lines

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Quoted 2 lines
Quoted 2 lines
Signature · 1 line
2.23.0

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#60 lore raw
Quoted 18 lines · Dr. David Alan Gilbert
Quoted 12 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 9 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

Quoted 4 lines

So I went back and forth on this but this is huge
and there's not a lot of time for review.
So I parked it + the security patches on a next branch in my tree. I will rebase once after rc1 is out, and then stop.

Quoted 4 lines

Thanks; I'll work on the extra sets that can go later (the threading and cleanups+fixes).

Dave

Apropos I would really like to figure out
a better way to know that we did not miss
anything when adding the security patchset.

Quoted 130 lines
Quoted 126 lines
Quoted 126 lines

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Quoted 2 lines
Quoted 2 lines
Signature · 1 line
2.23.0

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

#61 lore raw
Quoted 22 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 21 lines · Dr. David Alan Gilbert
Quoted 12 lines · * Michael S. Tsirkin ([email protected]) wrote:
Quoted 9 lines · Dr. David Alan Gilbert (git)

From: "Dr. David Alan Gilbert" <[email protected]>

Hi,
This is the 1st set for the virtiofsd - a daemon
that implements the user space side of virtiofs.

The kernel and qemu device parts recently went in,
so the daemon is the only thing missing to have a working set.

Quoted 4 lines

So I went back and forth on this but this is huge
and there's not a lot of time for review.
So I parked it + the security patches on a next branch in my tree. I will rebase once after rc1 is out, and then stop.

Thanks; I'll work on the extra sets that can go later (the threading and cleanups+fixes).

Quoted 6 lines
Quoted 1 line

Dave

Apropos I would really like to figure out
a better way to know that we did not miss
anything when adding the security patchset.

Yep, it's worth at least having a lot of eyes on it, and better finding some more automation for the path checking.

Dave

Quoted 127 lines
Quoted 127 lines
Quoted 127 lines
Quoted 127 lines

This set is the absolute minimal base set of patches;
it's not yet safe to use (from security or correctness);

I'll follow up with ~3 more series in the next few days with:

a) Security patches that add sandboxing and checking compared with normal fuse - that makes it safe.
b) Performance improvements including threading
c) Other fixes, including correctness.

but, this is a good start and gets things rolling.

The set pulls in a big chunk of the upstream libfuse library (unmodified so that it's easy to check it really is upstream), chops all the stuff out we don't need and then adds the new transport we need.

For new files I've formatted the code according to qemu standards; for files that are from upstream libfuse I've kept with their standards for ease of future updating.

We can't just link with libfuse, since we have to make ABI incompatible changes for the new transport.

Running this daemon is typically done with:

./virtiofsd -o vhost_user_socket=/path/socket -o source=/path/to/fs

connected to a qemu that's then started with:
-chardev socket,id=char0,path=/path/socket -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs

and then in the guest mount with:
mount -t virtiofs myfs /mnt

Our development branch is: https://gitlab.com/virtio-fs/qemu/tree/virtio-fs-dev

Dave

Dr. David Alan Gilbert (22):
virtiofsd: Pull in upstream headers
virtiofsd: Pull in kernel's fuse.h
virtiofsd: Add auxiliary .c's
virtiofsd: Add fuse_lowlevel.c
virtiofsd: Add passthrough_ll
virtiofsd: Trim down imported files
virtiofsd: Fix fuse_daemonize ignored return values
virtiofsd: Fix common header and define for QEMU builds
virtiofsd: fuse: Make iov_length usable outside fuse_lowlevel.c
virtiofsd: Add options for virtio
virtiofsd: Open vhost connection instead of mounting
virtiofsd: Start wiring up vhost-user
virtiofsd: Add main virtio loop
virtiofsd: get/set features callbacks
virtiofsd: Start queue threads
virtiofsd: Poll kick_fd for queue
virtiofsd: Start reading commands from queue
virtiofsd: Send replies to messages
virtiofsd: Keep track of replies
virtiofsd: Add Makefile wiring for virtiofsd contrib
virtiofsd: Fast path for virtio read
virtiofs: Add maintainers entry

Stefan Hajnoczi (7):
virtiofsd: remove mountpoint dummy argument
virtiofsd: remove unused notify reply support
virtiofsd: add -o source=PATH to help output
virtiofsd: add --fd=FDNUM fd passing option
virtiofsd: make -f (foreground) the default
virtiofsd: add vhost-user.json file
virtiofsd: add --print-capabilities option

Vivek Goyal (1):
virtiofsd: Make fsync work even if only inode is passed in

 .gitignore                                  |    1 +
 MAINTAINERS                                 |    8 +
 Makefile                                    |    9 +
 Makefile.objs                               |    1 +
 contrib/virtiofsd/50-qemu-virtiofsd.json.in |    5 +
 contrib/virtiofsd/Makefile.objs             |   10 +
 contrib/virtiofsd/buffer.c                  |  318 +++
 contrib/virtiofsd/fuse.h                    | 1268 ++++++++++
 contrib/virtiofsd/fuse_common.h             |  823 +++++++
 contrib/virtiofsd/fuse_i.h                  |  131 ++
 contrib/virtiofsd/fuse_kernel.h             |  858 +++++++
 contrib/virtiofsd/fuse_log.c                |   40 +
 contrib/virtiofsd/fuse_log.h                |   82 +
 contrib/virtiofsd/fuse_loop_mt.c            |   54 +
 contrib/virtiofsd/fuse_lowlevel.c           | 2302 +++++++++++++++++++
 contrib/virtiofsd/fuse_lowlevel.h           | 2024 ++++++++++++++++
 contrib/virtiofsd/fuse_misc.h               |   59 +
 contrib/virtiofsd/fuse_opt.c                |  422 ++++
 contrib/virtiofsd/fuse_opt.h                |  271 +++
 contrib/virtiofsd/fuse_signals.c            |   90 +
 contrib/virtiofsd/fuse_virtio.c             |  717 ++++++
 contrib/virtiofsd/fuse_virtio.h             |   33 +
 contrib/virtiofsd/helper.c                  |  300 +++
 contrib/virtiofsd/passthrough_helpers.h     |   76 +
 contrib/virtiofsd/passthrough_ll.c          | 1341 +++++++++++
 docs/interop/vhost-user.json                |    4 +-
 26 files changed, 11246 insertions(+), 1 deletion(-)
 create mode 100644 contrib/virtiofsd/50-qemu-virtiofsd.json.in
 create mode 100644 contrib/virtiofsd/Makefile.objs
 create mode 100644 contrib/virtiofsd/buffer.c
 create mode 100644 contrib/virtiofsd/fuse.h
 create mode 100644 contrib/virtiofsd/fuse_common.h
 create mode 100644 contrib/virtiofsd/fuse_i.h
 create mode 100644 contrib/virtiofsd/fuse_kernel.h
 create mode 100644 contrib/virtiofsd/fuse_log.c
 create mode 100644 contrib/virtiofsd/fuse_log.h
 create mode 100644 contrib/virtiofsd/fuse_loop_mt.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.c
 create mode 100644 contrib/virtiofsd/fuse_lowlevel.h
 create mode 100644 contrib/virtiofsd/fuse_misc.h
 create mode 100644 contrib/virtiofsd/fuse_opt.c
 create mode 100644 contrib/virtiofsd/fuse_opt.h
 create mode 100644 contrib/virtiofsd/fuse_signals.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.c
 create mode 100644 contrib/virtiofsd/fuse_virtio.h
 create mode 100644 contrib/virtiofsd/helper.c
 create mode 100644 contrib/virtiofsd/passthrough_helpers.h
 create mode 100644 contrib/virtiofsd/passthrough_ll.c
Quoted 4 lines
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Signature · 1 line
2.23.0

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK

--
Dr. David Alan Gilbert / [email protected] / Manchester, UK