vhost-user block device backend implementation
v7 cover vhost-user block device backend implementation
v7
- fix docker-test-debug@fedora errors by freeing malloced memory
v6
- add missing license header and include guard
- vhost-user server only serve one client one time
- fix a bug in custom vu_message_read
- using qemu-storage-daemon to start vhost-user-blk-server
- a bug fix to pass docker-test-clang@ubuntu
v5:
* re-use vu_kick_cb in libvhost-user
* keeping processing VhostUserMsg in the same coroutine until there is detachment/attachment of AIOContext
* Spawn separate coroutine for each VuVirtqElement
* Other changes including relocating vhost-user-blk-server.c, coding style etc.
v4:
* add object properties in class_init
* relocate vhost-user-blk-test
* other changes including using SocketAddress, coding style, etc.
v3:
* separate generic vhost-user-server code from vhost-user-blk-server code
* re-write vu_message_read and kick hander function as coroutines to directly call blk_co_preadv, blk_co_pwritev, etc.
* add aio_context notifier functions to support multi-threading model
* other fixes regarding coding style, warning report, etc.
v2:
* Only enable this feature for Linux because eventfd is a Linux-specific feature
This patch series is an implementation of vhost-user block device backend server, thanks to Stefan and Kevin's guidance.
Vhost-user block device backend server is a UserCreatable object and can be started using object_add,
(qemu) object_add vhost-user-blk-server,id=ID,unix-socket=/tmp/vhost-user-blk_vhost.socket,node-name=DRIVE_NAME,writable=off,blk-size=512 (qemu) object_del ID
or appending the "-object" option when starting QEMU,
$ -object vhost-user-blk-server,id=disk,unix-socket=/tmp/vhost-user-blk_vhost.socket,node-name=DRIVE_NAME,writable=off,blk-size=512
Then vhost-user client can connect to the server backend. For example, QEMU could act as a client,
$ -m 256 -object memory-backend-memfd,id=mem,size=256M,share=on -numa node,memdev=mem -chardev socket,id=char1,path=/tmp/vhost-user-blk_vhost.socket -device vhost-user-blk-pci,id=blk0,chardev=char1
And guest OS could access this vhost-user block device after mounting it.
Coiby Xu (5):
allow vu_message_read to be replaced generic vhost user server
vhost-user block device backend server new qTest case to test the vhost-user-blk-server
block/Makefile.objs | 1 + block/export/vhost-user-blk-server.c | 716 ++++++++++++++++++++ block/export/vhost-user-blk-server.h | 34 + contrib/libvhost-user/libvhost-user-glib.c | 2 +- contrib/libvhost-user/libvhost-user.c | 11 +- contrib/libvhost-user/libvhost-user.h | 21 + softmmu/vl.c | 4 + tests/Makefile.include | 3 +- tests/qtest/Makefile.include | 2 + tests/qtest/libqos/vhost-user-blk.c | 126 ++++ tests/qtest/libqos/vhost-user-blk.h | 44 ++ tests/qtest/libqtest.c | 44 +- tests/qtest/libqtest.h | 38 ++ tests/qtest/vhost-user-blk-test.c | 743 +++++++++++++++++++++ tests/vhost-user-bridge.c | 2 + tools/virtiofsd/fuse_virtio.c | 4 +- util/Makefile.objs | 1 + util/vhost-user-server.c | 406 +++++++++++ util/vhost-user-server.h | 59 ++ 19 files changed, 2221 insertions(+), 40 deletions(-) create mode 100644 block/export/vhost-user-blk-server.c create mode 100644 block/export/vhost-user-blk-server.h create mode 100644 tests/qtest/libqos/vhost-user-blk.c create mode 100644 tests/qtest/libqos/vhost-user-blk.h create mode 100644 tests/qtest/vhost-user-blk-test.c create mode 100644 util/vhost-user-server.c create mode 100644 util/vhost-user-server.h
--
2.26.2
PATCH v7 1/4 Allow vu_message_read to be replaced
Allow vu_message_read to be replaced by one which will make use of the QIOChannel functions. Thus reading vhost-user message won't stall the guest.
--- contrib/libvhost-user/libvhost-user-glib.c | 2 +- contrib/libvhost-user/libvhost-user.c | 11 ++++++----- contrib/libvhost-user/libvhost-user.h | 21 +++++++++++++++++++++ tests/vhost-user-bridge.c | 2 ++ tools/virtiofsd/fuse_virtio.c | 4 ++-- 5 files changed, 32 insertions(+), 8 deletions(-)
contrib/libvhost-user/libvhost-user-glib.c +1 -1
@@ -147,7 +147,7 @@ vug_init(VugDev *dev, uint16_t max_queues, int socket,g_assert (dev);g_assert (iface); -if (!vu_init (&dev->parent, max_queues, socket, panic, set_watch, +if (!vu_init (&dev->parent, max_queues, socket, panic,NULL , set_watch, remove_watch, iface)) {return false; }
contrib/libvhost-user/libvhost-user.c +6 -5
@@ -67,8 +67,6 @@/* The version of inflight buffer */ #define INFLIGHT_VERSION 1 -#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64) -/* The version of the protocol we support */ #define VHOST_USER_VERSION 1#define LIBVHOST_USER_DEBUG 0
@@ -412,7 +410,7 @@ vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg) goto out; } -if (!vu_message_read (dev, dev->slave_fd, &msg_reply)) { +if (!dev->read_msg (dev, dev->slave_fd, &msg_reply)) { goto out; }
@@ -647,7 +645,7 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)/* Wait for QEMU to confirm that it's registered the handler for the * faults. */ -if (!vu_message_read (dev, dev->sock, vmsg) || +if (!dev->read_msg (dev, dev->sock, vmsg) || vmsg->size !=sizeof (vmsg->payload.u64) || vmsg->payload.u64 !=0 ) {vu_panic (dev,"failed to receive valid ack for postcopy set-mem-table" );
@@ -1653,7 +1651,7 @@ vu_dispatch(VuDev *dev)int reply_requested;bool need_reply, success = false; -if (!vu_message_read (dev, dev->sock, &vmsg)) { +if (!dev->read_msg (dev, dev->sock, &vmsg)) { goto end; }
@@ -1704,6 +1702,7 @@ vu_deinit(VuDev *dev) }if (vq->kick_fd !=-1 ) { + dev->remove_watch (dev, vq->kick_fd);close (vq->kick_fd); vq->kick_fd =-1 ; }
@@ -1751,6 +1750,7 @@ vu_init(VuDev *dev,uint16_t max_queues, int socket, vu_panic_cb panic, + vu_read_msg_cb read_msg, vu_set_watch_cb set_watch, vu_remove_watch_cb remove_watch, const VuDevIface *iface)
@@ -1768,6 +1768,7 @@ vu_init(VuDev *dev, dev->sock = socket; dev->panic = panic; + dev->read_msg = read_msg ? read_msg : vu_message_read; dev->set_watch = set_watch; dev->remove_watch = remove_watch; dev->iface = iface;
contrib/libvhost-user/libvhost-user.h +21 -0
@@ -30,6 +30,8 @@#define VHOST_MEMORY_MAX_NREGIONS 8 +#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64) + typedef enum VhostSetConfigType { VHOST_SET_CONFIG_TYPE_MASTER = 0, VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
@@ -205,6 +207,7 @@ typedef uint64_t (*vu_get_features_cb) (VuDev *dev);typedef void (*vu_set_features_cb ) (VuDev *dev,uint64_t features);typedef int (*vu_process_msg_cb ) (VuDev *dev,VhostUserMsg *vmsg,int *do_reply); +typedef bool (*vu_read_msg_cb ) (VuDev *dev,int sock,VhostUserMsg *vmsg);typedef void (*vu_queue_set_started_cb ) (VuDev *dev,int qidx,bool started);typedef bool (*vu_queue_is_processed_in_order_cb ) (VuDev *dev,int qidx);typedef int (*vu_get_config_cb ) (VuDev *dev,uint8_t *config,uint32_t len);
@@ -373,6 +376,23 @@ struct VuDev {bool broken;uint16_t max_queues; +/* @read_msg: custom method to read vhost-user message +* +* Read data from vhost_user socket fd and fill up +* the passed VhostUserMsg *vmsg struct. +* +* If reading fails, it should close the received set of file +* descriptors as socket message's auxiliary data. +* +* For the details, please refer to vu_message_read in libvhost-user.c +* which will be used by default if not custom method is provided when +* calling vu_init +* +* Returns: true if vhost-user message successfully received, +* otherwise return false. +* +*/ +vu_read_msg_cb read_msg;/* @set_watch: add or update the given fd to the watch set, * call cb when condition is met */ vu_set_watch_cb set_watch;
@@ -416,6 +436,7 @@ bool vu_init(VuDev *dev,uint16_t max_queues, int socket, vu_panic_cb panic, + vu_read_msg_cb read_msg, vu_set_watch_cb set_watch, vu_remove_watch_cb remove_watch, const VuDevIface *iface);
tests/vhost-user-bridge.c +2 -0
@@ -520,6 +520,7 @@ vubr_accept_cb(int sock, void *ctx)VHOST_USER_BRIDGE_MAX_QUEUES , conn_fd, vubr_panic, +NULL , vubr_set_watch, vubr_remove_watch, &vuiface)) {
@@ -573,6 +574,7 @@ vubr_new(const char *path, bool client)VHOST_USER_BRIDGE_MAX_QUEUES , dev->sock, vubr_panic, +NULL , vubr_set_watch, vubr_remove_watch, &vuiface)) {
tools/virtiofsd/fuse_virtio.c +2 -2
@@ -980,8 +980,8 @@ int virtio_session_mount(struct fuse_session *se) se->vu_socketfd = data_sock; se->virtio_dev->se = se;pthread_rwlock_init (&se->virtio_dev->vu_dispatch_rwlock,NULL ); -vu_init (&se->virtio_dev->dev,2 , se->vu_socketfd, fv_panic, fv_set_watch, - fv_remove_watch, &fv_iface); +vu_init (&se->virtio_dev->dev,2 , se->vu_socketfd, fv_panic,NULL , + fv_set_watch, fv_remove_watch, &fv_iface);return 0 ; }
Signature · 1 line
2.26.2
PATCH v7 2/4 generic vhost user server
In reply to Coiby XuSharing QEMU devices via vhost-user protocol.
Only one vhost-user client can connect to the server one time.
--- util/Makefile.objs | 1 + util/vhost-user-server.c | 406 +++++++++++++++++++++++++++++++++++++++ util/vhost-user-server.h | 59 ++++++ 3 files changed, 466 insertions(+) create mode 100644 util/vhost-user-server.c create mode 100644 util/vhost-user-server.h
util/Makefile.objs +1 -0
@@ -40,6 +40,7 @@ util-obj-y += readline.o util-obj-y += rcu.o util-obj-$(CONFIG_MEMBARRIER) += sys_membarrier.o util-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o +util-obj-$(CONFIG_LINUX) += vhost-user-server.o util-obj-y += qemu-coroutine-sleep.o util-obj-y += qemu-co-shared-resource.o util-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
util/vhost-user-server.c new file +406 -0
406 lines not on this page · raw message
util/vhost-user-server.h new file +59 -0
59 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH v7 3/4 vhost-user block device backend server
In reply to Coiby XuBy making use of libvhost-user, block device drive can be shared to the connected vhost-user client. Only one client can connect to the server one time.
Since vhost-user-server needs a block drive to be created first, delay the creation of this object.
--- block/Makefile.objs | 1 + block/export/vhost-user-blk-server.c | 716 +++++++++++++++++++++++++++ block/export/vhost-user-blk-server.h | 34 ++ softmmu/vl.c | 4 + 4 files changed, 755 insertions(+) create mode 100644 block/export/vhost-user-blk-server.c create mode 100644 block/export/vhost-user-blk-server.h
block/Makefile.objs +1 -0
@@ -24,6 +24,7 @@ block-obj-y += throttle-groups.o block-obj-$(CONFIG_LINUX) += nvme.o block-obj-y += nbd.o +block-obj-$(CONFIG_LINUX) += export/vhost-user-blk-server.o ../contrib/libvhost-user/libvhost-user.o block-obj-$(CONFIG_SHEEPDOG) += sheepdog.o block-obj-$(CONFIG_LIBISCSI) += iscsi.o block-obj-$(if $(CONFIG_LIBISCSI),y,n) += iscsi-opts.o
block/export/vhost-user-blk-server.c new file +716 -0
716 lines not on this page · raw message
block/export/vhost-user-blk-server.h new file +34 -0
@@ -0,0 +1,34 @@ +/* +* Sharing QEMU block devices via vhost-user protocal +* +* Author: Coiby Xu <[email protected]> +* +* This work is licensed under the terms of the GNU GPL, version 2 or +* later. See the COPYING file in the top-level directory. +*/ + +#ifndef VHOST_USER_BLK_SERVER_H +#define VHOST_USER_BLK_SERVER_H +#include "util/vhost-user-server.h" + +typedef struct VuBlockDev VuBlockDev ; +#define TYPE_VHOST_USER_BLK_SERVER "vhost-user-blk-server" +#define VHOST_USER_BLK_SERVER (obj) \ + OBJECT_CHECK(VuBlockDev, obj, TYPE_VHOST_USER_BLK_SERVER) + +/* vhost user block device */ +struct VuBlockDev { +Object parent_obj; +char *node_name; +SocketAddress *addr; +AioContext *ctx; +VuServer vu_server; +uint32_t blk_size; +BlockBackend *backend; +QIOChannelSocket *sioc; +QTAILQ_ENTRY (VuBlockDev) next; +struct virtio_blk_config blkcfg; +bool writable; +}; + +#endif /* VHOST_USER_BLK_SERVER_H */
softmmu/vl.c +4 -0
@@ -2520,6 +2520,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts) }#endif +/* Reason: vhost-user-blk-server property "node-name" */ +if (g_str_equal (type,"vhost-user-blk-server" )) { +return false; + }/* * Reason: filter-* property "netdev" etc. */
Signature · 1 line
2.26.2
PATCH v7 4/4 new qTest case to test the vhost-user-blk-server
In reply to Coiby XuThis test case has the same tests as tests/virtio-blk-test.c except for tests have block_resize. Since vhost-user server can only server one client one time, two instances of qemu-storage-daemon are launched for the hotplug test.
In order to not block scripts/tap-driver.pl, vhost-user-blk-server will send "quit" command to qemu-storage-daemon's QMP monitor. So a function is added to libqtest.c to establish socket connection with socket server.
--- tests/Makefile.include | 3 +- tests/qtest/Makefile.include | 2 + tests/qtest/libqos/vhost-user-blk.c | 126 +++++ tests/qtest/libqos/vhost-user-blk.h | 44 ++ tests/qtest/libqtest.c | 44 +- tests/qtest/libqtest.h | 38 ++ tests/qtest/vhost-user-blk-test.c | 743 ++++++++++++++++++++++++++++ 7 files changed, 968 insertions(+), 32 deletions(-) create mode 100644 tests/qtest/libqos/vhost-user-blk.c create mode 100644 tests/qtest/libqos/vhost-user-blk.h create mode 100644 tests/qtest/vhost-user-blk-test.c
tests/Makefile.include +2 -1
@@ -636,7 +636,8 @@ endef $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: %-softmmu/all $(check-qtest-y) $(call do_test_human,$(check-qtest-$*-y:%=tests/qtest/%$(EXESUF)) $(check-qtest-generic-y:%=tests/qtest/%$(EXESUF)), \ QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \ - QTEST_QEMU_IMG=qemu-img$(EXESUF)) + QTEST_QEMU_IMG=./qemu-img$(EXESUF) \ + QTEST_QEMU_STORAGE_DAEMON_BINARY=./qemu-storage-daemon$(EXESUF)) check-unit: $(check-unit-y) $(call do_test_human, $^)
tests/qtest/Makefile.include +2 -0
@@ -186,6 +186,7 @@ libqos-obj-y += tests/qtest/libqos/virtio.o libqos-obj-$(CONFIG_VIRTFS) += tests/qtest/libqos/virtio-9p.o libqos-obj-y += tests/qtest/libqos/virtio-balloon.o libqos-obj-y += tests/qtest/libqos/virtio-blk.o +libqos-obj-$(CONFIG_LINUX) += tests/qtest/libqos/vhost-user-blk.o libqos-obj-y += tests/qtest/libqos/virtio-mmio.o libqos-obj-y += tests/qtest/libqos/virtio-net.o libqos-obj-y += tests/qtest/libqos/virtio-pci.o
@@ -230,6 +231,7 @@ qos-test-obj-$(CONFIG_VHOST_NET_USER) += tests/qtest/vhost-user-test.o $(chardev qos-test-obj-y += tests/qtest/virtio-test.o qos-test-obj-$(CONFIG_VIRTFS) += tests/qtest/virtio-9p-test.o qos-test-obj-y += tests/qtest/virtio-blk-test.o +qos-test-obj-$(CONFIG_LINUX) += tests/qtest/vhost-user-blk-test.o qos-test-obj-y += tests/qtest/virtio-net-test.o qos-test-obj-y += tests/qtest/virtio-rng-test.o qos-test-obj-y += tests/qtest/virtio-scsi-test.o
tests/qtest/libqos/vhost-user-blk.c new file +126 -0
126 lines not on this page · raw message
tests/qtest/libqos/vhost-user-blk.h new file +44 -0
44 lines not on this page · raw message
tests/qtest/libqtest.c +13 -31
59 lines not on this page · raw message
tests/qtest/libqtest.h +38 -0
@@ -20,8 +20,38 @@#include "qapi/qmp/qobject.h" #include "qapi/qmp/qdict.h" +#define MAX_IRQ 256 +typedef struct QTestState QTestState ; +typedef void (*QTestSendFn )(QTestState *s,const char *buf); +typedef void (*ExternalSendFn )(void *s,const char *buf); +typedef GString * (*QTestRecvFn )(QTestState *); + +typedef struct QTestClientTransportOps { +QTestSendFn send;/* for sending qtest commands */ + +/* +* use external_send to send qtest command strings through functions which +* do not accept a QTestState as the first parameter. +*/ +ExternalSendFn external_send; + +QTestRecvFn recv_line;/* for receiving qtest command responses */ +}QTestTransportOps ; + +struct QTestState { +int fd; +int qmp_fd; +pid_t qemu_pid;/* our child QEMU process */ +int wstatus; +int expected_status; +bool big_endian; +bool irq_level[MAX_IRQ ]; +GString *rx; +QTestTransportOps ops; +}; +/** * qtest_initf: * @fmt...: Format for creating other arguments to pass to QEMU, formatted
@@ -626,6 +656,14 @@ void qtest_add_data_func_full(const char *str, void *data,void qtest_add_abrt_handler (GHookFunc fn,const void *data); +/** +* qtest_socket_client: +* @server_socket_path: the socket server's path +* +* Connect to a socket server. +*/ +int qtest_socket_client (char *server_socket_path); +/** * qtest_qmp_assert_success: * @qts: QTestState instance to operate on
tests/qtest/vhost-user-blk-test.c new file +743 -0
743 lines not on this page · raw message
Signature · 1 line
2.26.2