Support host notifiers in vhost-user
v2 cover Support host notifiers in vhost-user
More details about this patch set can be found from:
http://lists.gnu.org/archive/html/qemu-devel/2018-04/msg01779.html
http://lists.gnu.org/archive/html/qemu-devel/2018-04/msg04626.html
v1 -> v2:
- Drop the VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD patch; (MST)
- In PATCH 2/5, for net/vhost-user, only save VhostUserState in queue index 0, i.e. where chr is saved;
- In PATCH 4/5, check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER before sending msg (including fd) to master; (MST)
Tiwei Bie (5):
vhost: allow backends to filter memory sections
vhost-user: introduce shared vhost-user state
vhost-user: support registering external host notifiers
libvhost-user: support host notifier
vhost-user-bridge: support host notifier
backends/cryptodev-vhost-user.c | 20 +++- contrib/libvhost-user/libvhost-user.c | 80 +++++++++++-- contrib/libvhost-user/libvhost-user.h | 31 +++++ docs/interop/vhost-user.txt | 33 ++++++ hw/block/vhost-user-blk.c | 22 +++- hw/scsi/vhost-user-scsi.c | 20 +++- hw/virtio/Makefile.objs | 2 +- hw/virtio/vhost-stub.c | 10 ++ hw/virtio/vhost-user.c | 165 ++++++++++++++++++++++++-- hw/virtio/vhost.c | 9 +- include/hw/virtio/vhost-backend.h | 4 + include/hw/virtio/vhost-user-blk.h | 2 + include/hw/virtio/vhost-user-scsi.h | 2 + include/hw/virtio/vhost-user.h | 28 +++++ net/vhost-user.c | 57 ++++++--- tests/vhost-user-bridge.c | 98 ++++++++++++++- 16 files changed, 542 insertions(+), 41 deletions(-) create mode 100644 include/hw/virtio/vhost-user.h
Signature · 1 line
2.17.0
PATCH v2 1/5 vhost: allow backends to filter memory sections
This patch introduces a vhost op for vhost backends to allow them to filter the memory sections that they can handle.
--- hw/virtio/vhost-user.c | 11 +++++++++++ hw/virtio/vhost.c | 9 +++++++-- include/hw/virtio/vhost-backend.h | 4 ++++ 3 files changed, 22 insertions(+), 2 deletions(-)
hw/virtio/vhost-user.c +11 -0
@@ -1620,6 +1620,16 @@ vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)return 0 ; } +static bool vhost_user_mem_section_filter (struct vhost_dev *dev, +MemoryRegionSection *section) +{ +bool result; + + result =memory_region_get_fd (section->mr) >=0 ; + +return result; +} +const VhostOps user_ops = { .backend_type =VHOST_BACKEND_TYPE_USER , .vhost_backend_init = vhost_user_init,
@@ -1650,4 +1660,5 @@ const VhostOps user_ops = { .vhost_set_config = vhost_user_set_config, .vhost_crypto_create_session = vhost_user_crypto_create_session, .vhost_crypto_close_session = vhost_user_crypto_close_session, + .vhost_backend_mem_section_filter = vhost_user_mem_section_filter, };
hw/virtio/vhost.c +7 -2
@@ -386,7 +386,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,return r; } -static bool vhost_section (MemoryRegionSection *section) +static bool vhost_section (struct vhost_dev *dev,MemoryRegionSection *section) {bool result; bool log_dirty = memory_region_get_dirty_log_mask(section->mr) &
@@ -399,6 +399,11 @@ static bool vhost_section(MemoryRegionSection *section)*/ result &= !log_dirty; +if (result && dev->vhost_ops->vhost_backend_mem_section_filter) { + result &= + dev->vhost_ops->vhost_backend_mem_section_filter (dev, section); + } +trace_vhost_section (section->mr->name, result);return result; }
@@ -632,7 +637,7 @@ static void vhost_region_addnop(MemoryListener *listener, struct vhost_dev *dev = container_of(listener, struct vhost_dev, memory_listener); -if (!vhost_section (section)) { +if (!vhost_section (dev, section)) {return ; }vhost_region_add_section (dev, section);
include/hw/virtio/vhost-backend.h +4 -0
@@ -101,6 +101,9 @@ typedef int (*vhost_crypto_create_session_op)(struct vhost_dev *dev,typedef int (*vhost_crypto_close_session_op )(struct vhost_dev *dev,uint64_t session_id); +typedef bool (*vhost_backend_mem_section_filter_op )(struct vhost_dev *dev, +MemoryRegionSection *section); + typedef struct VhostOps { VhostBackendType backend_type; vhost_backend_init vhost_backend_init;
@@ -138,6 +141,7 @@ typedef struct VhostOps {vhost_set_config_op vhost_set_config;vhost_crypto_create_session_op vhost_crypto_create_session;vhost_crypto_close_session_op vhost_crypto_close_session; +vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter; } VhostOps;extern const VhostOps user_ops;
Signature · 1 line
2.17.0
PATCH v2 2/5 vhost-user: introduce shared vhost-user state
In reply to Tiwei BieWhen multi queue is enabled e.g. for a virtio-net device, each queue pair will have a vhost_dev, and the only thing shared between vhost devs currently is the chardev. This patch introduces a vhost-user state structure which will be shared by all vhost devs of the same virtio device.
--- backends/cryptodev-vhost-user.c | 20 +++++++++- hw/block/vhost-user-blk.c | 22 ++++++++++- hw/scsi/vhost-user-scsi.c | 20 +++++++++- hw/virtio/Makefile.objs | 2 +- hw/virtio/vhost-stub.c | 10 +++++ hw/virtio/vhost-user.c | 31 +++++++++++----- include/hw/virtio/vhost-user-blk.h | 2 + include/hw/virtio/vhost-user-scsi.h | 2 + include/hw/virtio/vhost-user.h | 20 ++++++++++ net/vhost-user.c | 57 +++++++++++++++++++++-------- 10 files changed, 158 insertions(+), 28 deletions(-) create mode 100644 include/hw/virtio/vhost-user.h
backends/cryptodev-vhost-user.c +19 -1
@@ -26,6 +26,7 @@#include "qapi/error.h" #include "qapi/qmp/qerror.h" #include "qemu/error-report.h" +#include "hw/virtio/vhost-user.h" #include "standard-headers/linux/virtio_crypto.h" #include "sysemu/cryptodev-vhost.h" #include "chardev/char-fe.h"
@@ -46,6 +47,7 @@ typedef struct CryptoDevBackendVhostUser { CryptoDevBackend parent_obj; + VhostUserState *vhost_user; CharBackend chr; char *chr_name; bool opened;
@@ -102,7 +104,7 @@ cryptodev_vhost_user_start(int queues,continue ; } - options.opaque = &s->chr; + options.opaque = s->vhost_user; options.backend_type =VHOST_BACKEND_TYPE_USER ; options.cc = b->conf.peers.ccs[i]; s->vhost_crypto[i] =cryptodev_vhost_init (&options);
@@ -185,6 +187,7 @@ static void cryptodev_vhost_user_init(size_t i;Error *local_err =NULL ;Chardev *chr; +VhostUserState *user;CryptoDevBackendClient *cc;CryptoDevBackendVhostUser *s =CRYPTODEV_BACKEND_VHOST_USER (backend);
@@ -215,6 +218,15 @@ static void cryptodev_vhost_user_init( } } + user =vhost_user_init (); +if (!user) { +error_setg (errp,"Failed to init vhost_user" ); +return ; + } + + user->chr = &s->chr; + s->vhost_user = user; +qemu_chr_fe_set_handlers (&s->chr,NULL ,NULL , cryptodev_vhost_user_event,NULL , s,NULL , true);
@@ -299,6 +311,12 @@ static void cryptodev_vhost_user_cleanup( backend->conf.peers.ccs[i] =NULL ; } } + +if (s->vhost_user) { +vhost_user_cleanup (s->vhost_user); +g_free (s->vhost_user); + s->vhost_user =NULL ; + } } static void cryptodev_vhost_user_set_chardev(Object *obj,
hw/block/vhost-user-blk.c +21 -1
@@ -226,6 +226,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp) {VirtIODevice *vdev =VIRTIO_DEVICE (dev);VHostUserBlk *s =VHOST_USER_BLK (vdev); +VhostUserState *user;int i, ret;if (!s->chardev.chr) {
@@ -243,6 +244,15 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)return ; } + user =vhost_user_init (); +if (!user) { +error_setg (errp,"vhost-user-blk: failed to init vhost_user" ); +return ; + } + + user->chr = &s->chardev; + s->vhost_user = user; +virtio_init (vdev,"virtio-blk" ,VIRTIO_ID_BLOCK ,sizeof (struct virtio_blk_config ));
@@ -258,7 +268,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)vhost_dev_set_config_notifier (&s->dev, &blk_ops); - ret =vhost_dev_init (&s->dev, &s->chardev,VHOST_BACKEND_TYPE_USER ,0 ); + ret =vhost_dev_init (&s->dev, s->vhost_user,VHOST_BACKEND_TYPE_USER ,0 );if (ret <0 ) {error_setg (errp,"vhost-user-blk: vhost initialization failed: %s" ,strerror (-ret));
@@ -283,6 +293,10 @@ vhost_err: virtio_err:g_free (s->dev.vqs);virtio_cleanup (vdev); + +vhost_user_cleanup (user); +g_free (user); + s->vhost_user =NULL ; }static void vhost_user_blk_device_unrealize (DeviceState *dev,Error **errp)
@@ -294,6 +308,12 @@ static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp)vhost_dev_cleanup (&s->dev);g_free (s->dev.vqs);virtio_cleanup (vdev); + +if (s->vhost_user) { +vhost_user_cleanup (s->vhost_user); +g_free (s->vhost_user); + s->vhost_user =NULL ; + } }static void vhost_user_blk_instance_init (Object *obj)
hw/scsi/vhost-user-scsi.c +19 -1
@@ -69,6 +69,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)VirtIOSCSICommon *vs =VIRTIO_SCSI_COMMON (dev);VHostUserSCSI *s =VHOST_USER_SCSI (dev);VHostSCSICommon *vsc =VHOST_SCSI_COMMON (s); +VhostUserState *user;Error *err =NULL ;int ret;
@@ -85,19 +86,30 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)return ; } + user =vhost_user_init (); +if (!user) { +error_setg (errp,"vhost-user-scsi: failed to init vhost_user" ); +return ; + } + user->chr = &vs->conf.chardev; + vsc->dev.nvqs =2 + vs->conf.num_queues; vsc->dev.vqs = g_new(struct vhost_virtqueue, vsc->dev.nvqs); vsc->dev.vq_index =0 ; vsc->dev.backend_features =0 ; - ret =vhost_dev_init (&vsc->dev, (void *)&vs->conf.chardev, + ret =vhost_dev_init (&vsc->dev, user,VHOST_BACKEND_TYPE_USER ,0 );if (ret <0 ) {error_setg (errp,"vhost-user-scsi: vhost initialization failed: %s" ,strerror (-ret)); +vhost_user_cleanup (user); +g_free (user);return ; } + s->vhost_user = user; +/* Channel and lun both are 0 for bootable vhost-user-scsi disk */ vsc->channel =0 ; vsc->lun =0 ;
@@ -117,6 +129,12 @@ static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp)g_free (vsc->dev.vqs);virtio_scsi_common_unrealize (dev, errp); + +if (s->vhost_user) { +vhost_user_cleanup (s->vhost_user); +g_free (s->vhost_user); + s->vhost_user =NULL ; + } } static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev,
hw/virtio/Makefile.objs +1 -1
@@ -11,5 +11,5 @@ obj-y += virtio-crypto.o obj-$(CONFIG_VIRTIO_PCI) += virtio-crypto-pci.o endif -common-obj-$(call lnot,$(CONFIG_LINUX)) += vhost-stub.o +common-obj-$(call lnot,$(call land,$(CONFIG_VIRTIO),$(CONFIG_LINUX))) += vhost-stub.o common-obj-$(CONFIG_ALL) += vhost-stub.o
hw/virtio/vhost-stub.c +10 -0
@@ -1,7 +1,17 @@#include "qemu/osdep.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" bool vhost_has_free_slot (void ) {return true; } + +VhostUserState *vhost_user_init (void ) +{ +return NULL ; +} + +void vhost_user_cleanup (VhostUserState *user) +{ +}
hw/virtio/vhost-user.c +22 -9
@@ -11,6 +11,7 @@#include "qemu/osdep.h" #include "qapi/error.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-backend.h" #include "hw/virtio/virtio-net.h" #include "chardev/char-fe.h"
@@ -173,7 +174,8 @@ static VhostUserMsg m __attribute__ ((unused));struct vhost_user {struct vhost_dev *dev; -CharBackend *chr; +/* Shared between vhost devs of the same virtio device */ +VhostUserState *user;int slave_fd;NotifierWithReturn postcopy_notifier;struct PostCopyFD postcopy_fd;
@@ -199,7 +201,7 @@ static bool ioeventfd_enabled(void)static int vhost_user_read (struct vhost_dev *dev,VhostUserMsg *msg) {struct vhost_user *u = dev->opaque; -CharBackend *chr = u->chr; +CharBackend *chr = u->user->chr;uint8_t *p = (uint8_t *) msg;int r, size =VHOST_USER_HDR_SIZE ;
@@ -285,7 +287,7 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg, int *fds, int fd_num) {struct vhost_user *u = dev->opaque; -CharBackend *chr = u->chr; +CharBackend *chr = u->user->chr;int ret, size =VHOST_USER_HDR_SIZE + msg->hdr.size;/*
@@ -1083,7 +1085,7 @@ static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,static int vhost_user_postcopy_advise (struct vhost_dev *dev,Error **errp) {struct vhost_user *u = dev->opaque; -CharBackend *chr = u->chr; +CharBackend *chr = u->user->chr;int ufd;VhostUserMsg msg = { .hdr.request =VHOST_USER_POSTCOPY_ADVISE ,
@@ -1221,7 +1223,7 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,return 0 ; } -static int vhost_user_init (struct vhost_dev *dev,void *opaque) +static int vhost_user_backend_init (struct vhost_dev *dev,void *opaque) {uint64_t features, protocol_features;struct vhost_user *u;
@@ -1230,7 +1232,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)assert (dev->vhost_ops->backend_type ==VHOST_BACKEND_TYPE_USER ); u = g_new0(struct vhost_user, 1); - u->chr = opaque; + u->user = opaque; u->slave_fd =-1 ; u->dev = dev; dev->opaque = u;
@@ -1306,7 +1308,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)return 0 ; } -static int vhost_user_cleanup (struct vhost_dev *dev) +static int vhost_user_backend_cleanup (struct vhost_dev *dev) {struct vhost_user *u;
@@ -1630,10 +1632,21 @@ static bool vhost_user_mem_section_filter(struct vhost_dev *dev,return result; } +VhostUserState *vhost_user_init (void ) +{ + VhostUserState *user = g_new0(struct VhostUserState, 1); + +return user; +} + +void vhost_user_cleanup (VhostUserState *user) +{ +} +const VhostOps user_ops = { .backend_type =VHOST_BACKEND_TYPE_USER , - .vhost_backend_init = vhost_user_init, - .vhost_backend_cleanup = vhost_user_cleanup, + .vhost_backend_init = vhost_user_backend_init, + .vhost_backend_cleanup = vhost_user_backend_cleanup, .vhost_backend_memslots_limit = vhost_user_memslots_limit, .vhost_set_log_base = vhost_user_set_log_base, .vhost_set_mem_table = vhost_user_set_mem_table,
include/hw/virtio/vhost-user-blk.h +2 -0
@@ -21,6 +21,7 @@#include "hw/block/block.h" #include "chardev/char-fe.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #define TYPE_VHOST_USER_BLK "vhost-user-blk"#define VHOST_USER_BLK (obj) \
@@ -36,6 +37,7 @@ typedef struct VHostUserBlk {uint32_t config_wce;uint32_t config_ro;struct vhost_dev dev; + VhostUserState *vhost_user; } VHostUserBlk;#endif
include/hw/virtio/vhost-user-scsi.h +2 -0
@@ -21,6 +21,7 @@#include "hw/qdev.h" #include "hw/virtio/virtio-scsi.h" #include "hw/virtio/vhost.h" +#include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-scsi-common.h" #define TYPE_VHOST_USER_SCSI "vhost-user-scsi"
@@ -30,6 +31,7 @@typedef struct VHostUserSCSI {VHostSCSICommon parent_obj;uint64_t host_features; +VhostUserState *vhost_user; }VHostUserSCSI ;#endif /* VHOST_USER_SCSI_H */
include/hw/virtio/vhost-user.h new file +20 -0
@@ -0,0 +1,20 @@ +/* +* Copyright (c) 2017-2018 Intel Corporation +* +* This work is licensed under the terms of the GNU GPL, version 2. +* See the COPYING file in the top-level directory. +*/ + +#ifndef HW_VIRTIO_VHOST_USER_H +#define HW_VIRTIO_VHOST_USER_H + +#include "chardev/char-fe.h" + +typedef struct VhostUserState { +CharBackend *chr; +}VhostUserState ; + +VhostUserState *vhost_user_init (void ); +void vhost_user_cleanup (VhostUserState *user); + +#endif
net/vhost-user.c +42 -15
129 lines not on this page · raw message
Signature · 1 line
2.17.0
PATCH v2 3/5 vhost-user: support registering external host notifiers
In reply to Tiwei BieThis patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER. With this feature negotiated, vhost-user backend can register memory region based host notifiers. And it will allow the guest driver in the VM to notify the hardware accelerator at the vhost-user backend directly.
--- docs/interop/vhost-user.txt | 33 +++++++++ hw/virtio/vhost-user.c | 123 +++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-user.h | 8 +++ 3 files changed, 164 insertions(+)
docs/interop/vhost-user.txt +33 -0
@@ -132,6 +132,16 @@ Depending on the request type, payload can be: Payload: Size bytes array holding the contents of the virtio device's configuration space + * Vring area description + ----------------------- + | u64 | size | offset | + ----------------------- + + u64: a 64-bit integer contains vring index and flags + Size: a 64-bit size of this area + Offset: a 64-bit offset of this area from the start of the + supplied file descriptor + In QEMU the vhost-user message is implemented with the following struct: typedef struct VhostUserMsg {
@@ -146,6 +156,7 @@ typedef struct VhostUserMsg { VhostUserLog log; struct vhost_iotlb_msg iotlb; VhostUserConfig config; + VhostUserVringArea area; }; } QEMU_PACKED VhostUserMsg;
@@ -380,6 +391,7 @@ Protocol features #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7 #define VHOST_USER_PROTOCOL_F_PAGEFAULT 8 #define VHOST_USER_PROTOCOL_F_CONFIG 9 +#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 10 Master message types --------------------
@@ -777,6 +789,27 @@ Slave message types the VHOST_USER_NEED_REPLY flag, master must respond with zero when operation is successfully completed, or non-zero otherwise. + * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG + + Id: 3 + Equivalent ioctl: N/A + Slave payload: vring area description + Master payload: N/A + + Sets host notifier for a specified queue. The queue index is contained + in the u64 field of the vring area description. The host notifier is + described by the file descriptor (typically it's a VFIO device fd) which + is passed as ancillary data and the size (which is mmap size and should + be the same as host page size) and offset (which is mmap offset) carried + in the vring area description. QEMU can mmap the file descriptor based + on the size and offset to get a memory range. Registering a host notifier + means mapping this memory range to the VM as the specified queue's notify + MMIO region. Slave sends this request to tell QEMU to de-register the + existing notifier if any and register the new notifier if the request is + sent with a file descriptor. + This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER + protocol feature has been successfully negotiated. + VHOST_USER_PROTOCOL_F_REPLY_ACK: ------------------------------- The original vhost-user specification only demands replies for certain
hw/virtio/vhost-user.c +123 -0
186 lines not on this page · raw message
include/hw/virtio/vhost-user.h +8 -0
@@ -9,9 +9,17 @@#define HW_VIRTIO_VHOST_USER_H #include "chardev/char-fe.h" +#include "hw/virtio/virtio.h" + +typedef struct VhostUserHostNotifier { +MemoryRegion mr; +void *addr; +bool set; +}VhostUserHostNotifier ;typedef struct VhostUserState {CharBackend *chr; +VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX ]; }VhostUserState ;VhostUserState *vhost_user_init (void );
Signature · 1 line
2.17.0
Re: [Qemu-devel] [PATCH v2 3/5] vhost-user: support registering external host notifiers
Quoted 209 lines · Tiwei Bie
This patch introduces
VHOST_USER_PROTOCOL_F_HOST_NOTIFIER. With this feature negotiated, vhost-user backend can register memory region based host notifiers. And it will allow the guest driver in the VM to notify the hardware accelerator at the vhost-user backend directly.Signed-off-by Tiwei Bie <[email protected]>--- docs/interop/vhost-user.txt | 33 +++++++++ hw/virtio/vhost-user.c | 123 +++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-user.h | 8 +++ 3 files changed, 164 insertions(+)docs/interop/vhost-user.txt +33
-0@@ -132,6 +132,16 @@ Depending on the request type, payload can be: Payload: Size bytes array holding the contents of the virtio device's configuration space + * Vring area description + ----------------------- + | u64 | size | offset | + ----------------------- + + u64: a 64-bit integer contains vring index and flags + Size: a 64-bit size of this area + Offset: a 64-bit offset of this area from the start of the + supplied file descriptor + In QEMU the vhost-user message is implemented with the following struct: typedef struct VhostUserMsg {@@ -146,6 +156,7 @@ typedef struct VhostUserMsg { VhostUserLog log; struct vhost_iotlb_msg iotlb; VhostUserConfig config; + VhostUserVringArea area; }; } QEMU_PACKED VhostUserMsg;@@ -380,6 +391,7 @@ Protocol features #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7 #define VHOST_USER_PROTOCOL_F_PAGEFAULT 8 #define VHOST_USER_PROTOCOL_F_CONFIG 9 +#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 10 Master message types --------------------@@ -777,6 +789,27 @@ Slave message types the VHOST_USER_NEED_REPLY flag, master must respond with zero when operation is successfully completed, or non-zero otherwise. + * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG + + Id: 3 + Equivalent ioctl: N/A + Slave payload: vring area description + Master payload: N/A + + Sets host notifier for a specified queue. The queue index is contained + in the u64 field of the vring area description. The host notifier is + described by the file descriptor (typically it's a VFIO device fd) which + is passed as ancillary data and the size (which is mmap size and should + be the same as host page size) and offset (which is mmap offset) carried + in the vring area description. QEMU can mmap the file descriptor based + on the size and offset to get a memory range. Registering a host notifier + means mapping this memory range to the VM as the specified queue's notify + MMIO region. Slave sends this request to tell QEMU to de-register the + existing notifier if any and register the new notifier if the request is + sent with a file descriptor. + This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER + protocol feature has been successfully negotiated. + VHOST_USER_PROTOCOL_F_REPLY_ACK: ------------------------------- The original vhost-user specification only demands replies for certainhw/virtio/vhost-user.c +71
-0@@ -13,6 +13,7 @@#include "hw/virtio/vhost.h" #include "hw/virtio/vhost-user.h" #include "hw/virtio/vhost-backend.h" +#include "hw/virtio/virtio.h" #include "hw/virtio/virtio-net.h" #include "chardev/char-fe.h" #include "sysemu/kvm.h" @@ -48,6 +49,7 @@ enum VhostUserProtocolFeature {VHOST_USER_PROTOCOL_F_CRYPTO_SESSION =7 ,VHOST_USER_PROTOCOL_F_PAGEFAULT =8 ,VHOST_USER_PROTOCOL_F_CONFIG =9 , +VHOST_USER_PROTOCOL_F_HOST_NOTIFIER =10 , VHOST_USER_PROTOCOL_F_MAX };@@ -92,6 +94,7 @@ typedef enum VhostUserSlaveRequest {VHOST_USER_SLAVE_NONE =0 ,VHOST_USER_SLAVE_IOTLB_MSG =1 ,VHOST_USER_SLAVE_CONFIG_CHANGE_MSG =2 , +VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG =3 , VHOST_USER_SLAVE_MAX } VhostUserSlaveRequest;@@ -136,6 +139,12 @@ static VhostUserConfig c __attribute__ ((unused)); + sizeof(c.size) \ + sizeof(c.flags)) +typedef struct VhostUserVringArea { +uint64_t u64; +uint64_t size; +uint64_t offset; +}VhostUserVringArea ; + typedef struct { VhostUserRequest request;@@ -157,6 +166,7 @@ typedef union {struct vhost_iotlb_msg iotlb;VhostUserConfig config;VhostUserCryptoSession session; +VhostUserVringArea area; } VhostUserPayload; typedef struct VhostUserMsg {@@ -638,9 +648,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,return vhost_set_vring (dev,VHOST_USER_SET_VRING_NUM , ring); } +static void vhost_user_host_notifier_restore (struct vhost_dev *dev, +int queue_idx) +{ +struct vhost_user *u = dev->opaque; +VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; +VirtIODevice *vdev = dev->vdev; + +if (n->addr && !n->set) { +virtio_queue_set_host_notifier_mr (vdev, queue_idx, &n->mr, true); + n->set = true; + } +} + +static void vhost_user_host_notifier_remove (struct vhost_dev *dev, +int queue_idx) +{ +struct vhost_user *u = dev->opaque; +VhostUserHostNotifier *n = &u->user->notifier[queue_idx]; +VirtIODevice *vdev = dev->vdev; + +if (n->addr && n->set) { +virtio_queue_set_host_notifier_mr (vdev, queue_idx, &n->mr, false); + n->set = false; + } +} +static int vhost_user_set_vring_base (struct vhost_dev *dev,struct vhost_vring_state *ring) { +vhost_user_host_notifier_restore (dev, ring->index); +return vhost_set_vring (dev,VHOST_USER_SET_VRING_BASE , ring); }@@ -674,6 +712,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.state), }; +vhost_user_host_notifier_remove (dev, ring->index); +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; }@@ -847,6 +887,76 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)return ret; } +static int vhost_user_slave_handle_vring_host_notifier (struct vhost_dev *dev, +VhostUserVringArea *area, +int fd) +{ +int queue_idx = area->u64 &VHOST_USER_VRING_IDX_MASK ; +size_t page_size = qemu_real_host_page_size; +struct vhost_user *u = dev->opaque; +VhostUserState *user = u->user; +VirtIODevice *vdev = dev->vdev; +VhostUserHostNotifier *n; +int ret =0 ; +void *addr; +char *name; + +if (!virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_HOST_NOTIFIER ) || + vdev ==NULL || queue_idx >=virtio_get_num_queues (vdev)) { + ret =-1 ; + goto out; + } + + n = &user->notifier[queue_idx]; + +if (n->addr) { +virtio_queue_set_host_notifier_mr (vdev, queue_idx, &n->mr, false); +object_unparent (OBJECT (&n->mr)); +munmap (n->addr, page_size); + n->addr =NULL ; + } + +if (area->u64 &VHOST_USER_VRING_NOFD_MASK ) {
hw/virtio/vhost-user.c static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) · line 930
Quoted 10 lines
hw/virtio/vhost-user.c +10
-0+ goto out; + } + +/* Sanity check. */ +if (area->size != page_size) { + ret =-1 ; + goto out; + } + + addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
It looks to me that PROT_WRITE is sufficient here?
Thanks
Quoted 81 lines
hw/virtio/vhost-user.c +42
-0+ fd, area->offset); +if (addr ==MAP_FAILED ) { + ret =-1 ; + goto out; + } + + name =g_strdup_printf ("vhost-user/host-notifier@%p mmaps[%d]" , + user, queue_idx); +memory_region_init_ram_device_ptr (&n->mr,OBJECT (vdev), name, + page_size, addr); +g_free (name); + +if (virtio_queue_set_host_notifier_mr (vdev, queue_idx, &n->mr, true)) { +munmap (addr, page_size); + ret =-1 ; + goto out; + } + + n->addr = addr; + n->set = true; + +out: +/* Always close the fd. */ +if (fd !=-1 ) { +close (fd); + } +return ret; +} +static void slave_read (void *opaque) {struct vhost_dev *dev = opaque;@@ -913,6 +1023,10 @@ static void slave_read(void *opaque)case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG : ret =vhost_user_slave_handle_config_change (dev);break ; +case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG : + ret =vhost_user_slave_handle_vring_host_notifier (dev, &payload.area, + fd); +break ;default :error_report ("Received unexpected msg type." );if (fd !=-1 ) {@@ -1641,6 +1755,15 @@ VhostUserState *vhost_user_init(void)void vhost_user_cleanup (VhostUserState *user) { +int i; + +for (i =0 ; i <VIRTIO_QUEUE_MAX ; i++) { +if (user->notifier[i].addr) { +object_unparent (OBJECT (&user->notifier[i].mr)); +munmap (user->notifier[i].addr, qemu_real_host_page_size); + user->notifier[i].addr =NULL ; + } + } }const VhostOps user_ops = {include/hw/virtio/vhost-user.h +8
-0@@ -9,9 +9,17 @@#define HW_VIRTIO_VHOST_USER_H #include "chardev/char-fe.h" +#include "hw/virtio/virtio.h" + +typedef struct VhostUserHostNotifier { +MemoryRegion mr; +void *addr; +bool set; +}VhostUserHostNotifier ;typedef struct VhostUserState {CharBackend *chr; +VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX ]; }VhostUserState ;VhostUserState *vhost_user_init (void );
Re: [Qemu-devel] [PATCH v2 3/5] vhost-user: support registering external host notifiers
Quoted 1 line · Jason Wang
On 2018年05月25日 19:04, Tiwei Bie wrote:
[...]
Quoted 44 lines
Quoted 31 lines
hw/virtio/vhost-user.c +31
-0+static int vhost_user_slave_handle_vring_host_notifier (struct vhost_dev *dev, +VhostUserVringArea *area, +int fd) +{ +int queue_idx = area->u64 &VHOST_USER_VRING_IDX_MASK ; +size_t page_size = qemu_real_host_page_size; +struct vhost_user *u = dev->opaque; +VhostUserState *user = u->user; +VirtIODevice *vdev = dev->vdev; +VhostUserHostNotifier *n; +int ret =0 ; +void *addr; +char *name; + +if (!virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_HOST_NOTIFIER ) || + vdev ==NULL || queue_idx >=virtio_get_num_queues (vdev)) { + ret =-1 ; + goto out; + } + + n = &user->notifier[queue_idx]; + +if (n->addr) { +virtio_queue_set_host_notifier_mr (vdev, queue_idx, &n->mr, false); +object_unparent (OBJECT (&n->mr)); +munmap (n->addr, page_size); + n->addr =NULL ; + } + +if (area->u64 &VHOST_USER_VRING_NOFD_MASK ) {hw/virtio/vhost-user.c static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) · line 930
Quoted 10 lines
hw/virtio/vhost-user.c +10
-0+ goto out; + } + +/* Sanity check. */ +if (area->size != page_size) { + ret =-1 ; + goto out; + } + + addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,It looks to me that
PROT_WRITEis sufficient here?
Quoted 2 lines
Thanks
I think it's safer to add PROT_READ. And there's
no harm to do that.
Best regards,
Tiwei Bie
PATCH v2 4/5 libvhost-user: support host notifier
In reply to Tiwei BieThis patch introduces the host notifier support in libvhost-user. A new API is added to support setting host notifier for each queue.
--- contrib/libvhost-user/libvhost-user.c | 80 ++++++++++++++++++++++++--- contrib/libvhost-user/libvhost-user.h | 31 +++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-)
contrib/libvhost-user/libvhost-user.c +72 -8
@@ -314,11 +314,6 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg) msg.msg_controllen =0 ; } -/* Set the version in the flags when sending the reply */ - vmsg->flags &= ~VHOST_USER_VERSION_MASK ; - vmsg->flags |=VHOST_USER_VERSION ; - vmsg->flags |=VHOST_USER_REPLY_MASK ; -do { rc =sendmsg (conn_fd, &msg,0 ); }while (rc <0 && (errno ==EINTR || errno ==EAGAIN ));
@@ -341,6 +336,39 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)return true; } +static bool +vu_send_reply (VuDev *dev,int conn_fd,VhostUserMsg *vmsg) +{ +/* Set the version in the flags when sending the reply */ + vmsg->flags &= ~VHOST_USER_VERSION_MASK ; + vmsg->flags |=VHOST_USER_VERSION ; + vmsg->flags |=VHOST_USER_REPLY_MASK ; + +return vu_message_write (dev, conn_fd, vmsg); +} + +static bool +vu_process_message_reply (VuDev *dev,const VhostUserMsg *vmsg) +{ +VhostUserMsg msg_reply; + +if ((vmsg->flags &VHOST_USER_NEED_REPLY_MASK ) ==0 ) { +return true; + } + +if (!vu_message_read (dev, dev->slave_fd, &msg_reply)) { +return false; + } + +if (msg_reply.request != vmsg->request) { +DPRINT ("Received unexpected msg type. Expected %d received %d" , + vmsg->request, msg_reply.request); +return false; + } + +return msg_reply.payload.u64 ==0 ; +} +/* Kick the log_call_fd if required. */ static void vu_log_kick (VuDev *dev)
@@ -536,7 +564,7 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)/* Send the message back to qemu with the addresses filled in */ vmsg->fd_num =0 ; -if (!vu_message_write (dev, dev->sock, vmsg)) { +if (!vu_send_reply (dev, dev->sock, vmsg)) {vu_panic (dev,"failed to respond to set-mem-table for postcopy" );return false; }
@@ -916,6 +944,41 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq, } } +bool vu_set_queue_host_notifier (VuDev *dev,VuVirtq *vq,int fd, +int size,int offset) +{ +int qidx = vq - dev->vq; +int fd_num =0 ; +VhostUserMsg vmsg = { + .request =VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG , + .flags =VHOST_USER_VERSION |VHOST_USER_NEED_REPLY_MASK , + .size =sizeof (vmsg.payload.area), + .payload.area = { + .u64 = qidx &VHOST_USER_VRING_IDX_MASK , + .size = size, + .offset = offset, + }, + }; + +if ((dev->protocol_features &VHOST_USER_PROTOCOL_F_HOST_NOTIFIER ) ==0 ) { +return false; + } + +if (fd ==-1 ) { + vmsg.payload.area.u64 |=VHOST_USER_VRING_NOFD_MASK ; + }else { + vmsg.fds[fd_num++] = fd; + } + + vmsg.fd_num = fd_num; + +if (!vu_message_write (dev, dev->slave_fd, &vmsg)) { +return false; + } + +return vu_process_message_reply (dev, &vmsg); +} +static bool vu_set_vring_call_exec (VuDev *dev,VhostUserMsg *vmsg) {
@@ -968,7 +1031,8 @@ static bool vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg) {uint64_t features =1ULL <<VHOST_USER_PROTOCOL_F_LOG_SHMFD | -1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_REQ ; +1ULL <<VHOST_USER_PROTOCOL_F_SLAVE_REQ | +1ULL <<VHOST_USER_PROTOCOL_F_HOST_NOTIFIER ;if (have_userfault ()) { features |=1ULL <<VHOST_USER_PROTOCOL_F_PAGEFAULT ;
@@ -1252,7 +1316,7 @@ vu_dispatch(VuDev *dev) goto end; } -if (!vu_message_write (dev, dev->sock, &vmsg)) { +if (!vu_send_reply (dev, dev->sock, &vmsg)) { goto end; }
contrib/libvhost-user/libvhost-user.h +31 -0
@@ -51,6 +51,7 @@ enum VhostUserProtocolFeature {VHOST_USER_PROTOCOL_F_CRYPTO_SESSION =7 ,VHOST_USER_PROTOCOL_F_PAGEFAULT =8 ,VHOST_USER_PROTOCOL_F_CONFIG =9 , +VHOST_USER_PROTOCOL_F_HOST_NOTIFIER =10 , VHOST_USER_PROTOCOL_F_MAX };
@@ -92,6 +93,14 @@ typedef enum VhostUserRequest {VHOST_USER_MAX } VhostUserRequest; +typedef enum VhostUserSlaveRequest { +VHOST_USER_SLAVE_NONE =0 , +VHOST_USER_SLAVE_IOTLB_MSG =1 , +VHOST_USER_SLAVE_CONFIG_CHANGE_MSG =2 , +VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG =3 , +VHOST_USER_SLAVE_MAX +}VhostUserSlaveRequest ; + typedef struct VhostUserMemoryRegion { uint64_t guest_phys_addr; uint64_t memory_size;
@@ -122,6 +131,12 @@ static VhostUserConfig c __attribute__ ((unused)); + sizeof(c.size) \ + sizeof(c.flags)) +typedef struct VhostUserVringArea { +uint64_t u64; +uint64_t size; +uint64_t offset; +}VhostUserVringArea ; +#if defined(_WIN32)# define VU_PACKED __attribute__((gcc_struct, packed))#else
@@ -133,6 +148,7 @@ typedef struct VhostUserMsg {#define VHOST_USER_VERSION_MASK (0x3)#define VHOST_USER_REPLY_MASK (0x1 << 2) +#define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)uint32_t flags;uint32_t size;/* the following payload size */
@@ -145,6 +161,7 @@ typedef struct VhostUserMsg {VhostUserMemory memory;VhostUserLog log;VhostUserConfig config; +VhostUserVringArea area; } payload;int fds[VHOST_MEMORY_MAX_NREGIONS ];
@@ -368,6 +385,20 @@ VuVirtq *vu_get_queue(VuDev *dev, int qidx);void vu_set_queue_handler (VuDev *dev,VuVirtq *vq,vu_queue_handler_cb handler); +/** +* vu_set_queue_host_notifier: +* @dev: a VuDev context +* @vq: a VuVirtq queue +* @fd: a file descriptor +* @size: host page size +* @offset: notifier offset in @fd file +* +* Set queue's host notifier. This function may be called several +* times for the same queue. If called with -1 @fd, the notifier +* is removed. +*/ +bool vu_set_queue_host_notifier (VuDev *dev,VuVirtq *vq,int fd, +int size,int offset);/** * vu_queue_set_notification:
Signature · 1 line
2.17.0
PATCH v2 5/5 vhost-user-bridge: support host notifier
In reply to Tiwei BieThis patch introduces the host notifier support in vhost-user-bridge. A new option (-H) is added to use the host notifier. This is mainly used to test the host notifier implementation in vhost user.
--- tests/vhost-user-bridge.c | 98 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-)
tests/vhost-user-bridge.c +95 -3
@@ -29,6 +29,7 @@#define _FILE_OFFSET_BITS 64 +#include "qemu/atomic.h" #include "qemu/osdep.h" #include "qemu/iov.h" #include "standard-headers/linux/virtio_net.h"
@@ -65,6 +66,11 @@ typedef struct VubrDev {int sock;int ready;int quit; +struct { +int fd; +void *addr; +pthread_t thread; + } notifier; } VubrDev; static void
@@ -445,14 +451,22 @@ static uint64_t vubr_get_features(VuDev *dev) {return 1ULL <<VIRTIO_NET_F_GUEST_ANNOUNCE | -1ULL <<VIRTIO_NET_F_MRG_RXBUF ; +1ULL <<VIRTIO_NET_F_MRG_RXBUF | +1ULL <<VIRTIO_F_VERSION_1 ; }static void vubr_queue_set_started (VuDev *dev,int qidx,bool started) { +VubrDev *vubr =container_of (dev, VubrDev, vudev);VuVirtq *vq =vu_get_queue (dev, qidx); +if (started && vubr->notifier.fd >=0 ) { +vu_set_queue_host_notifier (dev, vq, vubr->notifier.fd, +getpagesize (), + qidx *getpagesize ()); + } +if (qidx %2 ==1 ) {vu_set_queue_handler (dev, vq, started ? vubr_handle_tx :NULL ); }
@@ -522,6 +536,8 @@ vubr_new(const char *path, bool client)vubr_die ("socket" ); } + dev->notifier.fd =-1 ; + un.sun_family =AF_UNIX ;strcpy (un.sun_path, path); len =sizeof (un.sun_family) +strlen (path);
@@ -559,6 +575,73 @@ vubr_new(const char *path, bool client)return dev; } +static void *notifier_thread (void *arg) +{ +VuDev *dev = (VuDev *)arg; +VubrDev *vubr =container_of (dev, VubrDev, vudev); +int pagesize =getpagesize (); +int qidx; + +while (true) { +for (qidx =0 ; qidx <VHOST_MAX_NR_VIRTQUEUE ; qidx++) { +uint16_t *n = vubr->notifier.addr + pagesize * qidx; + +if (*n == qidx) { + *n =0xffff ; +/* We won't miss notifications if we reset +* the memory first. */ +smp_mb (); + +DPRINT ("Got a notification for queue%d via host notifier.\n" , + qidx); + +if (qidx %2 ==1 ) { +vubr_handle_tx (dev, qidx); + } + } +usleep (1000 ); + } + } + +return NULL ; +} + +static void +vubr_host_notifier_setup (VubrDev *dev) +{ +char template[] ="/tmp/vubr-XXXXXX" ; +pthread_t thread; +size_t length; +void *addr; +int fd; + + length =getpagesize () *VHOST_MAX_NR_VIRTQUEUE ; + + fd =mkstemp (template); +if (fd <0 ) { +vubr_die ("mkstemp()" ); + } + +if (posix_fallocate (fd,0 , length) !=0 ) { +vubr_die ("posix_fallocate()" ); + } + + addr =mmap (NULL , length,PROT_READ |PROT_WRITE ,MAP_SHARED , fd,0 ); +if (addr ==MAP_FAILED ) { +vubr_die ("mmap()" ); + } + +memset (addr,0xff , length); + +if (pthread_create (&thread,NULL , notifier_thread, &dev->vudev) !=0 ) { +vubr_die ("pthread_create()" ); + } + + dev->notifier.fd = fd; + dev->notifier.addr = addr; + dev->notifier.thread = thread; +} +static void vubr_set_host (struct sockaddr_in *saddr,const char *host) {
@@ -673,8 +756,9 @@ main(int argc, char *argv[])VubrDev *dev;int opt;bool client = false; +bool host_notifier = false; -while ((opt =getopt (argc, argv,"l:r:u:c" )) !=-1 ) { +while ((opt =getopt (argc, argv,"l:r:u:cH" )) !=-1 ) {switch (opt) {case 'l' :
@@ -693,6 +777,9 @@ main(int argc, char *argv[])case 'c' : client = true;break ; +case 'H' : + host_notifier = true; +break ;default : goto out; }
@@ -708,6 +795,10 @@ main(int argc, char *argv[])return 1 ; } +if (host_notifier) { +vubr_host_notifier_setup (dev); + } +vubr_backend_udp_setup (dev, lhost, lport, rhost, rport);vubr_run (dev);
@@ -717,7 +808,7 @@ main(int argc, char *argv[]) out:fprintf (stderr,"Usage: %s " , argv[0 ]); -fprintf (stderr,"[-c] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n" ); +fprintf (stderr,"[-c] [-H] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n" );fprintf (stderr,"\t-u path to unix doman socket. default: %s\n" ,DEFAULT_UD_SOCKET ); fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
@@ -725,6 +816,7 @@ out:fprintf (stderr,"\t-r remote host and port. default: %s:%s\n" ,DEFAULT_RHOST ,DEFAULT_RPORT );fprintf (stderr,"\t-c client mode\n" ); +fprintf (stderr,"\t-H use host notifier\n" );return 1 ; }
Signature · 1 line
2.17.0