Skip to contentSkip to conversation

Support host notifiers in vhost-user

PATCH In review Tiwei Bie 2 participants 8 messages May 25, 2018 Networking

lore
Share & tools
lore

Tiwei Bie

author
#0 lore raw

v2 cover Support host notifiers in vhost-user

More details about this patch set can be found from:

http://lists.gnu.org/archive/html/qemu-devel/2018-04/msg01779.html
http://lists.gnu.org/archive/html/qemu-devel/2018-04/msg04626.html

v1 -> v2:
- Drop the VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD patch; (MST)
- In PATCH 2/5, for net/vhost-user, only save VhostUserState in queue index 0, i.e. where chr is saved;
- In PATCH 4/5, check VHOST_USER_PROTOCOL_F_HOST_NOTIFIER before sending msg (including fd) to master; (MST)

Tiwei Bie (5):
vhost: allow backends to filter memory sections
vhost-user: introduce shared vhost-user state
vhost-user: support registering external host notifiers
libvhost-user: support host notifier
vhost-user-bridge: support host notifier

 backends/cryptodev-vhost-user.c       |  20 +++-
 contrib/libvhost-user/libvhost-user.c |  80 +++++++++++--
 contrib/libvhost-user/libvhost-user.h |  31 +++++
 docs/interop/vhost-user.txt           |  33 ++++++
 hw/block/vhost-user-blk.c             |  22 +++-
 hw/scsi/vhost-user-scsi.c             |  20 +++-
 hw/virtio/Makefile.objs               |   2 +-
 hw/virtio/vhost-stub.c                |  10 ++
 hw/virtio/vhost-user.c                | 165 ++++++++++++++++++++++++--
 hw/virtio/vhost.c                     |   9 +-
 include/hw/virtio/vhost-backend.h     |   4 +
 include/hw/virtio/vhost-user-blk.h    |   2 +
 include/hw/virtio/vhost-user-scsi.h   |   2 +
 include/hw/virtio/vhost-user.h        |  28 +++++
 net/vhost-user.c                      |  57 ++++++---
 tests/vhost-user-bridge.c             |  98 ++++++++++++++-
 16 files changed, 542 insertions(+), 41 deletions(-)
 create mode 100644 include/hw/virtio/vhost-user.h
Signature · 1 line
2.17.0
author
#1 lore raw

PATCH v2 1/5 vhost: allow backends to filter memory sections

This patch introduces a vhost op for vhost backends to allow them to filter the memory sections that they can handle.

---
 hw/virtio/vhost-user.c            | 11 +++++++++++
 hw/virtio/vhost.c                 |  9 +++++++--
 include/hw/virtio/vhost-backend.h |  4 ++++
 3 files changed, 22 insertions(+), 2 deletions(-)
hw/virtio/vhost-user.c +11 -0
index ca554d4ff1..da0756effe 100644
@@ -1620,6 +1620,16 @@ vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
     return 0;
 }
 
+static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
+                                          MemoryRegionSection *section)
+{
+    bool result;
+
+    result = memory_region_get_fd(section->mr) >= 0;
+
+    return result;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
         .vhost_backend_init = vhost_user_init,
@@ -1650,4 +1660,5 @@ const VhostOps user_ops = {
         .vhost_set_config = vhost_user_set_config,
         .vhost_crypto_create_session = vhost_user_crypto_create_session,
         .vhost_crypto_close_session = vhost_user_crypto_close_session,
+        .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
 };
hw/virtio/vhost.c +7 -2
index b08290036d..624ade9682 100644
@@ -386,7 +386,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,
     return r;
 }
 
-static bool vhost_section(MemoryRegionSection *section)
+static bool vhost_section(struct vhost_dev *dev, MemoryRegionSection *section)
 {
     bool result;
     bool log_dirty = memory_region_get_dirty_log_mask(section->mr) &
@@ -399,6 +399,11 @@ static bool vhost_section(MemoryRegionSection *section)
      */
     result &= !log_dirty;
 
+    if (result && dev->vhost_ops->vhost_backend_mem_section_filter) {
+        result &=
+            dev->vhost_ops->vhost_backend_mem_section_filter(dev, section);
+    }
+
     trace_vhost_section(section->mr->name, result);
     return result;
 }
@@ -632,7 +637,7 @@ static void vhost_region_addnop(MemoryListener *listener,
     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
                                          memory_listener);
 
-    if (!vhost_section(section)) {
+    if (!vhost_section(dev, section)) {
         return;
     }
     vhost_region_add_section(dev, section);
include/hw/virtio/vhost-backend.h +4 -0
index 5dac61f9ea..81283ec50f 100644
@@ -101,6 +101,9 @@ typedef int (*vhost_crypto_create_session_op)(struct vhost_dev *dev,
 typedef int (*vhost_crypto_close_session_op)(struct vhost_dev *dev,
                                              uint64_t session_id);
 
+typedef bool (*vhost_backend_mem_section_filter_op)(struct vhost_dev *dev,
+                                                MemoryRegionSection *section);
+
 typedef struct VhostOps {
     VhostBackendType backend_type;
     vhost_backend_init vhost_backend_init;
@@ -138,6 +141,7 @@ typedef struct VhostOps {
     vhost_set_config_op vhost_set_config;
     vhost_crypto_create_session_op vhost_crypto_create_session;
     vhost_crypto_close_session_op vhost_crypto_close_session;
+    vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
 } VhostOps;
 
 extern const VhostOps user_ops;
Signature · 1 line
2.17.0
author
#2 lore raw

PATCH v2 2/5 vhost-user: introduce shared vhost-user state

In reply to Tiwei Bie

When multi queue is enabled e.g. for a virtio-net device, each queue pair will have a vhost_dev, and the only thing shared between vhost devs currently is the chardev. This patch introduces a vhost-user state structure which will be shared by all vhost devs of the same virtio device.

---
 backends/cryptodev-vhost-user.c     | 20 +++++++++-
 hw/block/vhost-user-blk.c           | 22 ++++++++++-
 hw/scsi/vhost-user-scsi.c           | 20 +++++++++-
 hw/virtio/Makefile.objs             |  2 +-
 hw/virtio/vhost-stub.c              | 10 +++++
 hw/virtio/vhost-user.c              | 31 +++++++++++-----
 include/hw/virtio/vhost-user-blk.h  |  2 +
 include/hw/virtio/vhost-user-scsi.h |  2 +
 include/hw/virtio/vhost-user.h      | 20 ++++++++++
 net/vhost-user.c                    | 57 +++++++++++++++++++++--------
 10 files changed, 158 insertions(+), 28 deletions(-)
 create mode 100644 include/hw/virtio/vhost-user.h
backends/cryptodev-vhost-user.c +19 -1
index 862d4f2580..d52daccfcd 100644
@@ -26,6 +26,7 @@
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/error-report.h"
+#include "hw/virtio/vhost-user.h"
 #include "standard-headers/linux/virtio_crypto.h"
 #include "sysemu/cryptodev-vhost.h"
 #include "chardev/char-fe.h"
@@ -46,6 +47,7 @@
 typedef struct CryptoDevBackendVhostUser {
     CryptoDevBackend parent_obj;
 
+    VhostUserState *vhost_user;
     CharBackend chr;
     char *chr_name;
     bool opened;
@@ -102,7 +104,7 @@ cryptodev_vhost_user_start(int queues,
             continue;
         }
 
-        options.opaque = &s->chr;
+        options.opaque = s->vhost_user;
         options.backend_type = VHOST_BACKEND_TYPE_USER;
         options.cc = b->conf.peers.ccs[i];
         s->vhost_crypto[i] = cryptodev_vhost_init(&options);
@@ -185,6 +187,7 @@ static void cryptodev_vhost_user_init(
     size_t i;
     Error *local_err = NULL;
     Chardev *chr;
+    VhostUserState *user;
     CryptoDevBackendClient *cc;
     CryptoDevBackendVhostUser *s =
                       CRYPTODEV_BACKEND_VHOST_USER(backend);
@@ -215,6 +218,15 @@ static void cryptodev_vhost_user_init(
         }
     }
 
+    user = vhost_user_init();
+    if (!user) {
+        error_setg(errp, "Failed to init vhost_user");
+        return;
+    }
+
+    user->chr = &s->chr;
+    s->vhost_user = user;
+
     qemu_chr_fe_set_handlers(&s->chr, NULL, NULL,
                      cryptodev_vhost_user_event, NULL, s, NULL, true);
 
@@ -299,6 +311,12 @@ static void cryptodev_vhost_user_cleanup(
             backend->conf.peers.ccs[i] = NULL;
         }
     }
+
+    if (s->vhost_user) {
+        vhost_user_cleanup(s->vhost_user);
+        g_free(s->vhost_user);
+        s->vhost_user = NULL;
+    }
 }
 
 static void cryptodev_vhost_user_set_chardev(Object *obj,
hw/block/vhost-user-blk.c +21 -1
index 975eae6211..7c3fa8bb1c 100644
@@ -226,6 +226,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VHostUserBlk *s = VHOST_USER_BLK(vdev);
+    VhostUserState *user;
     int i, ret;
 
     if (!s->chardev.chr) {
@@ -243,6 +244,15 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    user = vhost_user_init();
+    if (!user) {
+        error_setg(errp, "vhost-user-blk: failed to init vhost_user");
+        return;
+    }
+
+    user->chr = &s->chardev;
+    s->vhost_user = user;
+
     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
                 sizeof(struct virtio_blk_config));
 
@@ -258,7 +268,7 @@ static void vhost_user_blk_device_realize(DeviceState *dev, Error **errp)
 
     vhost_dev_set_config_notifier(&s->dev, &blk_ops);
 
-    ret = vhost_dev_init(&s->dev, &s->chardev, VHOST_BACKEND_TYPE_USER, 0);
+    ret = vhost_dev_init(&s->dev, s->vhost_user, VHOST_BACKEND_TYPE_USER, 0);
     if (ret < 0) {
         error_setg(errp, "vhost-user-blk: vhost initialization failed: %s",
                    strerror(-ret));
@@ -283,6 +293,10 @@ vhost_err:
 virtio_err:
     g_free(s->dev.vqs);
     virtio_cleanup(vdev);
+
+    vhost_user_cleanup(user);
+    g_free(user);
+    s->vhost_user = NULL;
 }
 
 static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp)
@@ -294,6 +308,12 @@ static void vhost_user_blk_device_unrealize(DeviceState *dev, Error **errp)
     vhost_dev_cleanup(&s->dev);
     g_free(s->dev.vqs);
     virtio_cleanup(vdev);
+
+    if (s->vhost_user) {
+        vhost_user_cleanup(s->vhost_user);
+        g_free(s->vhost_user);
+        s->vhost_user = NULL;
+    }
 }
 
 static void vhost_user_blk_instance_init(Object *obj)
hw/scsi/vhost-user-scsi.c +19 -1
index 9389ed48e0..9355cfdf07 100644
@@ -69,6 +69,7 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
     VHostUserSCSI *s = VHOST_USER_SCSI(dev);
     VHostSCSICommon *vsc = VHOST_SCSI_COMMON(s);
+    VhostUserState *user;
     Error *err = NULL;
     int ret;
 
@@ -85,19 +86,30 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error **errp)
         return;
     }
 
+    user = vhost_user_init();
+    if (!user) {
+        error_setg(errp, "vhost-user-scsi: failed to init vhost_user");
+        return;
+    }
+    user->chr = &vs->conf.chardev;
+
     vsc->dev.nvqs = 2 + vs->conf.num_queues;
     vsc->dev.vqs = g_new(struct vhost_virtqueue, vsc->dev.nvqs);
     vsc->dev.vq_index = 0;
     vsc->dev.backend_features = 0;
 
-    ret = vhost_dev_init(&vsc->dev, (void *)&vs->conf.chardev,
+    ret = vhost_dev_init(&vsc->dev, user,
                          VHOST_BACKEND_TYPE_USER, 0);
     if (ret < 0) {
         error_setg(errp, "vhost-user-scsi: vhost initialization failed: %s",
                    strerror(-ret));
+        vhost_user_cleanup(user);
+        g_free(user);
         return;
     }
 
+    s->vhost_user = user;
+
     /* Channel and lun both are 0 for bootable vhost-user-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -117,6 +129,12 @@ static void vhost_user_scsi_unrealize(DeviceState *dev, Error **errp)
     g_free(vsc->dev.vqs);
 
     virtio_scsi_common_unrealize(dev, errp);
+
+    if (s->vhost_user) {
+        vhost_user_cleanup(s->vhost_user);
+        g_free(s->vhost_user);
+        s->vhost_user = NULL;
+    }
 }
 
 static uint64_t vhost_user_scsi_get_features(VirtIODevice *vdev,
hw/virtio/Makefile.objs +1 -1
index 765d363c1f..030969e28c 100644
@@ -11,5 +11,5 @@ obj-y += virtio-crypto.o
 obj-$(CONFIG_VIRTIO_PCI) += virtio-crypto-pci.o
 endif
 
-common-obj-$(call lnot,$(CONFIG_LINUX)) += vhost-stub.o
+common-obj-$(call lnot,$(call land,$(CONFIG_VIRTIO),$(CONFIG_LINUX))) += vhost-stub.o
 common-obj-$(CONFIG_ALL) += vhost-stub.o
hw/virtio/vhost-stub.c +10 -0
index 2d76cdebdc..049089b5e2 100644
@@ -1,7 +1,17 @@
 #include "qemu/osdep.h"
 #include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
 
 bool vhost_has_free_slot(void)
 {
     return true;
 }
+
+VhostUserState *vhost_user_init(void)
+{
+    return NULL;
+}
+
+void vhost_user_cleanup(VhostUserState *user)
+{
+}
hw/virtio/vhost-user.c +22 -9
index da0756effe..5f72ed68da 100644
@@ -11,6 +11,7 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
 #include "hw/virtio/vhost-backend.h"
 #include "hw/virtio/virtio-net.h"
 #include "chardev/char-fe.h"
@@ -173,7 +174,8 @@ static VhostUserMsg m __attribute__ ((unused));
 
 struct vhost_user {
     struct vhost_dev *dev;
-    CharBackend *chr;
+    /* Shared between vhost devs of the same virtio device */
+    VhostUserState *user;
     int slave_fd;
     NotifierWithReturn postcopy_notifier;
     struct PostCopyFD  postcopy_fd;
@@ -199,7 +201,7 @@ static bool ioeventfd_enabled(void)
 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 {
     struct vhost_user *u = dev->opaque;
-    CharBackend *chr = u->chr;
+    CharBackend *chr = u->user->chr;
     uint8_t *p = (uint8_t *) msg;
     int r, size = VHOST_USER_HDR_SIZE;
 
@@ -285,7 +287,7 @@ static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
                             int *fds, int fd_num)
 {
     struct vhost_user *u = dev->opaque;
-    CharBackend *chr = u->chr;
+    CharBackend *chr = u->user->chr;
     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
 
     /*
@@ -1083,7 +1085,7 @@ static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
 {
     struct vhost_user *u = dev->opaque;
-    CharBackend *chr = u->chr;
+    CharBackend *chr = u->user->chr;
     int ufd;
     VhostUserMsg msg = {
         .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
@@ -1221,7 +1223,7 @@ static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
     return 0;
 }
 
-static int vhost_user_init(struct vhost_dev *dev, void *opaque)
+static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
 {
     uint64_t features, protocol_features;
     struct vhost_user *u;
@@ -1230,7 +1232,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 
     u = g_new0(struct vhost_user, 1);
-    u->chr = opaque;
+    u->user = opaque;
     u->slave_fd = -1;
     u->dev = dev;
     dev->opaque = u;
@@ -1306,7 +1308,7 @@ static int vhost_user_init(struct vhost_dev *dev, void *opaque)
     return 0;
 }
 
-static int vhost_user_cleanup(struct vhost_dev *dev)
+static int vhost_user_backend_cleanup(struct vhost_dev *dev)
 {
     struct vhost_user *u;
 
@@ -1630,10 +1632,21 @@ static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
     return result;
 }
 
+VhostUserState *vhost_user_init(void)
+{
+    VhostUserState *user = g_new0(struct VhostUserState, 1);
+
+    return user;
+}
+
+void vhost_user_cleanup(VhostUserState *user)
+{
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
-        .vhost_backend_init = vhost_user_init,
-        .vhost_backend_cleanup = vhost_user_cleanup,
+        .vhost_backend_init = vhost_user_backend_init,
+        .vhost_backend_cleanup = vhost_user_backend_cleanup,
         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
         .vhost_set_log_base = vhost_user_set_log_base,
         .vhost_set_mem_table = vhost_user_set_mem_table,
include/hw/virtio/vhost-user-blk.h +2 -0
index 5804cc904a..f1258ae545 100644
@@ -21,6 +21,7 @@
 #include "hw/block/block.h"
 #include "chardev/char-fe.h"
 #include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
 
 #define TYPE_VHOST_USER_BLK "vhost-user-blk"
 #define VHOST_USER_BLK(obj) \
@@ -36,6 +37,7 @@ typedef struct VHostUserBlk {
     uint32_t config_wce;
     uint32_t config_ro;
     struct vhost_dev dev;
+    VhostUserState *vhost_user;
 } VHostUserBlk;
 
 #endif
include/hw/virtio/vhost-user-scsi.h +2 -0
index 01861f78d0..3ec34ae867 100644
@@ -21,6 +21,7 @@
 #include "hw/qdev.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-user.h"
 #include "hw/virtio/vhost-scsi-common.h"
 
 #define TYPE_VHOST_USER_SCSI "vhost-user-scsi"
@@ -30,6 +31,7 @@
 typedef struct VHostUserSCSI {
     VHostSCSICommon parent_obj;
     uint64_t host_features;
+    VhostUserState *vhost_user;
 } VHostUserSCSI;
 
 #endif /* VHOST_USER_SCSI_H */
include/hw/virtio/vhost-user.h new file +20 -0
new file mode 100644
index 0000000000..eb8bc0d90d
@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2017-2018 Intel Corporation
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef HW_VIRTIO_VHOST_USER_H
+#define HW_VIRTIO_VHOST_USER_H
+
+#include "chardev/char-fe.h"
+
+typedef struct VhostUserState {
+    CharBackend *chr;
+} VhostUserState;
+
+VhostUserState *vhost_user_init(void);
+void vhost_user_cleanup(VhostUserState *user);
+
+#endif
net/vhost-user.c +42 -15

129 lines not on this page · raw message

Signature · 1 line
2.17.0
author
#3 lore raw

PATCH v2 3/5 vhost-user: support registering external host notifiers

In reply to Tiwei Bie

This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER. With this feature negotiated, vhost-user backend can register memory region based host notifiers. And it will allow the guest driver in the VM to notify the hardware accelerator at the vhost-user backend directly.

---
 docs/interop/vhost-user.txt    |  33 +++++++++
 hw/virtio/vhost-user.c         | 123 +++++++++++++++++++++++++++++++++
 include/hw/virtio/vhost-user.h |   8 +++
 3 files changed, 164 insertions(+)
docs/interop/vhost-user.txt +33 -0
index 534caab18a..9e57b36b20 100644
@@ -132,6 +132,16 @@ Depending on the request type, payload can be:
    Payload: Size bytes array holding the contents of the virtio
        device's configuration space
 
+ * Vring area description
+   -----------------------
+   | u64 | size | offset |
+   -----------------------
+
+   u64: a 64-bit integer contains vring index and flags
+   Size: a 64-bit size of this area
+   Offset: a 64-bit offset of this area from the start of the
+       supplied file descriptor
+
 In QEMU the vhost-user message is implemented with the following struct:
 
 typedef struct VhostUserMsg {
@@ -146,6 +156,7 @@ typedef struct VhostUserMsg {
         VhostUserLog log;
         struct vhost_iotlb_msg iotlb;
         VhostUserConfig config;
+        VhostUserVringArea area;
     };
 } QEMU_PACKED VhostUserMsg;
 
@@ -380,6 +391,7 @@ Protocol features
 #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7
 #define VHOST_USER_PROTOCOL_F_PAGEFAULT      8
 #define VHOST_USER_PROTOCOL_F_CONFIG         9
+#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER  10
 
 Master message types
 --------------------
@@ -777,6 +789,27 @@ Slave message types
      the VHOST_USER_NEED_REPLY flag, master must respond with zero when
      operation is successfully completed, or non-zero otherwise.
 
+ * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
+
+      Id: 3
+      Equivalent ioctl: N/A
+      Slave payload: vring area description
+      Master payload: N/A
+
+      Sets host notifier for a specified queue. The queue index is contained
+      in the u64 field of the vring area description. The host notifier is
+      described by the file descriptor (typically it's a VFIO device fd) which
+      is passed as ancillary data and the size (which is mmap size and should
+      be the same as host page size) and offset (which is mmap offset) carried
+      in the vring area description. QEMU can mmap the file descriptor based
+      on the size and offset to get a memory range. Registering a host notifier
+      means mapping this memory range to the VM as the specified queue's notify
+      MMIO region. Slave sends this request to tell QEMU to de-register the
+      existing notifier if any and register the new notifier if the request is
+      sent with a file descriptor.
+      This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
+      protocol feature has been successfully negotiated.
+
 VHOST_USER_PROTOCOL_F_REPLY_ACK:
 -------------------------------
 The original vhost-user specification only demands replies for certain
hw/virtio/vhost-user.c +123 -0

186 lines not on this page · raw message

include/hw/virtio/vhost-user.h +8 -0
index eb8bc0d90d..fd660393a0 100644
@@ -9,9 +9,17 @@
 #define HW_VIRTIO_VHOST_USER_H
 
 #include "chardev/char-fe.h"
+#include "hw/virtio/virtio.h"
+
+typedef struct VhostUserHostNotifier {
+    MemoryRegion mr;
+    void *addr;
+    bool set;
+} VhostUserHostNotifier;
 
 typedef struct VhostUserState {
     CharBackend *chr;
+    VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
 } VhostUserState;
 
 VhostUserState *vhost_user_init(void);
Signature · 1 line
2.17.0
#4 lore raw

Re: [Qemu-devel] [PATCH v2 3/5] vhost-user: support registering external host notifiers

Quoted 209 lines · Tiwei Bie

This patch introduces VHOST_USER_PROTOCOL_F_HOST_NOTIFIER. With this feature negotiated, vhost-user backend can register memory region based host notifiers. And it will allow the guest driver in the VM to notify the hardware accelerator at the vhost-user backend directly.

---
  docs/interop/vhost-user.txt    |  33 +++++++++
  hw/virtio/vhost-user.c         | 123 +++++++++++++++++++++++++++++++++
  include/hw/virtio/vhost-user.h |   8 +++
  3 files changed, 164 insertions(+)
docs/interop/vhost-user.txt +33 -0
index 534caab18a..9e57b36b20 100644
@@ -132,6 +132,16 @@ Depending on the request type, payload can be:
     Payload: Size bytes array holding the contents of the virtio
         device's configuration space
 
+ * Vring area description
+   -----------------------
+   | u64 | size | offset |
+   -----------------------
+
+   u64: a 64-bit integer contains vring index and flags
+   Size: a 64-bit size of this area
+   Offset: a 64-bit offset of this area from the start of the
+       supplied file descriptor
+
  In QEMU the vhost-user message is implemented with the following struct:
 
  typedef struct VhostUserMsg {
@@ -146,6 +156,7 @@ typedef struct VhostUserMsg {
          VhostUserLog log;
          struct vhost_iotlb_msg iotlb;
          VhostUserConfig config;
+        VhostUserVringArea area;
      };
  } QEMU_PACKED VhostUserMsg;
 
@@ -380,6 +391,7 @@ Protocol features
  #define VHOST_USER_PROTOCOL_F_CRYPTO_SESSION 7
  #define VHOST_USER_PROTOCOL_F_PAGEFAULT      8
  #define VHOST_USER_PROTOCOL_F_CONFIG         9
+#define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER  10
 
  Master message types
  --------------------
@@ -777,6 +789,27 @@ Slave message types
       the VHOST_USER_NEED_REPLY flag, master must respond with zero when
       operation is successfully completed, or non-zero otherwise.
 
+ * VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
+
+      Id: 3
+      Equivalent ioctl: N/A
+      Slave payload: vring area description
+      Master payload: N/A
+
+      Sets host notifier for a specified queue. The queue index is contained
+      in the u64 field of the vring area description. The host notifier is
+      described by the file descriptor (typically it's a VFIO device fd) which
+      is passed as ancillary data and the size (which is mmap size and should
+      be the same as host page size) and offset (which is mmap offset) carried
+      in the vring area description. QEMU can mmap the file descriptor based
+      on the size and offset to get a memory range. Registering a host notifier
+      means mapping this memory range to the VM as the specified queue's notify
+      MMIO region. Slave sends this request to tell QEMU to de-register the
+      existing notifier if any and register the new notifier if the request is
+      sent with a file descriptor.
+      This request should be sent only when VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
+      protocol feature has been successfully negotiated.
+
  VHOST_USER_PROTOCOL_F_REPLY_ACK:
  -------------------------------
  The original vhost-user specification only demands replies for certain
hw/virtio/vhost-user.c +71 -0
index 5f72ed68da..259a841af9 100644
@@ -13,6 +13,7 @@
  #include "hw/virtio/vhost.h"
  #include "hw/virtio/vhost-user.h"
  #include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio.h"
  #include "hw/virtio/virtio-net.h"
  #include "chardev/char-fe.h"
  #include "sysemu/kvm.h"
@@ -48,6 +49,7 @@ enum VhostUserProtocolFeature {
      VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
      VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
      VHOST_USER_PROTOCOL_F_CONFIG = 9,
+    VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 10,
      VHOST_USER_PROTOCOL_F_MAX
  };
 
@@ -92,6 +94,7 @@ typedef enum VhostUserSlaveRequest {
      VHOST_USER_SLAVE_NONE = 0,
      VHOST_USER_SLAVE_IOTLB_MSG = 1,
      VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
+    VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
      VHOST_USER_SLAVE_MAX
  }  VhostUserSlaveRequest;
 
@@ -136,6 +139,12 @@ static VhostUserConfig c __attribute__ ((unused));
                                     + sizeof(c.size) \
                                     + sizeof(c.flags))
 
+typedef struct VhostUserVringArea {
+    uint64_t u64;
+    uint64_t size;
+    uint64_t offset;
+} VhostUserVringArea;
+
  typedef struct {
      VhostUserRequest request;
 
@@ -157,6 +166,7 @@ typedef union {
          struct vhost_iotlb_msg iotlb;
          VhostUserConfig config;
          VhostUserCryptoSession session;
+        VhostUserVringArea area;
  } VhostUserPayload;
 
  typedef struct VhostUserMsg {
@@ -638,9 +648,37 @@ static int vhost_user_set_vring_num(struct vhost_dev *dev,
      return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
  }
 
+static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
+                                             int queue_idx)
+{
+    struct vhost_user *u = dev->opaque;
+    VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
+    VirtIODevice *vdev = dev->vdev;
+
+    if (n->addr && !n->set) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
+        n->set = true;
+    }
+}
+
+static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
+                                            int queue_idx)
+{
+    struct vhost_user *u = dev->opaque;
+    VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
+    VirtIODevice *vdev = dev->vdev;
+
+    if (n->addr && n->set) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
+        n->set = false;
+    }
+}
+
  static int vhost_user_set_vring_base(struct vhost_dev *dev,
                                       struct vhost_vring_state *ring)
  {
+    vhost_user_host_notifier_restore(dev, ring->index);
+
      return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
  }
 
@@ -674,6 +712,8 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
          .hdr.size = sizeof(msg.payload.state),
      };
 
+    vhost_user_host_notifier_remove(dev, ring->index);
+
      if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
          return -1;
      }
@@ -847,6 +887,76 @@ static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
      return ret;
  }
 
+static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
+                                                       VhostUserVringArea *area,
+                                                       int fd)
+{
+    int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_user *u = dev->opaque;
+    VhostUserState *user = u->user;
+    VirtIODevice *vdev = dev->vdev;
+    VhostUserHostNotifier *n;
+    int ret = 0;
+    void *addr;
+    char *name;
+
+    if (!virtio_has_feature(dev->protocol_features,
+                            VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
+        vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
+        ret = -1;
+        goto out;
+    }
+
+    n = &user->notifier[queue_idx];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+
+    if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {

hw/virtio/vhost-user.c static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) · line 930

Quoted 10 lines
hw/virtio/vhost-user.c +10 -0
+        goto out;
+    }
+
+    /* Sanity check. */
+    if (area->size != page_size) {
+        ret = -1;
+        goto out;
+    }
+
+    addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,

It looks to me that PROT_WRITE is sufficient here?

Thanks

Quoted 81 lines
hw/virtio/vhost-user.c +42 -0
+                fd, area->offset);
+    if (addr == MAP_FAILED) {
+        ret = -1;
+        goto out;
+    }
+
+    name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
+                           user, queue_idx);
+    memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+                                      page_size, addr);
+    g_free(name);
+
+    if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
+        munmap(addr, page_size);
+        ret = -1;
+        goto out;
+    }
+
+    n->addr = addr;
+    n->set = true;
+
+out:
+    /* Always close the fd. */
+    if (fd != -1) {
+        close(fd);
+    }
+    return ret;
+}
+
  static void slave_read(void *opaque)
  {
      struct vhost_dev *dev = opaque;
@@ -913,6 +1023,10 @@ static void slave_read(void *opaque)
      case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
          ret = vhost_user_slave_handle_config_change(dev);
          break;
+    case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
+        ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
+                                                          fd);
+        break;
      default:
          error_report("Received unexpected msg type.");
          if (fd != -1) {
@@ -1641,6 +1755,15 @@ VhostUserState *vhost_user_init(void)
 
  void vhost_user_cleanup(VhostUserState *user)
  {
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        if (user->notifier[i].addr) {
+            object_unparent(OBJECT(&user->notifier[i].mr));
+            munmap(user->notifier[i].addr, qemu_real_host_page_size);
+            user->notifier[i].addr = NULL;
+        }
+    }
  }
 
  const VhostOps user_ops = {
include/hw/virtio/vhost-user.h +8 -0
index eb8bc0d90d..fd660393a0 100644
@@ -9,9 +9,17 @@
  #define HW_VIRTIO_VHOST_USER_H
 
  #include "chardev/char-fe.h"
+#include "hw/virtio/virtio.h"
+
+typedef struct VhostUserHostNotifier {
+    MemoryRegion mr;
+    void *addr;
+    bool set;
+} VhostUserHostNotifier;
 
  typedef struct VhostUserState {
      CharBackend *chr;
+    VhostUserHostNotifier notifier[VIRTIO_QUEUE_MAX];
  } VhostUserState;
 
  VhostUserState *vhost_user_init(void);
author
#5 lore raw

Re: [Qemu-devel] [PATCH v2 3/5] vhost-user: support registering external host notifiers

Quoted 1 line · Jason Wang

On 2018年05月25日 19:04, Tiwei Bie wrote:

[...]

Quoted 44 lines
Quoted 31 lines
hw/virtio/vhost-user.c +31 -0
+static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
+                                                       VhostUserVringArea *area,
+                                                       int fd)
+{
+    int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_user *u = dev->opaque;
+    VhostUserState *user = u->user;
+    VirtIODevice *vdev = dev->vdev;
+    VhostUserHostNotifier *n;
+    int ret = 0;
+    void *addr;
+    char *name;
+
+    if (!virtio_has_feature(dev->protocol_features,
+                            VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
+        vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
+        ret = -1;
+        goto out;
+    }
+
+    n = &user->notifier[queue_idx];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+
+    if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {

hw/virtio/vhost-user.c static int vhost_user_slave_handle_config_change(struct vhost_dev *dev) · line 930

Quoted 10 lines
hw/virtio/vhost-user.c +10 -0
+        goto out;
+    }
+
+    /* Sanity check. */
+    if (area->size != page_size) {
+        ret = -1;
+        goto out;
+    }
+
+    addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,

It looks to me that PROT_WRITE is sufficient here?

Quoted 2 lines

Thanks

I think it's safer to add PROT_READ. And there's
no harm to do that.

Best regards,
Tiwei Bie

author
#6 lore raw

PATCH v2 4/5 libvhost-user: support host notifier

In reply to Tiwei Bie

This patch introduces the host notifier support in libvhost-user. A new API is added to support setting host notifier for each queue.

---
 contrib/libvhost-user/libvhost-user.c | 80 ++++++++++++++++++++++++---
 contrib/libvhost-user/libvhost-user.h | 31 +++++++++++
 2 files changed, 103 insertions(+), 8 deletions(-)
contrib/libvhost-user/libvhost-user.c +72 -8
index 54e643d871..6b0cdfbe8e 100644
@@ -314,11 +314,6 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
         msg.msg_controllen = 0;
     }
 
-    /* Set the version in the flags when sending the reply */
-    vmsg->flags &= ~VHOST_USER_VERSION_MASK;
-    vmsg->flags |= VHOST_USER_VERSION;
-    vmsg->flags |= VHOST_USER_REPLY_MASK;
-
     do {
         rc = sendmsg(conn_fd, &msg, 0);
     } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
@@ -341,6 +336,39 @@ vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
     return true;
 }
 
+static bool
+vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    /* Set the version in the flags when sending the reply */
+    vmsg->flags &= ~VHOST_USER_VERSION_MASK;
+    vmsg->flags |= VHOST_USER_VERSION;
+    vmsg->flags |= VHOST_USER_REPLY_MASK;
+
+    return vu_message_write(dev, conn_fd, vmsg);
+}
+
+static bool
+vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
+{
+    VhostUserMsg msg_reply;
+
+    if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
+        return true;
+    }
+
+    if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {
+        return false;
+    }
+
+    if (msg_reply.request != vmsg->request) {
+        DPRINT("Received unexpected msg type. Expected %d received %d",
+               vmsg->request, msg_reply.request);
+        return false;
+    }
+
+    return msg_reply.payload.u64 == 0;
+}
+
 /* Kick the log_call_fd if required. */
 static void
 vu_log_kick(VuDev *dev)
@@ -536,7 +564,7 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)
 
     /* Send the message back to qemu with the addresses filled in */
     vmsg->fd_num = 0;
-    if (!vu_message_write(dev, dev->sock, vmsg)) {
+    if (!vu_send_reply(dev, dev->sock, vmsg)) {
         vu_panic(dev, "failed to respond to set-mem-table for postcopy");
         return false;
     }
@@ -916,6 +944,41 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
     }
 }
 
+bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
+                                int size, int offset)
+{
+    int qidx = vq - dev->vq;
+    int fd_num = 0;
+    VhostUserMsg vmsg = {
+        .request = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
+        .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
+        .size = sizeof(vmsg.payload.area),
+        .payload.area = {
+            .u64 = qidx & VHOST_USER_VRING_IDX_MASK,
+            .size = size,
+            .offset = offset,
+        },
+    };
+
+    if ((dev->protocol_features & VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) == 0) {
+        return false;
+    }
+
+    if (fd == -1) {
+        vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
+    } else {
+        vmsg.fds[fd_num++] = fd;
+    }
+
+    vmsg.fd_num = fd_num;
+
+    if (!vu_message_write(dev, dev->slave_fd, &vmsg)) {
+        return false;
+    }
+
+    return vu_process_message_reply(dev, &vmsg);
+}
+
 static bool
 vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
 {
@@ -968,7 +1031,8 @@ static bool
 vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
 {
     uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
-                        1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ;
+                        1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ |
+                        1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER;
 
     if (have_userfault()) {
         features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
@@ -1252,7 +1316,7 @@ vu_dispatch(VuDev *dev)
         goto end;
     }
 
-    if (!vu_message_write(dev, dev->sock, &vmsg)) {
+    if (!vu_send_reply(dev, dev->sock, &vmsg)) {
         goto end;
     }
 
contrib/libvhost-user/libvhost-user.h +31 -0
index b27075ea3b..90806a25f9 100644
@@ -51,6 +51,7 @@ enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
     VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
     VHOST_USER_PROTOCOL_F_CONFIG = 9,
+    VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 10,
 
     VHOST_USER_PROTOCOL_F_MAX
 };
@@ -92,6 +93,14 @@ typedef enum VhostUserRequest {
     VHOST_USER_MAX
 } VhostUserRequest;
 
+typedef enum VhostUserSlaveRequest {
+    VHOST_USER_SLAVE_NONE = 0,
+    VHOST_USER_SLAVE_IOTLB_MSG = 1,
+    VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
+    VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
+    VHOST_USER_SLAVE_MAX
+}  VhostUserSlaveRequest;
+
 typedef struct VhostUserMemoryRegion {
     uint64_t guest_phys_addr;
     uint64_t memory_size;
@@ -122,6 +131,12 @@ static VhostUserConfig c __attribute__ ((unused));
                                    + sizeof(c.size) \
                                    + sizeof(c.flags))
 
+typedef struct VhostUserVringArea {
+    uint64_t u64;
+    uint64_t size;
+    uint64_t offset;
+} VhostUserVringArea;
+
 #if defined(_WIN32)
 # define VU_PACKED __attribute__((gcc_struct, packed))
 #else
@@ -133,6 +148,7 @@ typedef struct VhostUserMsg {
 
 #define VHOST_USER_VERSION_MASK     (0x3)
 #define VHOST_USER_REPLY_MASK       (0x1 << 2)
+#define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
     uint32_t flags;
     uint32_t size; /* the following payload size */
 
@@ -145,6 +161,7 @@ typedef struct VhostUserMsg {
         VhostUserMemory memory;
         VhostUserLog log;
         VhostUserConfig config;
+        VhostUserVringArea area;
     } payload;
 
     int fds[VHOST_MEMORY_MAX_NREGIONS];
@@ -368,6 +385,20 @@ VuVirtq *vu_get_queue(VuDev *dev, int qidx);
 void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
                           vu_queue_handler_cb handler);
 
+/**
+ * vu_set_queue_host_notifier:
+ * @dev: a VuDev context
+ * @vq: a VuVirtq queue
+ * @fd: a file descriptor
+ * @size: host page size
+ * @offset: notifier offset in @fd file
+ *
+ * Set queue's host notifier. This function may be called several
+ * times for the same queue. If called with -1 @fd, the notifier
+ * is removed.
+ */
+bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
+                                int size, int offset);
 
 /**
  * vu_queue_set_notification:
Signature · 1 line
2.17.0
author
#7 lore raw

PATCH v2 5/5 vhost-user-bridge: support host notifier

In reply to Tiwei Bie

This patch introduces the host notifier support in vhost-user-bridge. A new option (-H) is added to use the host notifier. This is mainly used to test the host notifier implementation in vhost user.

---
 tests/vhost-user-bridge.c | 98 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 95 insertions(+), 3 deletions(-)
tests/vhost-user-bridge.c +95 -3
index e0605a529e..0884294141 100644
@@ -29,6 +29,7 @@
 
 #define _FILE_OFFSET_BITS 64
 
+#include "qemu/atomic.h"
 #include "qemu/osdep.h"
 #include "qemu/iov.h"
 #include "standard-headers/linux/virtio_net.h"
@@ -65,6 +66,11 @@ typedef struct VubrDev {
     int sock;
     int ready;
     int quit;
+    struct {
+        int fd;
+        void *addr;
+        pthread_t thread;
+    } notifier;
 } VubrDev;
 
 static void
@@ -445,14 +451,22 @@ static uint64_t
 vubr_get_features(VuDev *dev)
 {
     return 1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE |
-        1ULL << VIRTIO_NET_F_MRG_RXBUF;
+        1ULL << VIRTIO_NET_F_MRG_RXBUF |
+        1ULL << VIRTIO_F_VERSION_1;
 }
 
 static void
 vubr_queue_set_started(VuDev *dev, int qidx, bool started)
 {
+    VubrDev *vubr = container_of(dev, VubrDev, vudev);
     VuVirtq *vq = vu_get_queue(dev, qidx);
 
+    if (started && vubr->notifier.fd >= 0) {
+        vu_set_queue_host_notifier(dev, vq, vubr->notifier.fd,
+                                   getpagesize(),
+                                   qidx * getpagesize());
+    }
+
     if (qidx % 2 == 1) {
         vu_set_queue_handler(dev, vq, started ? vubr_handle_tx : NULL);
     }
@@ -522,6 +536,8 @@ vubr_new(const char *path, bool client)
         vubr_die("socket");
     }
 
+    dev->notifier.fd = -1;
+
     un.sun_family = AF_UNIX;
     strcpy(un.sun_path, path);
     len = sizeof(un.sun_family) + strlen(path);
@@ -559,6 +575,73 @@ vubr_new(const char *path, bool client)
     return dev;
 }
 
+static void *notifier_thread(void *arg)
+{
+    VuDev *dev = (VuDev *)arg;
+    VubrDev *vubr = container_of(dev, VubrDev, vudev);
+    int pagesize = getpagesize();
+    int qidx;
+
+    while (true) {
+        for (qidx = 0; qidx < VHOST_MAX_NR_VIRTQUEUE; qidx++) {
+            uint16_t *n = vubr->notifier.addr + pagesize * qidx;
+
+            if (*n == qidx) {
+                *n = 0xffff;
+                /* We won't miss notifications if we reset
+                 * the memory first. */
+                smp_mb();
+
+                DPRINT("Got a notification for queue%d via host notifier.\n",
+                       qidx);
+
+                if (qidx % 2 == 1) {
+                    vubr_handle_tx(dev, qidx);
+                }
+            }
+            usleep(1000);
+        }
+    }
+
+    return NULL;
+}
+
+static void
+vubr_host_notifier_setup(VubrDev *dev)
+{
+    char template[] = "/tmp/vubr-XXXXXX";
+    pthread_t thread;
+    size_t length;
+    void *addr;
+    int fd;
+
+    length = getpagesize() * VHOST_MAX_NR_VIRTQUEUE;
+
+    fd = mkstemp(template);
+    if (fd < 0) {
+        vubr_die("mkstemp()");
+    }
+
+    if (posix_fallocate(fd, 0, length) != 0) {
+        vubr_die("posix_fallocate()");
+    }
+
+    addr = mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (addr == MAP_FAILED) {
+        vubr_die("mmap()");
+    }
+
+    memset(addr, 0xff, length);
+
+    if (pthread_create(&thread, NULL, notifier_thread, &dev->vudev) != 0) {
+        vubr_die("pthread_create()");
+    }
+
+    dev->notifier.fd = fd;
+    dev->notifier.addr = addr;
+    dev->notifier.thread = thread;
+}
+
 static void
 vubr_set_host(struct sockaddr_in *saddr, const char *host)
 {
@@ -673,8 +756,9 @@ main(int argc, char *argv[])
     VubrDev *dev;
     int opt;
     bool client = false;
+    bool host_notifier = false;
 
-    while ((opt = getopt(argc, argv, "l:r:u:c")) != -1) {
+    while ((opt = getopt(argc, argv, "l:r:u:cH")) != -1) {
 
         switch (opt) {
         case 'l':
@@ -693,6 +777,9 @@ main(int argc, char *argv[])
         case 'c':
             client = true;
             break;
+        case 'H':
+            host_notifier = true;
+            break;
         default:
             goto out;
         }
@@ -708,6 +795,10 @@ main(int argc, char *argv[])
         return 1;
     }
 
+    if (host_notifier) {
+        vubr_host_notifier_setup(dev);
+    }
+
     vubr_backend_udp_setup(dev, lhost, lport, rhost, rport);
     vubr_run(dev);
 
@@ -717,7 +808,7 @@ main(int argc, char *argv[])
 
 out:
     fprintf(stderr, "Usage: %s ", argv[0]);
-    fprintf(stderr, "[-c] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
+    fprintf(stderr, "[-c] [-H] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
     fprintf(stderr, "\t-u path to unix doman socket. default: %s\n",
             DEFAULT_UD_SOCKET);
     fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
@@ -725,6 +816,7 @@ out:
     fprintf(stderr, "\t-r remote host and port. default: %s:%s\n",
             DEFAULT_RHOST, DEFAULT_RPORT);
     fprintf(stderr, "\t-c client mode\n");
+    fprintf(stderr, "\t-H use host notifier\n");
 
     return 1;
 }
Signature · 1 line
2.17.0