Skip to contentSkip to conversation

vhost-vDPA: doorbell mapping support

PATCH Superseded Jason Wang 3 participants 8 messages Apr 15, 2021 – May 6, 2021

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Jason Wang

author
#0 lore raw

cover vhost-vDPA: doorbell mapping support

Hi All:

This series tries to implement doorbell mapping support for vhost-vDPA. Tested with virtio-pci vDPA driver.

Please review.

Jason Wang (2):
vhost-vdpa: skip ram device from the IOTLB mapping
vhost-vdpa: doorbell mapping support

 hw/virtio/vhost-vdpa.c         | 97 ++++++++++++++++++++++++++++++----
 include/hw/virtio/vhost-vdpa.h |  7 +++
 2 files changed, 94 insertions(+), 10 deletions(-)
Signature · 1 line
2.25.1
author
#1 lore raw

PATCH 1/2 vhost-vdpa: skip ram device from the IOTLB mapping

vDPA is not tie to any specific hardware, for safety and simplicity, vhost-vDPA doesn't allow MMIO area to be mapped via IOTLB. Only the doorbell could be mapped via mmap(). So this patch exclude skip the ram device from the IOTLB mapping.

---
 hw/virtio/vhost-vdpa.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)
hw/virtio/vhost-vdpa.c +2 -10
index 01d2101d09..dd4321bac2 100644
@@ -27,6 +27,8 @@ static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
 {
     return (!memory_region_is_ram(section->mr) &&
             !memory_region_is_iommu(section->mr)) ||
+           /* vhost-vDPA doesn't allow MMIO to be mapped  */
+            memory_region_is_ram_device(section->mr) ||
            /*
             * Sizing an enabled 64-bit BAR can cause spurious mappings to
             * addresses in the upper part of the 64-bit address space.  These
@@ -171,22 +173,12 @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener,
                              vaddr, section->readonly);
     if (ret) {
         error_report("vhost vdpa map fail!");
-        if (memory_region_is_ram_device(section->mr)) {
-            /* Allow unexpected mappings not to be fatal for RAM devices */
-            error_report("map ram fail!");
-          return ;
-        }
         goto fail;
     }
 
     return;
 
 fail:
-    if (memory_region_is_ram_device(section->mr)) {
-        error_report("failed to vdpa_dma_map. pci p2p may not work");
-        return;
-
-    }
     /*
      * On the initfn path, store the first error in the container so we
      * can gracefully fail.  Runtime, there's not much we can do other
Signature · 1 line
2.25.1
author
#2 lore raw

PATCH 2/2 vhost-vdpa: doorbell mapping support

In reply to Jason Wang

This patch implements the doorbell mapping support for vhost-vDPA. This is simply done by using mmap()/munmap() for the vhost-vDPA fd during device start/stop. For the device without doorbell support, we fall back to eventfd based notification gracefully.

---
 hw/virtio/vhost-vdpa.c         | 85 ++++++++++++++++++++++++++++++++++
 include/hw/virtio/vhost-vdpa.h |  7 +++
 2 files changed, 92 insertions(+)
hw/virtio/vhost-vdpa.c +85 -0
index dd4321bac2..c3a3b7566f 100644
@@ -285,12 +285,95 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
     return 0;
 }
 
+static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
+                                            int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+
+    n = &v->notifier[queue_index];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+}
+
+static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
+{
+    int i;
+
+    for (i = 0; i < n; i++) {
+        vhost_vdpa_host_notifier_uninit(dev, i);
+    }
+}
+
+static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+    int fd = v->device_fd;
+    void *addr;
+    char *name;
+
+    vhost_vdpa_host_notifier_uninit(dev, queue_index);
+
+    n = &v->notifier[queue_index];
+
+    addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
+                queue_index * page_size);
+    if (addr == MAP_FAILED) {
+        goto err;
+    }
+
+    name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
+                           v, queue_index);
+    memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+                                      page_size, addr);
+    g_free(name);
+
+    if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
+        munmap(addr, page_size);
+        goto err;
+    }
+    n->addr = addr;
+
+    return 0;
+
+err:
+    return -1;
+}
+
+static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->nvqs; i++) {
+        if (vhost_vdpa_host_notifier_init(dev, i)) {
+            goto err;
+        }
+    }
+
+    return;
+
+err:
+    vhost_vdpa_host_notifiers_uninit(dev, i);
+    return;
+}
+
 static int vhost_vdpa_cleanup(struct vhost_dev *dev)
 {
     struct vhost_vdpa *v;
     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
     v = dev->opaque;
     trace_vhost_vdpa_cleanup(dev, v);
+    vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
     memory_listener_unregister(&v->listener);
 
     dev->opaque = NULL;
@@ -467,6 +550,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
     if (started) {
         uint8_t status = 0;
         memory_listener_register(&v->listener, &address_space_memory);
+        vhost_vdpa_host_notifiers_init(dev);
         vhost_vdpa_set_vring_ready(dev);
         vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
         vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
@@ -476,6 +560,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
         vhost_vdpa_reset_device(dev);
         vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
                                    VIRTIO_CONFIG_S_DRIVER);
+        vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
         memory_listener_unregister(&v->listener);
 
         return 0;
include/hw/virtio/vhost-vdpa.h +7 -0
index 9b81a409da..0f11ecff34 100644
@@ -14,11 +14,18 @@
 
 #include "hw/virtio/virtio.h"
 
+typedef struct VhostVDPAHostNotifier {
+    MemoryRegion mr;
+    void *addr;
+} VhostVDPAHostNotifier;
+
 typedef struct vhost_vdpa {
     int device_fd;
     uint32_t msg_type;
     MemoryListener listener;
     struct vhost_dev *dev;
+    VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
+    bool host_notifier_set;
 } VhostVDPA;
 
 extern AddressSpace address_space_memory;
Signature · 1 line
2.25.1
#3 lore raw

Re: [PATCH 2/2] vhost-vdpa: doorbell mapping support

Quoted 81 lines · Jason Wang

This patch implements the doorbell mapping support for vhost-vDPA. This is simply done by using mmap()/munmap() for the vhost-vDPA fd during device start/stop. For the device without doorbell support, we fall back to eventfd based notification gracefully.

---
  hw/virtio/vhost-vdpa.c         | 85 ++++++++++++++++++++++++++++++++++
  include/hw/virtio/vhost-vdpa.h |  7 +++
  2 files changed, 92 insertions(+)
hw/virtio/vhost-vdpa.c +61 -0
index dd4321bac2..c3a3b7566f 100644
@@ -285,12 +285,95 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
      return 0;
  }
 
+static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
+                                            int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+
+    n = &v->notifier[queue_index];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+}
+
+static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n)
+{
+    int i;
+
+    for (i = 0; i < n; i++) {
+        vhost_vdpa_host_notifier_uninit(dev, i);
+    }
+}
+
+static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+    int fd = v->device_fd;
+    void *addr;
+    char *name;
+
+    vhost_vdpa_host_notifier_uninit(dev, queue_index);
+
+    n = &v->notifier[queue_index];
+
+    addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
+                queue_index * page_size);
+    if (addr == MAP_FAILED) {
+        goto err;
+    }
+
+    name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
+                           v, queue_index);
+    memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+                                      page_size, addr);
+    g_free(name);
+
+    if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) {
+        munmap(addr, page_size);
+        goto err;
+    }
+    n->addr = addr;
+
+    return 0;
+

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 358

Quoted 10 lines
hw/virtio/vhost-vdpa.c +10 -0
+err:
+    return -1;
+}
+
+static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->nvqs; i++) {
+        if (vhost_vdpa_host_notifier_init(dev, i)) {

Shouldn't (hdev->vq_index + i) be used instead of i? or it's assumed to be single QP for vhost-vdpa for the moment? If the latter, would be good to add some comment.

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 366

Quoted 8 lines
hw/virtio/vhost-vdpa.c +8 -0
+            goto err;
+        }
+    }
+
+    return;
+
+err:
+    vhost_vdpa_host_notifiers_uninit(dev, i);

I'm not sure if it is really the intent to leave other vqs behind - I presume that either none of them is mapped, or all mappable should be mapped. Why here just uninit the first unmappable vq?

Quoted 39 lines
hw/virtio/vhost-vdpa.c +6 -0
+    return;
+}
+
  static int vhost_vdpa_cleanup(struct vhost_dev *dev)
  {
      struct vhost_vdpa *v;
      assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
      v = dev->opaque;
      trace_vhost_vdpa_cleanup(dev, v);
+    vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
      memory_listener_unregister(&v->listener);
 
      dev->opaque = NULL;
@@ -467,6 +550,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
      if (started) {
          uint8_t status = 0;
          memory_listener_register(&v->listener, &address_space_memory);
+        vhost_vdpa_host_notifiers_init(dev);
          vhost_vdpa_set_vring_ready(dev);
          vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
          vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
@@ -476,6 +560,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started)
          vhost_vdpa_reset_device(dev);
          vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
                                     VIRTIO_CONFIG_S_DRIVER);
+        vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs);
          memory_listener_unregister(&v->listener);
 
          return 0;
include/hw/virtio/vhost-vdpa.h +2 -0
index 9b81a409da..0f11ecff34 100644
@@ -14,11 +14,18 @@
 
  #include "hw/virtio/virtio.h"
 
+typedef struct VhostVDPAHostNotifier {
+    MemoryRegion mr;

include/hw/virtio/vhost-vdpa.h line 28

Quoted 10 lines
include/hw/virtio/vhost-vdpa.h +5 -0
+    void *addr;
+} VhostVDPAHostNotifier;
+
  typedef struct vhost_vdpa {
      int device_fd;
      uint32_t msg_type;
      MemoryListener listener;
      struct vhost_dev *dev;
+    VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
+    bool host_notifier_set;

What this host_notifier_set is used for? Doesn't seem it's ever set or referenced?

include/hw/virtio/vhost-vdpa.h line 31

Quoted 3 lines
include/hw/virtio/vhost-vdpa.h +0 -0
  } VhostVDPA;
 
  extern AddressSpace address_space_memory;

Thanks,
-Siwei

author
#4 lore raw

Re: [PATCH 2/2] vhost-vdpa: doorbell mapping support

Quoted 91 lines · Si-Wei Liu
Quoted 88 lines · Jason Wang

This patch implements the doorbell mapping support for vhost-vDPA. This is simply done by using mmap()/munmap() for the vhost-vDPA fd during device start/stop. For the device without doorbell support, we fall back to eventfd based notification gracefully.

---
  hw/virtio/vhost-vdpa.c         | 85 ++++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-vdpa.h |  7 +++

  2 files changed, 92 insertions(+)
hw/virtio/vhost-vdpa.c +63 -0
index dd4321bac2..c3a3b7566f 100644
@@ -285,12 +285,95 @@ static int vhost_vdpa_init(struct vhost_dev 
 *dev, void *opaque)
       return 0;
   }
   +static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
+                                            int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+
+    n = &v->notifier[queue_index];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, 
 false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+}
+
+static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, 
 int n)
+{
+    int i;
+
+    for (i = 0; i < n; i++) {
+        vhost_vdpa_host_notifier_uninit(dev, i);
+    }
+}
+
+static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int 
 queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+    int fd = v->device_fd;
+    void *addr;
+    char *name;
+
+    vhost_vdpa_host_notifier_uninit(dev, queue_index);
+
+    n = &v->notifier[queue_index];
+
+    addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
+                queue_index * page_size);
+    if (addr == MAP_FAILED) {
+        goto err;
+    }
+
+    name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
+                           v, queue_index);
+    memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+                                      page_size, addr);
+    g_free(name);
+
+    if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, 
 true)) {
+        munmap(addr, page_size);
+        goto err;
+    }
+    n->addr = addr;
+
+    return 0;
+
+err:
+    return -1;
+}
Quoted 8 lines

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 358

Quoted 7 lines
hw/virtio/vhost-vdpa.c +7 -0
+
+static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->nvqs; i++) {
+        if (vhost_vdpa_host_notifier_init(dev, i)) {

Shouldn't (hdev->vq_index + i) be used instead of i? or it's assumed to be single QP for vhost-vdpa for the moment?

Only single queue pair is supported, I'm working on the multiqueue support.

Quoted 1 line

If the latter, would be good to add some comment.

I agree, and I think it's better to use vq_index + i to avoid future changes.

Quoted 9 lines

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 366

Quoted 8 lines
hw/virtio/vhost-vdpa.c +8 -0
+            goto err;
+        }
+    }
+
+    return;
+
+err:
+    vhost_vdpa_host_notifiers_uninit(dev, i);

I'm not sure if it is really the intent to leave other vqs behind - I presume that either none of them is mapped, or all mappable should be mapped. Why here just uninit the first unmappable vq?

I'm not sure I get here, there's a loop in vhost_vdpa_host_notifiers_uninit(), so we either:

1) map all doorbells

or

2) no doorell is mapped

Quoted 41 lines
Quoted 41 lines
hw/virtio/vhost-vdpa.c +3 -0
+    return;
+}
+
   static int vhost_vdpa_cleanup(struct vhost_dev *dev)
   {
       struct vhost_vdpa *v;
       assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);

      v = dev->opaque; trace_vhost_vdpa_cleanup(dev, v);
+    vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); dev->opaque = NULL;

hw/virtio/vhost-vdpa.c +1 -0
@@ -467,6 +550,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev 
 *dev, bool started)
       if (started) {
           uint8_t status = 0;
           memory_listener_register(&v->listener, &address_space_memory);
+        vhost_vdpa_host_notifiers_init(dev);
           vhost_vdpa_set_vring_ready(dev);
           vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);

          vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);

hw/virtio/vhost-vdpa.c +0 -0
@@ -476,6 +560,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev 
 *dev, bool started)
           vhost_vdpa_reset_device(dev);

          vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER); +        vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); return 0;

include/hw/virtio/vhost-vdpa.h +4 -0
index 9b81a409da..0f11ecff34 100644
@@ -14,11 +14,18 @@
     #include "hw/virtio/virtio.h"
   +typedef struct VhostVDPAHostNotifier {
+    MemoryRegion mr;
+    void *addr;
+} VhostVDPAHostNotifier;
+
Quoted 8 lines

include/hw/virtio/vhost-vdpa.h line 28

Quoted 7 lines
include/hw/virtio/vhost-vdpa.h +2 -0
   typedef struct vhost_vdpa {
       int device_fd;
       uint32_t msg_type;
       MemoryListener listener;
       struct vhost_dev *dev;
+    VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
+    bool host_notifier_set;

What this host_notifier_set is used for? Doesn't seem it's ever set or referenced?

Right, will remove it.

Thanks

Quoted 6 lines

include/hw/virtio/vhost-vdpa.h line 31

Quoted 3 lines
include/hw/virtio/vhost-vdpa.h +0 -0
   } VhostVDPA;
     extern AddressSpace address_space_memory;

Thanks,
-Siwei

#5 lore raw

Re: [PATCH 2/2] vhost-vdpa: doorbell mapping support

Quoted 104 lines · Jason Wang
Quoted 92 lines · Si-Wei Liu
Quoted 89 lines · Jason Wang

This patch implements the doorbell mapping support for vhost-vDPA. This is simply done by using mmap()/munmap() for the vhost-vDPA fd during device start/stop. For the device without doorbell support, we fall back to eventfd based notification gracefully.

---
  hw/virtio/vhost-vdpa.c         | 85
++++++++++++++++++++++++++++++++++
  include/hw/virtio/vhost-vdpa.h |  7 +++

  2 files changed, 92 insertions(+)
hw/virtio/vhost-vdpa.c +63 -0
index dd4321bac2..c3a3b7566f 100644
@@ -285,12 +285,95 @@ static int vhost_vdpa_init(struct vhost_dev 
 *dev, void *opaque)
       return 0;
   }
   +static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev,
+                                            int queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+
+    n = &v->notifier[queue_index];
+
+    if (n->addr) {
+        virtio_queue_set_host_notifier_mr(vdev, queue_index, 
 &n->mr, false);
+        object_unparent(OBJECT(&n->mr));
+        munmap(n->addr, page_size);
+        n->addr = NULL;
+    }
+}
+
+static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, 
 int n)
+{
+    int i;
+
+    for (i = 0; i < n; i++) {
+        vhost_vdpa_host_notifier_uninit(dev, i);
+    }
+}
+
+static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int 
 queue_index)
+{
+    size_t page_size = qemu_real_host_page_size;
+    struct vhost_vdpa *v = dev->opaque;
+    VirtIODevice *vdev = dev->vdev;
+    VhostVDPAHostNotifier *n;
+    int fd = v->device_fd;
+    void *addr;
+    char *name;
+
+    vhost_vdpa_host_notifier_uninit(dev, queue_index);
+
+    n = &v->notifier[queue_index];
+
+    addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
+                queue_index * page_size);
+    if (addr == MAP_FAILED) {
+        goto err;
+    }
+
+    name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]",
+                           v, queue_index);
+    memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
+                                      page_size, addr);
+    g_free(name);
+
+    if (virtio_queue_set_host_notifier_mr(vdev, queue_index, 
 &n->mr, true)) {
+        munmap(addr, page_size);
+        goto err;
+    }
+    n->addr = addr;
+
+    return 0;
+
+err:
+    return -1;
+}
Quoted 8 lines

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 358

Quoted 7 lines
hw/virtio/vhost-vdpa.c +7 -0
+
+static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->nvqs; i++) {
+        if (vhost_vdpa_host_notifier_init(dev, i)) {

Shouldn't (hdev->vq_index + i) be used instead of i? or it's assumed to be single QP for vhost-vdpa for the moment?

Quoted 1 line

Only single queue pair is supported, I'm working on the multiqueue support.

OK, I see.

Quoted 6 lines
Quoted 3 lines

If the latter, would be good to add some comment.

I agree, and I think it's better to use vq_index + i to avoid future changes.

That'll be fine. I think that depends on the way how mq will be modeled for vhost-vdpa, i.e. it doesn't need to be 1:1 between struct vhost_dev and a queue pair, like what vhost-kernel is modeled after for mq.

Quoted 13 lines
Quoted 11 lines

hw/virtio/vhost-vdpa.c static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) · line 366

Quoted 10 lines
hw/virtio/vhost-vdpa.c +8 -0
+            goto err;
+        }
+    }
+
+    return;
+
+err:
+    vhost_vdpa_host_notifiers_uninit(dev, i);

I'm not sure if it is really the intent to leave other vqs behind - I presume that either none of them is mapped, or all mappable should be mapped. Why here just uninit the first unmappable vq?

Quoted 7 lines

I'm not sure I get here, there's a loop in vhost_vdpa_host_notifiers_uninit(), so we either:

1) map all doorbells

or

2) no doorell is mapped

Oops, I missed the 's' in vhost_vdpa_host_notifiers_uninit() and thought it was vhost_vdpa_host_notifier_uninit(). Sorry for the false alarm. The error handling looks fine then.

Thanks!
-Siwei

Quoted 65 lines
Quoted 43 lines
Quoted 43 lines
hw/virtio/vhost-vdpa.c +3 -0
+    return;
+}
+
   static int vhost_vdpa_cleanup(struct vhost_dev *dev)
   {
       struct vhost_vdpa *v;
       assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);

      v = dev->opaque; trace_vhost_vdpa_cleanup(dev, v);
+    vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); dev->opaque = NULL;

hw/virtio/vhost-vdpa.c +1 -0
@@ -467,6 +550,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev 
 *dev, bool started)
       if (started) {
           uint8_t status = 0;
           memory_listener_register(&v->listener, &address_space_memory);
+        vhost_vdpa_host_notifiers_init(dev);
           vhost_vdpa_set_vring_ready(dev);
           vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);

          vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);

hw/virtio/vhost-vdpa.c +0 -0
@@ -476,6 +560,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev 
 *dev, bool started)
           vhost_vdpa_reset_device(dev);

          vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER); +        vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); return 0;

include/hw/virtio/vhost-vdpa.h +4 -0
index 9b81a409da..0f11ecff34 100644
@@ -14,11 +14,18 @@
     #include "hw/virtio/virtio.h"
   +typedef struct VhostVDPAHostNotifier {
+    MemoryRegion mr;
+    void *addr;
+} VhostVDPAHostNotifier;
+
Quoted 8 lines

include/hw/virtio/vhost-vdpa.h line 28

Quoted 7 lines
include/hw/virtio/vhost-vdpa.h +2 -0
   typedef struct vhost_vdpa {
       int device_fd;
       uint32_t msg_type;
       MemoryListener listener;
       struct vhost_dev *dev;
+    VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX];
+    bool host_notifier_set;

What this host_notifier_set is used for? Doesn't seem it's ever set or referenced?

Right, will remove it.

Thanks

Quoted 6 lines

include/hw/virtio/vhost-vdpa.h line 31

Quoted 3 lines
include/hw/virtio/vhost-vdpa.h +0 -0
   } VhostVDPA;
     extern AddressSpace address_space_memory;

Thanks,
-Siwei

#6 lore raw
In reply to Jason Wang
Quoted 4 lines · Jason Wang

Hi All:

This series tries to implement doorbell mapping support for vhost-vDPA. Tested with virtio-pci vDPA driver.

BTW I'd rather we used the kick/call terminology from the virtio spec. doorbell terminology seems to be originating from e1000 which has a register named like this.

Quoted 12 lines

Please review.

Jason Wang (2):
vhost-vdpa: skip ram device from the IOTLB mapping
vhost-vdpa: doorbell mapping support

 hw/virtio/vhost-vdpa.c         | 97 ++++++++++++++++++++++++++++++----
 include/hw/virtio/vhost-vdpa.h |  7 +++
 2 files changed, 94 insertions(+), 10 deletions(-)
Signature · 1 line
2.25.1
author
#7 lore raw
Quoted 8 lines · Michael S. Tsirkin
Quoted 4 lines · Jason Wang

Hi All:

This series tries to implement doorbell mapping support for vhost-vDPA. Tested with virtio-pci vDPA driver.

BTW I'd rather we used the kick/call terminology from the virtio spec. doorbell terminology seems to be originating from e1000 which has a register named like this.

Ok, will rename in next version.

Thanks

Quoted 14 lines
Quoted 13 lines

Please review.

Jason Wang (2):
vhost-vdpa: skip ram device from the IOTLB mapping
vhost-vdpa: doorbell mapping support

  hw/virtio/vhost-vdpa.c         | 97 ++++++++++++++++++++++++++++++----
  include/hw/virtio/vhost-vdpa.h |  7 +++
  2 files changed, 94 insertions(+), 10 deletions(-)
Signature · 1 line
2.25.1