Skip to contentSkip to conversation

vDPA support in qemu

RFC Superseded Cindy Lu 7 participants 41 messages May 29, 2020 – Jun 17, 2020 Networking

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Cindy Lu

author
#0 lore raw

v3 cover vDPA support in qemu

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. This RFC introduce the vDPA support in qemu
TODO:
1) vIOMMU support
2) live migration support

change from v1
separate the patch of introduce vhost_set_vring_ready method separate the patch of qemu_get_peer
separate the patch  of vhost_set_state
introduce the new macro specific for vDPA in configure introduce the function to pass the fd from cmdline introduce the docmation in qemu-options.hx
the other comments form last version

change from v2
change the work process of vhost set status
introduce vhost_get_device_id
test based on qemu v5.0.0-rc4
the other comments from last version

Cindy Lu (3):
net: introduce qemu_get_peer
vhost_net: use the function qemu_get_peer
vhost-backend: export the vhost backend helper

Jason Wang (3):
virtio-bus: introduce queue_enabled method
virtio-pci: implement queue_enabled method
vhost: introduce vhost_set_vring_ready method

Tiwei Bie (2):
vhost-vdpa: introduce vhost-vdpa backend
vhost-vdpa: introduce vhost-vdpa net client

 configure                         |  21 ++
 hw/net/vhost_net-stub.c           |   9 +
 hw/net/vhost_net.c                |  72 +++++-
 hw/virtio/Makefile.objs           |   1 +
 hw/virtio/vhost-backend.c         |  39 +--
 hw/virtio/vhost-vdpa.c            | 399 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |  37 ++-
 hw/virtio/virtio-pci.c            |  13 +
 hw/virtio/virtio.c                |   6 +
 include/hw/virtio/vhost-backend.h |  38 ++-
 include/hw/virtio/vhost-vdpa.h    |  26 ++
 include/hw/virtio/vhost.h         |   2 +
 include/hw/virtio/virtio-bus.h    |   4 +
 include/net/net.h                 |   1 +
 include/net/vhost-vdpa.h          |  19 ++
 include/net/vhost_net.h           |   3 +-
 net/Makefile.objs                 |   2 +-
 net/clients.h                     |   2 +
 net/net.c                         |   9 +
 net/vhost-vdpa.c                  | 235 ++++++++++++++++++
 qapi/net.json                     |  26 +-
 qemu-options.hx                   |  15 ++
 22 files changed, 951 insertions(+), 28 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
Signature · 1 line
2.21.1
author
#14 lore raw

RFC v3 6/8 vhost-backend: export the vhost backend helper

export the helper then we can reuse some of them in vhost-vdpa

---
 hw/virtio/vhost-backend.c         | 34 ++++++++++++++++++-------------
 include/hw/virtio/vhost-backend.h | 28 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 14 deletions(-)
hw/virtio/vhost-backend.c +20 -14
index 48905383f8..42efb4967b 100644
@@ -14,7 +14,7 @@
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "standard-headers/linux/vhost_types.h"
-
+#include "hw/virtio/vhost-vdpa.h"
 #ifdef CONFIG_VHOST_KERNEL
 #include <linux/vhost.h>
 #include <sys/ioctl.h>
@@ -22,10 +22,16 @@
 static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
                              void *arg)
 {
-    int fd = (uintptr_t) dev->opaque;
-
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
-
+    int fd = -1;
+    struct vhost_vdpa *v = NULL;
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL) {
+        fd  = (uintptr_t) dev->opaque;
+    }
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        v = dev->opaque;
+        fd = v->device_fd;
+    }
+    assert(fd != -1);
     return ioctl(fd, request, arg);
 }
 
@@ -89,7 +95,7 @@ static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version
     return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
 }
 
-static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
+int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
                                      struct vhost_log *log)
 {
     return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
@@ -101,7 +107,7 @@ static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
     return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
 }
 
-static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
+int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
                                        struct vhost_vring_addr *addr)
 {
     return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
@@ -113,31 +119,31 @@ static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
     return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
 }
 
-static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
+int vhost_kernel_set_vring_num(struct vhost_dev *dev,
                                       struct vhost_vring_state *ring)
 {
     return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
 }
 
-static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
+int vhost_kernel_set_vring_base(struct vhost_dev *dev,
                                        struct vhost_vring_state *ring)
 {
     return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
 }
 
-static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
+int vhost_kernel_get_vring_base(struct vhost_dev *dev,
                                        struct vhost_vring_state *ring)
 {
     return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
 }
 
-static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
+int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
                                        struct vhost_vring_file *file)
 {
     return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
 }
 
-static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
+int vhost_kernel_set_vring_call(struct vhost_dev *dev,
                                        struct vhost_vring_file *file)
 {
     return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
@@ -155,13 +161,13 @@ static int vhost_kernel_set_features(struct vhost_dev *dev,
     return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
 }
 
-static int vhost_kernel_get_features(struct vhost_dev *dev,
+int vhost_kernel_get_features(struct vhost_dev *dev,
                                      uint64_t *features)
 {
     return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
 }
 
-static int vhost_kernel_set_owner(struct vhost_dev *dev)
+int vhost_kernel_set_owner(struct vhost_dev *dev)
 {
     return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
 }
include/hw/virtio/vhost-backend.h +28 -0
index 6f6670783f..300b59c172 100644
@@ -172,4 +172,32 @@ int vhost_backend_handle_iotlb_msg(struct vhost_dev *dev,
 
 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd);
 
+
+int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log);
+
+int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr);
+
+int vhost_kernel_set_vring_num(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring);
+
+int vhost_kernel_set_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+
+int vhost_kernel_get_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+
+int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+
+int vhost_kernel_set_vring_call(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+
+int vhost_kernel_set_owner(struct vhost_dev *dev);
+
+int vhost_kernel_get_features(struct vhost_dev *dev,
+                                     uint64_t *features);
+
+
 #endif /* VHOST_BACKEND_H */
Signature · 1 line
2.21.1
author
#1 lore raw

RFC v3 1/8 net: introduce qemu_get_peer

In reply to Cindy Lu

This is a small function that can get the peer from given NetClientState and queue_index

---
 include/net/net.h | 1 +
 net/net.c         | 6 ++++++
 2 files changed, 7 insertions(+)
include/net/net.h +1 -0
index 39085d9444..e7ef42d62b 100644
@@ -176,6 +176,7 @@ void hmp_info_network(Monitor *mon, const QDict *qdict);
 void net_socket_rs_init(SocketReadState *rs,
                         SocketReadStateFinalize *finalize,
                         bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
 /* NIC info */
 
net/net.c +6 -0
index 38778e831d..599fb61028 100644
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
     return nic->opaque;
 }
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    assert(nc != NULL);
+    NetClientState *ncs = nc + queue_index;
+    return ncs->peer;
+}
 
 static void qemu_cleanup_net_client(NetClientState *nc)
 {
Signature · 1 line
2.21.1
#32 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu
Quoted 10 lines · Cindy Lu

From: Tiwei Bie <[email protected]>

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is specified by a "vhostdev" parameter.

---

net/vhost-vdpa.c line 166

Quoted 4 lines
net/vhost-vdpa.c new file +4 -0
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev,
+                               bool has_fd, char *fd)
+{

fd is usually an int, not a string.

Quoted 8 lines
net/vhost-vdpa.c new file +8 -0
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd = -1;
+    Error *err = NULL;
+    int ret = 0;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);

net/vhost-vdpa.c line 184

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (has_fd) {
+        vdpa_device_fd = monitor_fd_param(cur_mon, fd, &err);
+    } else{
+        vdpa_device_fd = open(vhostdev, O_RDWR);
+    }

Oh, you're trying to use the old way for passing in fds. The preferred way is to use qemu_open(), at which point you can pass in fds via the add-fd QMP command, and then pass the string "/dev/fdset/NNN" as vhostdev. Then you don't need a special fd parameter here.

Quoted 15 lines

+++ b/qapi/net.json

qapi/net.json +10 -0
@@ -428,6 +428,27 @@
      '*vhostforce':    'bool',
      '*queues':        'int' } }
 
+##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a vdpa dev path in sysfs
+#            (default path:/dev/vhost-vdpa-$ID)
+#
+# @fd: file descriptor of an already opened vdpa device
+#

qapi/net.json line 450

Quoted 10 lines
qapi/net.json +10 -0
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1)
+#
+# Since: 5.1
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*fd':           'str',
+    '*queues':       'int' } }

Instead of having vhostdev and fd both be optional (but where the user has to specify exactly one of them), you should only have vhostdev be mandatory, and rely on the /dev/fdset/NNN string as a way to get vhostdev to point to a previously-passed fd.

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
author
#4 lore raw

RFC v3 2/8 vhost_net: use the function qemu_get_peer

In reply to Cindy Lu

user the qemu_get_peer to replace the old process

---
 hw/net/vhost_net.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
hw/net/vhost_net.c +9 -5
index 6b82803fa7..d1d421e3d9 100644
@@ -306,7 +306,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
+    struct vhost_net *net;
     int r, e, i;
+    NetClientState *peer;
 
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
@@ -314,9 +316,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        struct vhost_net *net;
 
-        net = get_vhost_net(ncs[i].peer);
+        peer = qemu_get_peer(ncs, i);
+        net = get_vhost_net(peer);
         vhost_net_set_vq_index(net, i * 2);
 
         /* Suppress the masking guest notifiers on vhost user
@@ -335,7 +337,8 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs, i);
+        r = vhost_net_start_one(get_vhost_net(peer), dev);
 
         if (r < 0) {
             goto err_start;
@@ -343,7 +346,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
         if (ncs[i].peer->vring_enable) {
             /* restore vring enable state */
-            r = vhost_set_vring_enable(ncs[i].peer, ncs[i].peer->vring_enable);
+            r = vhost_set_vring_enable(peer, peer->vring_enable);
 
             if (r < 0) {
                 goto err_start;
@@ -355,7 +358,8 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
 err_start:
     while (--i >= 0) {
-        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs , i);
+        vhost_net_stop_one(get_vhost_net(peer), dev);
     }
     e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
     if (e < 0) {
Signature · 1 line
2.21.1
author
#6 lore raw

RFC v3 3/8 virtio-bus: introduce queue_enabled method

In reply to Cindy Lu

From: Jason Wang <[email protected]>

This patch introduces queue_enabled() method which allows the transport to implement its own way to report whether or not a queue is enabled.

0005-virtio-bus-introduce-queue_enabled-method.patch

---
 hw/virtio/virtio.c             | 6 ++++++
 include/hw/virtio/virtio-bus.h | 4 ++++
 2 files changed, 10 insertions(+)
hw/virtio/virtio.c +6 -0
index b6c8ef5bc0..445a4ed760 100644
@@ -3285,6 +3285,12 @@ hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 
 bool virtio_queue_enabled(VirtIODevice *vdev, int n)
 {
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (k->queue_enabled) {
+        return k->queue_enabled(qbus->parent, n);
+    }
     return virtio_queue_get_desc_addr(vdev, n) != 0;
 }
 
include/hw/virtio/virtio-bus.h +4 -0
index 38c9399cd4..0f6f215925 100644
@@ -83,6 +83,10 @@ typedef struct VirtioBusClass {
      */
     int (*ioeventfd_assign)(DeviceState *d, EventNotifier *notifier,
                             int n, bool assign);
+    /*
+     * Whether queue number n is enabled.
+     */
+    bool (*queue_enabled)(DeviceState *d, int n);
     /*
      * Does the transport have variable vring alignment?
      * (ie can it ever call virtio_queue_set_align()?)
Signature · 1 line
2.21.1
author
#9 lore raw

RFC v3 4/8 virtio-pci: implement queue_enabled method

In reply to Cindy Lu

From: Jason Wang <[email protected]>

With version 1, we can detect whether a queue is enabled via queue_enabled.

---
 hw/virtio/virtio-pci.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
hw/virtio/virtio-pci.c +13 -0
index 4cb784389c..2c82ed5246 100644
@@ -1107,6 +1107,18 @@ static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
     return pci_get_address_space(dev);
 }
 
+static bool virtio_pci_queue_enabled(DeviceState *d, int n)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return proxy->vqs[vdev->queue_sel].enabled;
+    }
+
+    return virtio_queue_get_desc_addr(vdev, n) != 0;
+}
+
 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
                                    struct virtio_pci_cap *cap)
 {
@@ -2059,6 +2071,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
     k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
     k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
     k->get_dma_as = virtio_pci_get_dma_as;
+    k->queue_enabled = virtio_pci_queue_enabled;
 }
 
 static const TypeInfo virtio_pci_bus_info = {
Signature · 1 line
2.21.1
author
#11 lore raw

RFC v3 5/8 vhost: introduce vhost_set_vring_ready method

In reply to Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

---
 hw/net/vhost_net-stub.c |  4 ++++
 hw/net/vhost_net.c      | 11 ++++++++++-
 include/net/vhost_net.h |  1 +
 3 files changed, 15 insertions(+), 1 deletion(-)
hw/net/vhost_net-stub.c +4 -0
index aac0e98228..43e93e1a9a 100644
@@ -86,6 +86,10 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
hw/net/vhost_net.c +10 -1
index d1d421e3d9..e2bc7de2eb 100644
@@ -344,7 +344,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
             goto err_start;
         }
 
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
             /* restore vring enable state */
             r = vhost_set_vring_enable(peer, peer->vring_enable);
 
@@ -455,6 +455,15 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+    return 0;
+}
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     const VhostOps *vhost_ops = net->dev.vhost_ops;
include/net/vhost_net.h +1 -0
index 77e47398c4..8a6f208189 100644
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
 
 int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState *nc);
 
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
Signature · 1 line
2.21.1
author
#17 lore raw

RFC v3 7/8 vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu

From: Tiwei Bie <[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
---
 configure                         |  21 ++
 hw/net/vhost_net-stub.c           |   5 +
 hw/net/vhost_net.c                |  47 +++-
 hw/virtio/Makefile.objs           |   1 +
 hw/virtio/vhost-backend.c         |   5 +
 hw/virtio/vhost-vdpa.c            | 399 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |  37 ++-
 include/hw/virtio/vhost-backend.h |  10 +-
 include/hw/virtio/vhost-vdpa.h    |  26 ++
 include/hw/virtio/vhost.h         |   2 +
 include/net/vhost_net.h           |   4 +-
 qemu-options.hx                   |  15 ++
 12 files changed, 566 insertions(+), 6 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
configure +21 -0
index 23b5e93752..53679ee57f 100755
@@ -1557,6 +1557,10 @@ for opt do
   ;;
   --enable-vhost-user) vhost_user="yes"
   ;;
+  --disable-vhost-vdpa) vhost_vdpa="no"
+  ;;
+  --enable-vhost-vdpa) vhost_vdpa="yes"
+  ;;
   --disable-vhost-kernel) vhost_kernel="no"
   ;;
   --enable-vhost-kernel) vhost_kernel="yes"
@@ -1846,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   vhost-crypto    vhost-user-crypto backend support
   vhost-kernel    vhost kernel backend support
   vhost-user      vhost-user backend support
+  vhost-vdpa      vhost-vdpa kernel backend support
   spice           spice
   rbd             rados block device (rbd)
   libiscsi        iscsi support
@@ -2336,6 +2341,10 @@ test "$vhost_user" = "" && vhost_user=yes
 if test "$vhost_user" = "yes" && test "$mingw32" = "yes"; then
   error_exit "vhost-user isn't available on win32"
 fi
+test "$vhost_vdpa" = "" && vhost_vdpa=$linux
+if test "$vhost_vdpa" = "yes" && test "$linux" != "yes"; then
+  error_exit "vhost-vdpa is only available on Linux"
+fi
 test "$vhost_kernel" = "" && vhost_kernel=$linux
 if test "$vhost_kernel" = "yes" && test "$linux" != "yes"; then
   error_exit "vhost-kernel is only available on Linux"
@@ -2364,6 +2373,11 @@ test "$vhost_user_fs" = "" && vhost_user_fs=$vhost_user
 if test "$vhost_user_fs" = "yes" && test "$vhost_user" = "no"; then
   error_exit "--enable-vhost-user-fs requires --enable-vhost-user"
 fi
+#vhost-vdpa backends
+test "$vhost_net_vdpa" = "" && vhost_net_vdpa=$vhost_vdpa
+if test "$vhost_net_vdpa" = "yes" && test "$vhost_vdpa" = "no"; then
+  error_exit "--enable-vhost-net-vdpa requires --enable-vhost-vdpa"
+fi
 
 # OR the vhost-kernel and vhost-user values for simplicity
 if test "$vhost_net" = ""; then
@@ -6673,6 +6687,7 @@ echo "vhost-scsi support $vhost_scsi"
 echo "vhost-vsock support $vhost_vsock"
 echo "vhost-user support $vhost_user"
 echo "vhost-user-fs support $vhost_user_fs"
+echo "vhost-vdpa support $vhost_vdpa"
 echo "Trace backends    $trace_backends"
 if have_backend "simple"; then
 echo "Trace output file $trace_file-<pid>"
@@ -7170,6 +7185,9 @@ fi
 if test "$vhost_net_user" = "yes" ; then
   echo "CONFIG_VHOST_NET_USER=y" >> $config_host_mak
 fi
+if test "$vhost_net_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_NET_VDPA=y" >> $config_host_mak
+fi
 if test "$vhost_crypto" = "yes" ; then
   echo "CONFIG_VHOST_CRYPTO=y" >> $config_host_mak
 fi
@@ -7182,6 +7200,9 @@ fi
 if test "$vhost_user" = "yes" ; then
   echo "CONFIG_VHOST_USER=y" >> $config_host_mak
 fi
+if test "$vhost_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_VDPA=y" >> $config_host_mak
+fi
 if test "$vhost_user_fs" = "yes" ; then
   echo "CONFIG_VHOST_USER_FS=y" >> $config_host_mak
 fi
hw/net/vhost_net-stub.c +5 -0
index 43e93e1a9a..ab77a92a7d 100644
@@ -94,3 +94,8 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
 }
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return 0;
+}
+
hw/net/vhost_net.c +46 -1

88 lines not on this page · raw message

hw/virtio/Makefile.objs +1 -0
index 4e4d39a0a4..6b1b1a5fce 100644
@@ -5,6 +5,7 @@ obj-y += virtio.o
 obj-$(CONFIG_VHOST) += vhost.o vhost-backend.o
 common-obj-$(call lnot,$(CONFIG_VHOST)) += vhost-stub.o
 obj-$(CONFIG_VHOST_USER) += vhost-user.o
+obj-$(CONFIG_VHOST_VDPA) += vhost-vdpa.o
 
 common-obj-$(CONFIG_VIRTIO_RNG) += virtio-rng.o
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
hw/virtio/vhost-backend.c +5 -0
index 42efb4967b..420341e8c5 100644
@@ -291,6 +291,11 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
     case VHOST_BACKEND_TYPE_USER:
         dev->vhost_ops = &user_ops;
         break;
+#endif
+#ifdef CONFIG_VHOST_VDPA
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
 #endif
     default:
         error_report("Unknown vhost backend type");
hw/virtio/vhost-vdpa.c new file +399 -0

399 lines not on this page · raw message

hw/virtio/vhost.c +35 -2
index 01ebe12f28..b97aa02a4c 100644
@@ -756,6 +756,12 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
+    /*vDPA need to use the phys address here to set to hardware*/
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }
     int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
         VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
@@ -1506,6 +1512,14 @@ int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
     return -1;
 }
 
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id)
+{
+    assert(hdev->vhost_ops);
+    if (hdev->vhost_ops->vhost_get_device_id) {
+        return hdev->vhost_ops->vhost_get_device_id(hdev, device_id);
+    }
+    return -1;
+}
 void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
                                    const VhostDevConfigOps *ops)
 {
@@ -1661,7 +1675,13 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         }
     }
 
-    if (vhost_dev_has_iommu(hdev)) {
+    r = vhost_set_state(hdev, true);
+    if (r) {
+        goto fail_log;
+    }
+
+    if (vhost_dev_has_iommu(hdev) &&
+        hdev->vhost_ops->vhost_set_iotlb_callback) {
         hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
 
         /* Update used ring information for IOTLB to work correctly,
@@ -1697,6 +1717,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     /* should only be called after backend is connected */
     assert(hdev->vhost_ops);
 
+    vhost_set_state(hdev, false);
+
     for (i = 0; i < hdev->nvqs; ++i) {
         vhost_virtqueue_stop(hdev,
                              vdev,
@@ -1705,7 +1727,9 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
 
     if (vhost_dev_has_iommu(hdev)) {
-        hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        if (hdev->vhost_ops->vhost_set_iotlb_callback) {
+            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        }
         memory_listener_unregister(&hdev->iommu_listener);
     }
     vhost_log_put(hdev, true);
@@ -1722,3 +1746,12 @@ int vhost_net_set_backend(struct vhost_dev *hdev,
 
     return -1;
 }
+
+int vhost_set_state(struct vhost_dev *hdev, bool started)
+{
+    if (hdev->vhost_ops->vhost_set_state) {
+        return hdev->vhost_ops->vhost_set_state(hdev, started);
+    }
+
+    return 0;
+}
include/hw/virtio/vhost-backend.h +9 -1
index 300b59c172..1ebe3785cf 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
     VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
 } VhostBackendType;
 
 typedef enum VhostSetConfigType {
@@ -77,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
 typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
 typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                          int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
 typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
 typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                        char *mac_addr);
@@ -112,6 +114,8 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
 typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                         struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, bool started);
+typedef int (*vhost_get_device_id_op)(struct vhost_dev *dev, uint32_t *dev_id);
 typedef struct VhostOps {
     VhostBackendType backend_type;
     vhost_backend_init vhost_backend_init;
@@ -138,6 +142,7 @@ typedef struct VhostOps {
     vhost_reset_device_op vhost_reset_device;
     vhost_get_vq_index_op vhost_get_vq_index;
     vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
     vhost_requires_shm_log_op vhost_requires_shm_log;
     vhost_migration_done_op vhost_migration_done;
     vhost_backend_can_merge_op vhost_backend_can_merge;
@@ -152,9 +157,12 @@ typedef struct VhostOps {
     vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
     vhost_get_inflight_fd_op vhost_get_inflight_fd;
     vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
+    vhost_get_device_id_op vhost_get_device_id;
 } VhostOps;
 
 extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
 int vhost_set_backend_type(struct vhost_dev *dev,
                            VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +26 -0
new file mode 100644
index 0000000000..6455663388
@@ -0,0 +1,26 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    uint32_t msg_type;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+extern int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id);
+#endif
include/hw/virtio/vhost.h +2 -0
index 085450c6f8..b682545f51 100644
@@ -124,6 +124,7 @@ int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config,
                          uint32_t config_len);
 int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *data,
                          uint32_t offset, uint32_t size, uint32_t flags);
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id);
 /* notifier callback in case vhost device config space changed
  */
 void vhost_dev_set_config_notifier(struct vhost_dev *dev,
@@ -137,4 +138,5 @@ int vhost_dev_set_inflight(struct vhost_dev *dev,
                            struct vhost_inflight *inflight);
 int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
                            struct vhost_inflight *inflight);
+int vhost_set_state(struct vhost_dev *dev, bool started);
 #endif
include/net/vhost_net.h +2 -2
index 8a6f208189..56e67fe164 100644
@@ -40,5 +40,5 @@ int vhost_set_vring_ready(NetClientState *nc);
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
-
-#endif
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);
+endif
qemu-options.hx +15 -0
index 292d4e7c0c..c19e10ce9c 100644
@@ -2409,6 +2409,10 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
 #ifdef CONFIG_POSIX
     "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
     "                configure a vhost-user network, backed by a chardev 'dev'\n"
+#endif
+#ifdef CONFIG_POSIX
+    "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
+    "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
 #endif
     "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
     "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
@@ -2428,6 +2432,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
 #endif
 #ifdef CONFIG_POSIX
     "vhost-user|"
+#endif
+#ifdef CONFIG_POSIX
+    "vhost-vdpa|"
 #endif
     "socket][,option][,...][mac=macaddr]\n"
     "                initialize an on-board / default host NIC (using MAC address\n"
@@ -2896,6 +2903,14 @@ SRST
     hubport to another netdev with ID nd by using the ``netdev=nd``
     option.
 
+``-netdev vhost-vdpa,vhostdev=/path/to/dev ``
+    Establish a vhost-vdpa netdev.
+
+    vDPA device is a device that uses a datapath which complies with
+    the virtio specifications with a vendor specific control path.
+    vDPA devices can be both physically located on the hardware or
+    emulated by software.
+
 ``-net nic[,netdev=nd][,macaddr=mac][,model=type] [,name=name][,addr=addr][,vectors=v]``
     Legacy option to configure or create an on-board (or machine
     default) Network Interface Card(NIC) and connect it either to the
Signature · 1 line
2.21.1
author
#31 lore raw

RFC v3 8/8 vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu

From: Tiwei Bie <[email protected]>

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is specified by a "vhostdev" parameter.

---
 include/net/vhost-vdpa.h |  19 ++++
 include/net/vhost_net.h  |   2 +-
 net/Makefile.objs        |   2 +-
 net/clients.h            |   2 +
 net/net.c                |   3 +
 net/vhost-vdpa.c         | 235 +++++++++++++++++++++++++++++++++++++++
 qapi/net.json            |  26 ++++-
 7 files changed, 285 insertions(+), 4 deletions(-)
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
include/net/vhost-vdpa.h new file +19 -0
new file mode 100644
index 0000000000..6ce0d04f72
@@ -0,0 +1,19 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -1
index 56e67fe164..0b87d3c6e9 100644
@@ -41,4 +41,4 @@ uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
 int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);
-endif
+#endif
net/Makefile.objs +1 -1
index c5d076d19c..5ab45545db 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
 tap-obj-y ?= tap-stub.o
 common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
 common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_NET_VDPA) += vhost-vdpa.o
 vde.o-libs = $(VDE_LIBS)
 
 common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
 int net_init_vhost_user(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
 #endif /* QEMU_NET_CLIENTS_H */
net/net.c +3 -0
index 599fb61028..82624ea9ac 100644
@@ -965,6 +965,9 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
 #ifdef CONFIG_VHOST_NET_USER
         [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
 #endif
+#ifdef CONFIG_VHOST_NET_VDPA
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
+#endif
 #ifdef CONFIG_L2TPV3
         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
 #endif
net/vhost-vdpa.c new file +235 -0

235 lines not on this page · raw message

qapi/net.json +24 -2
index cebb1b52e3..37507ce9ba 100644
@@ -428,6 +428,27 @@
     '*vhostforce':    'bool',
     '*queues':        'int' } }
 
+##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a vdpa dev path in sysfs
+#            (default path:/dev/vhost-vdpa-$ID)
+#
+# @fd: file descriptor of an already opened vdpa device
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1)
+#
+# Since: 5.1
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*fd':           'str',
+    '*queues':       'int' } }
+
 ##
 # @NetClientDriver:
 #
@@ -437,7 +458,7 @@
 ##
 { 'enum': 'NetClientDriver',
   'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
 ##
 # @Netdev:
@@ -465,7 +486,8 @@
     'bridge':   'NetdevBridgeOptions',
     'hubport':  'NetdevHubPortOptions',
     'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
 ##
 # @NetLegacy:
Signature · 1 line
2.21.1
#38 lore raw
In reply to Cindy Lu
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1 === TEST SCRIPT END ===

CC x86_64-softmmu/hw/virtio/vhost-user-fs-pci.o
CC x86_64-softmmu/hw/virtio/virtio-iommu.o
/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c: In function 'vhost_vdpa_set_config':
/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c:323:17: error: 'VHOST_VDPA_MAX_CONFIG_SIZE' undeclared (first use in this function)

     if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {
                 ^

/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c:323:17: note: each undeclared identifier is reported only once for each function it appears in make[1]: *** [hw/virtio/vhost-vdpa.o] Error 1
make[1]: *** Waiting for unfinished jobs....
CC aarch64-softmmu/hw/vfio/common.o
CC aarch64-softmmu/hw/vfio/spapr.o
---
CC aarch64-softmmu/hw/virtio/vhost-vsock.o
CC aarch64-softmmu/hw/virtio/vhost-vsock-pci.o
/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c: In function 'vhost_vdpa_set_config':
/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c:323:17: error: 'VHOST_VDPA_MAX_CONFIG_SIZE' undeclared (first use in this function)

     if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {
                 ^

/tmp/qemu-test/src/hw/virtio/vhost-vdpa.c:323:17: note: each undeclared identifier is reported only once for each function it appears in make[1]: *** [hw/virtio/vhost-vdpa.o] Error 1
make[1]: *** Waiting for unfinished jobs....
make: *** [x86_64-softmmu/all] Error 2
make: *** Waiting for unfinished jobs....
make: *** [aarch64-softmmu/all] Error 2
Traceback (most recent call last):
File "./tests/docker/docker.py", line 664, in <module>

    sys.exit(main())

---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=c4d0aff9719e4e6986252b1cdad2d78a', '-u', '1001', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-jxkhgbnq/src/docker-src.2020-05-29-16.25.59.22140:/var/tmp/qemu:z,ro', 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=c4d0aff9719e4e6986252b1cdad2d78a
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-jxkhgbnq/src'
make: *** [docker-run-test-quick@centos7] Error 2

real 3m7.072s
user 0m9.059s

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]

#39 lore raw
In reply to Cindy Lu
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the asan build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1 === TEST SCRIPT END ===

CC accel/tcg/trace.o
CC backends/trace.o

Warning, treated as error:
/tmp/qemu-test/src/docs/../qemu-options.hx:2920:Inline literal start-string without end-string.
CC crypto/trace.o
CC monitor/trace.o
---

  CC      block/trace.o
  CC      io/trace.o
  CC      nbd/trace.o

make: *** [Makefile:1114: .docs_system_qemu.1_docs_system_qemu-block-drivers.7_docs_system_qemu-cpu-models.7.sentinel.] Error 2
make: *** Deleting file '.docs_system_qemu.1_docs_system_qemu-block-drivers.7_docs_system_qemu-cpu-models.7.sentinel.'
make: *** Waiting for unfinished jobs....

Warning, treated as error:
/tmp/qemu-test/src/docs/../qemu-options.hx:2920:Inline literal start-string without end-string.
make: *** [Makefile:1103: docs/system/index.html] Error 2
Traceback (most recent call last):
File "./tests/docker/docker.py", line 664, in <module>

    sys.exit(main())

---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=97acfdab68d34ed4abd8fdcbff72793c', '-u', '1001', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=x86_64-softmmu', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-l3clgv2k/src/docker-src.2020-05-29-16.30.04.2097:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-debug']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=97acfdab68d34ed4abd8fdcbff72793c
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-l3clgv2k/src'
make: *** [docker-run-test-debug@fedora] Error 2

real 3m32.660s
user 0m8.310s

The full log is available at
http://patchew.org/logs/[email protected]/testing.asan/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]

#40 lore raw
In reply to Cindy Lu
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the docker-mingw@fedora build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#! /bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-mingw@fedora J=14 NETWORK=1
=== TEST SCRIPT END ===

CC io/channel-websock.o
CC io/channel-util.o

Warning, treated as error:
/tmp/qemu-test/src/docs/../qemu-options.hx:2920:Inline literal start-string without end-string.
CC io/dns-resolver.o
CC io/net-listener.o
---
CC qom/container.o
CC qom/qom-qobject.o

Warning, treated as error:
/tmp/qemu-test/src/docs/../qemu-options.hx:2920:Inline literal start-string without end-string.
CC qom/object_interfaces.o
CC qemu-io.o
---

  CC      iothread.o
  CC      job-qmp.o
  CC      os-win32.o

make: *** [Makefile:1103: docs/system/index.html] Error 2
make: *** Waiting for unfinished jobs....
make: *** [Makefile:1114: .docs_system_qemu.1_docs_system_qemu-block-drivers.7_docs_system_qemu-cpu-models.7.sentinel.] Error 2
make: *** Deleting file '.docs_system_qemu.1_docs_system_qemu-block-drivers.7_docs_system_qemu-cpu-models.7.sentinel.'
Traceback (most recent call last):
File "./tests/docker/docker.py", line 664, in <module>
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=1b05e0f6710048c78da28e8db7addc87', '-u', '1001', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-9mffj7y4/src/docker-src.2020-05-29-16.35.13.7869:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-mingw']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=1b05e0f6710048c78da28e8db7addc87
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-9mffj7y4/src'
make: *** [docker-run-test-mingw@fedora] Error 2

real 1m50.552s
user 0m8.680s

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-mingw@fedora/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]

author
#33 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

In reply to Eric Blake
Quoted 12 lines · Eric Blake
Quoted 9 lines · Cindy Lu

From: Tiwei Bie <[email protected]>

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is specified by a "vhostdev" parameter.

---

Quoted 7 lines

net/vhost-vdpa.c line 166

Quoted 4 lines
net/vhost-vdpa.c new file +4 -0
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev,
+                               bool has_fd, char *fd)
+{

fd is usually an int, not a string.

will fix this

Quoted 15 lines
Quoted 15 lines
net/vhost-vdpa.c new file +15 -0
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd = -1;
+    Error *err = NULL;
+    int ret = 0;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (has_fd) {
+        vdpa_device_fd = monitor_fd_param(cur_mon, fd, &err);
Quoted 9 lines

net/vhost-vdpa.c line 184

Quoted 3 lines
net/vhost-vdpa.c new file +3 -0
+    } else{
+        vdpa_device_fd = open(vhostdev, O_RDWR);
+    }

Oh, you're trying to use the old way for passing in fds. The preferred way is to use qemu_open(), at which point you can pass in fds via the add-fd QMP command, and then pass the string "/dev/fdset/NNN" as vhostdev. Then you don't need a special fd parameter here.

Thanks Eric, I will try this.

Quoted 22 lines
Quoted 22 lines

+++ b/qapi/net.json

qapi/net.json +17 -0
@@ -428,6 +428,27 @@
      '*vhostforce':    'bool',
      '*queues':        'int' } }
 
+##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a vdpa dev path in sysfs
+#            (default path:/dev/vhost-vdpa-$ID)
+#
+# @fd: file descriptor of an already opened vdpa device
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1)
+#
+# Since: 5.1
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
Quoted 9 lines

qapi/net.json line 450

Quoted 3 lines
qapi/net.json +3 -0
+    '*vhostdev':     'str',
+    '*fd':           'str',
+    '*queues':       'int' } }

Instead of having vhostdev and fd both be optional (but where the user has to specify exactly one of them), you should only have vhostdev be mandatory, and rely on the /dev/fdset/NNN string as a way to get vhostdev to point to a previously-passed fd.

will fix this

Quoted 5 lines

--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org

#18 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu
Quoted 154 lines · Cindy Lu

From: Tiwei Bie <[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
---
  configure                         |  21 ++
  hw/net/vhost_net-stub.c           |   5 +
  hw/net/vhost_net.c                |  47 +++-
  hw/virtio/Makefile.objs           |   1 +
  hw/virtio/vhost-backend.c         |   5 +
  hw/virtio/vhost-vdpa.c            | 399 ++++++++++++++++++++++++++++++
  hw/virtio/vhost.c                 |  37 ++-
  include/hw/virtio/vhost-backend.h |  10 +-
  include/hw/virtio/vhost-vdpa.h    |  26 ++
  include/hw/virtio/vhost.h         |   2 +
  include/net/vhost_net.h           |   4 +-
  qemu-options.hx                   |  15 ++
  12 files changed, 566 insertions(+), 6 deletions(-)

create mode 100644 hw/virtio/vhost-vdpa.c create mode 100644 include/hw/virtio/vhost-vdpa.h

configure +21 -0
index 23b5e93752..53679ee57f 100755
@@ -1557,6 +1557,10 @@ for opt do
    ;;
    --enable-vhost-user) vhost_user="yes"
    ;;
+  --disable-vhost-vdpa) vhost_vdpa="no"
+  ;;
+  --enable-vhost-vdpa) vhost_vdpa="yes"
+  ;;
    --disable-vhost-kernel) vhost_kernel="no"
    ;;
    --enable-vhost-kernel) vhost_kernel="yes"
@@ -1846,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available:
    vhost-crypto    vhost-user-crypto backend support
    vhost-kernel    vhost kernel backend support
    vhost-user      vhost-user backend support
+  vhost-vdpa      vhost-vdpa kernel backend support
    spice           spice
    rbd             rados block device (rbd)
    libiscsi        iscsi support
@@ -2336,6 +2341,10 @@ test "$vhost_user" = "" && vhost_user=yes
  if test "$vhost_user" = "yes" && test "$mingw32" = "yes"; then
    error_exit "vhost-user isn't available on win32"
  fi
+test "$vhost_vdpa" = "" && vhost_vdpa=$linux
+if test "$vhost_vdpa" = "yes" && test "$linux" != "yes"; then
+  error_exit "vhost-vdpa is only available on Linux"
+fi
  test "$vhost_kernel" = "" && vhost_kernel=$linux
  if test "$vhost_kernel" = "yes" && test "$linux" != "yes"; then
    error_exit "vhost-kernel is only available on Linux"
@@ -2364,6 +2373,11 @@ test "$vhost_user_fs" = "" && vhost_user_fs=$vhost_user
  if test "$vhost_user_fs" = "yes" && test "$vhost_user" = "no"; then
    error_exit "--enable-vhost-user-fs requires --enable-vhost-user"
  fi
+#vhost-vdpa backends
+test "$vhost_net_vdpa" = "" && vhost_net_vdpa=$vhost_vdpa
+if test "$vhost_net_vdpa" = "yes" && test "$vhost_vdpa" = "no"; then
+  error_exit "--enable-vhost-net-vdpa requires --enable-vhost-vdpa"
+fi
 
  # OR the vhost-kernel and vhost-user values for simplicity
  if test "$vhost_net" = ""; then
@@ -6673,6 +6687,7 @@ echo "vhost-scsi support $vhost_scsi"
  echo "vhost-vsock support $vhost_vsock"
  echo "vhost-user support $vhost_user"
  echo "vhost-user-fs support $vhost_user_fs"
+echo "vhost-vdpa support $vhost_vdpa"
  echo "Trace backends    $trace_backends"
  if have_backend "simple"; then
  echo "Trace output file $trace_file-<pid>"
@@ -7170,6 +7185,9 @@ fi
  if test "$vhost_net_user" = "yes" ; then
    echo "CONFIG_VHOST_NET_USER=y" >> $config_host_mak
  fi
+if test "$vhost_net_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_NET_VDPA=y" >> $config_host_mak
+fi
  if test "$vhost_crypto" = "yes" ; then
    echo "CONFIG_VHOST_CRYPTO=y" >> $config_host_mak
  fi
@@ -7182,6 +7200,9 @@ fi
  if test "$vhost_user" = "yes" ; then
    echo "CONFIG_VHOST_USER=y" >> $config_host_mak
  fi
+if test "$vhost_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_VDPA=y" >> $config_host_mak
+fi
  if test "$vhost_user_fs" = "yes" ; then
    echo "CONFIG_VHOST_USER_FS=y" >> $config_host_mak
  fi
hw/net/vhost_net-stub.c +5 -0
index 43e93e1a9a..ab77a92a7d 100644
@@ -94,3 +94,8 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      return 0;
  }
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return 0;
+}
+
hw/net/vhost_net.c +16 -0
index e2bc7de2eb..25045cff59 100644
@@ -17,8 +17,10 @@
  #include "net/net.h"
  #include "net/tap.h"
  #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
  #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
  #include "hw/virtio/virtio-net.h"
  #include "net/vhost_net.h"
  #include "qemu/error-report.h"
@@ -85,6 +87,30 @@ static const int user_feature_bits[] = {
      VHOST_INVALID_FEATURE_BIT
  };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,

hw/net/vhost_net.c static const int user_feature_bits[] = { · line 113

Quoted 10 lines
hw/net/vhost_net.c +10 -0
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_F_RING_PACKED,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};

I think those feature bits should belong to net/vhost-vdpa.c, since it contains bits that are net specific.

Quoted 26 lines
hw/net/vhost_net.c +7 -1
  static const int *vhost_net_get_feature_bits(struct vhost_net *net)
  {
      const int *feature_bits = 0;
@@ -96,6 +122,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
      case NET_CLIENT_DRIVER_VHOST_USER:
          feature_bits = user_feature_bits;
          break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
      default:
          error_report("Feature bits not defined for this type: %d",
                  net->nc->info->type);
@@ -110,7 +139,10 @@ uint64_t vhost_net_get_features(struct vhost_net *net, uint64_t features)
      return vhost_get_features(&net->dev, vhost_net_get_feature_bits(net),
              features);
  }
-
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return vhost_dev_get_device_id(&net->dev, device_id);
+}
  void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
  {
      net->dev.acked_features = net->dev.backend_features;

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, · line 375

Quoted 7 lines
hw/net/vhost_net.c +4 -0
@@ -337,6 +369,11 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
      }
 
      for (i = 0; i < total_queues; i++) {
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }

So this may break vpda_sim since it call set_vq_ready() before set_vring_addr.

I think maybe it's better not introduce vhost_set_vring_ready() but enable virtqueue in vhost_vdpa_set_state() before setting DRIVER_OK.

Quoted 18 lines
hw/net/vhost_net.c +7 -0
+
          peer = qemu_get_peer(ncs, i);
          r = vhost_net_start_one(get_vhost_net(peer), dev);
 
@@ -433,6 +470,12 @@ VHostNetState *get_vhost_net(NetClientState *nc)
          vhost_net = vhost_user_get_vhost_net(nc);
          assert(vhost_net);
          break;
+#endif
+#ifdef CONFIG_VHOST_NET_VDPA
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
  #endif
      default:
          break;

hw/net/vhost_net.c int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) · line 521

Quoted 5 lines
hw/net/vhost_net.c +2 -0
@@ -474,3 +517,5 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
      return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
  }
+
+

One extra newline.

Quoted 213 lines
hw/virtio/Makefile.objs +1 -0
index 4e4d39a0a4..6b1b1a5fce 100644
@@ -5,6 +5,7 @@ obj-y += virtio.o
  obj-$(CONFIG_VHOST) += vhost.o vhost-backend.o
  common-obj-$(call lnot,$(CONFIG_VHOST)) += vhost-stub.o
  obj-$(CONFIG_VHOST_USER) += vhost-user.o
+obj-$(CONFIG_VHOST_VDPA) += vhost-vdpa.o
 
  common-obj-$(CONFIG_VIRTIO_RNG) += virtio-rng.o
  common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
hw/virtio/vhost-backend.c +5 -0
index 42efb4967b..420341e8c5 100644
@@ -291,6 +291,11 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
      case VHOST_BACKEND_TYPE_USER:
          dev->vhost_ops = &user_ops;
          break;
+#endif
+#ifdef CONFIG_VHOST_VDPA
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
  #endif
      default:
          error_report("Unknown vhost backend type");
hw/virtio/vhost-vdpa.c new file +179 -0
new file mode 100644
index 0000000000..2d136a8565
@@ -0,0 +1,399 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>
+#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/main-loop.h"
+#include <linux/kvm.h>
+#include "sysemu/kvm.h"
+
+
+static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings to
+            * addresses in the upper part of the 64-bit address space.  These
+            * are never accessed by the CPU and beyond the address width of
+            * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU width.
+            */
+           section->offset_within_address_space & (1ULL << 63);
+}
+
+static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
+                              void *vaddr, bool readonly)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.uaddr = (uint64_t)vaddr;
+    msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
+    msg.iotlb.type = VHOST_IOTLB_UPDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
+                                hwaddr size)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static void vhost_vdpa_listener_region_add(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
+                             vaddr, section->readonly);
+    if (ret) {
+        error_report("vhost vdpa map fail!");
+        if (memory_region_is_ram_device(section->mr)) {
+            /* Allow unexpected mappings not to be fatal for RAM devices */
+            error_report("map ram fail!");
+          return ;
+        }
+        goto fail;
+    }
+
+    return;
+
+fail:
+    if (memory_region_is_ram_device(section->mr)) {
+        error_report("failed to vdpa_dma_map. pci p2p may not work");
+        return;
+
+    }
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    error_report("vhost-vdpa: DMA mapping failed, unable to continue");
+    return;
+
+}
+
+static void vhost_vdpa_listener_region_del(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    int ret;
+    bool try_unmap = true;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    llsize = int128_sub(llend, int128_make64(iova));

hw/virtio/vhost-vdpa.c line 189

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+
+    if (try_unmap) {
+        ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
+        if (ret) {
+            error_report("vhost_vdpa dma unmap error!");
+        }
+    }
+
+    memory_region_unref(section->mr);
+}

newline is needed here.

hw/virtio/vhost-vdpa.c line 192

Quoted 3 lines
hw/virtio/vhost-vdpa.c new file +3 -0
+/* Register a new memory listener, only to get diffs from qemu,
+ * this help to reduce the tricky codes in vhost
+ * (e.g generating diffs of two rbtree as usnic did).*/

This comment needs some improvement. How about:

/* IOTLB API is used by vhost-vpda which requires incremental updating of the mapping. So we can not use generic vhost memory listener which depends on the addnop(). */

Quoted 31 lines
hw/virtio/vhost-vdpa.c new file +31 -0
+static const MemoryListener vhost_vdpa_memory_listener = {
+    .region_add = vhost_vdpa_listener_region_add,
+    .region_del = vhost_vdpa_listener_region_del,
+};
+
+static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
+                             void *arg)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    int fd = v->device_fd;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    return ioctl(fd, request, arg);
+}
+
+static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
+{
+    uint8_t s;
+
+    if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) {
+        return;
+    }
+
+    s |= status;
+
+    vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
+}
+
+static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
+{

hw/virtio/vhost-vdpa.c line 233

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    struct vhost_vdpa *v;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = opaque;
+    dev->opaque =  opaque ;
+
+    v->listener = vhost_vdpa_memory_listener;
+    v->msg_type = VHOST_IOTLB_MSG_V2;
+    memory_listener_register(&v->listener, &address_space_memory);

Let's move the memory listener register/unregister to vhost_vdpa_set_state(). Then we can avoid lots of unnecessary vhost IOTLB transactions before DRIVER_OK which vhost-vDPA doesn't care.

Quoted 109 lines
hw/virtio/vhost-vdpa.c new file +109 -0
+
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                               VIRTIO_CONFIG_S_DRIVER);
+
+    return 0;
+}
+
+static int vhost_vdpa_cleanup(struct vhost_dev *dev)
+{
+    struct vhost_vdpa *v;
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = dev->opaque;
+    memory_listener_unregister(&v->listener);
+
+    dev->opaque = NULL;
+    return 0;
+}
+
+static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
+{
+    return INT_MAX;
+}
+
+static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+
+    if (mem->padding) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_vdpa_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    int ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
+    uint8_t status = 0;
+
+    if (ret) {
+        return ret;
+    }
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
+    vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+    return !(status & VIRTIO_CONFIG_S_FEATURES_OK);
+}
+
+int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id)
+{
+    return vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
+}
+
+static int vhost_vdpa_reset_device(struct vhost_dev *dev)
+{
+    uint8_t status = 0;
+
+    return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
+}
+
+static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
+{
+    int i;
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num = 1,
+        };
+        vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
+    }
+    return 0;
+}
+
+static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags)
+{
+    struct vhost_vdpa_config config;
+    int ret;
+    if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {
+        return -1;
+    }
+    memset(&config, 0, sizeof(struct vhost_vdpa_config));
+    config.off = 0;
+    config.len = size;
+    memcpy(&config.buf, data, size);
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, &config);
+    return ret;
+}
+
+static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
+                                   uint32_t config_len)
+{
+    struct vhost_vdpa_config v_config;
+    int ret;
+
+    memset(&v_config, 0, sizeof(struct vhost_vdpa_config));
+    if (config == NULL) {
+        return -1;

hw/virtio/vhost-vdpa.c line 352

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    }
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, &v_config);
+    if ((v_config.len > config_len) || (v_config.len == 0)) {
+        return -EINVAL;
+    }
+    memcpy(config, &v_config.buf, config_len);
+    return ret;
+ }
+
+static int vhost_vdpa_set_state(struct vhost_dev *dev, bool started)

We probably need a better name, e.g vhost_vdpa_start()?

Quoted 35 lines
hw/virtio/vhost-vdpa.c new file +35 -0
+{
+    if (started) {
+        uint8_t status = 0;
+
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
+        vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+        return !(status & VIRTIO_CONFIG_S_DRIVER_OK);
+    } else {
+        vhost_vdpa_reset_device(dev);
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                                   VIRTIO_CONFIG_S_DRIVER);
+        return 0;
+    }
+}
+
+const VhostOps vdpa_ops = {
+        .backend_type = VHOST_BACKEND_TYPE_VDPA,
+        .vhost_backend_init = vhost_vdpa_init,
+        .vhost_backend_cleanup = vhost_vdpa_cleanup,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_set_vring_endian = NULL,
+        .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
+        .vhost_set_mem_table = vhost_vdpa_set_mem_table,
+        .vhost_set_features = vhost_vdpa_set_features,
+        .vhost_reset_device = vhost_vdpa_reset_device,
+        .vhost_get_vq_index = vhost_vdpa_get_vq_index,

hw/virtio/vhost-vdpa.c line 397

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
+        .vhost_get_config  = vhost_vdpa_get_config,
+        .vhost_set_config = vhost_vdpa_set_config,
+        .vhost_requires_shm_log = NULL,
+        .vhost_migration_done = NULL,
+        .vhost_backend_can_merge = NULL,
+        .vhost_net_set_mtu = NULL,
+        .vhost_set_iotlb_callback = NULL,
+        .vhost_send_device_iotlb_msg = NULL,
+        .vhost_set_state = vhost_vdpa_set_state,

Since it only accept boolean parameter I guess vhost_dev_start() is better?

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 759

Quoted 11 lines
hw/virtio/vhost-vdpa.c new file +2 -0
+        .vhost_get_device_id = vhost_vdpa_get_device_id,
+};
hw/virtio/vhost.c +1 -0
index 01ebe12f28..b97aa02a4c 100644
@@ -756,6 +756,12 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
          .log_guest_addr = vq->used_phys,
          .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
      };
+    /*vDPA need to use the phys address here to set to hardware*/

Actually it's "IOVA" instead of "phys address".

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 764

Quoted 5 lines
hw/virtio/vhost.c +5 -0
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }

Let's introduce a callback here instead of such hard-coded ones.

Quoted 19 lines
hw/virtio/vhost.c +8 -0
      int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
      if (r < 0) {
          VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
@@ -1506,6 +1512,14 @@ int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
      return -1;
  }
 
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id)
+{
+    assert(hdev->vhost_ops);
+    if (hdev->vhost_ops->vhost_get_device_id) {
+        return hdev->vhost_ops->vhost_get_device_id(hdev, device_id);
+    }
+    return -1;
+}
  void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
                                     const VhostDevConfigOps *ops)
  {

hw/virtio/vhost.c int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) · line 1681

Quoted 8 lines
hw/virtio/vhost.c +4 -1
@@ -1661,7 +1675,13 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
          }
      }
 
-    if (vhost_dev_has_iommu(hdev)) {
+    r = vhost_set_state(hdev, true);
+    if (r) {
+        goto fail_log;
+    }

Please use a separate patch for introducing vhost_set_state().

hw/virtio/vhost.c int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) · line 1684

Quoted 3 lines
hw/virtio/vhost.c +3 -0
+
+    if (vhost_dev_has_iommu(hdev) &&
+        hdev->vhost_ops->vhost_set_iotlb_callback) {

A new patch for checking vhost_set_iotlb_callback(), or just implement it for vhost-vpda that just warn about IOTLB miss.

Quoted 143 lines
hw/virtio/vhost.c +14 -1
          hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
 
          /* Update used ring information for IOTLB to work correctly,
@@ -1697,6 +1717,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
      /* should only be called after backend is connected */
      assert(hdev->vhost_ops);
 
+    vhost_set_state(hdev, false);
+
      for (i = 0; i < hdev->nvqs; ++i) {
          vhost_virtqueue_stop(hdev,
                               vdev,
@@ -1705,7 +1727,9 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
      }
 
      if (vhost_dev_has_iommu(hdev)) {
-        hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        if (hdev->vhost_ops->vhost_set_iotlb_callback) {
+            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        }
          memory_listener_unregister(&hdev->iommu_listener);
      }
      vhost_log_put(hdev, true);
@@ -1722,3 +1746,12 @@ int vhost_net_set_backend(struct vhost_dev *hdev,
 
      return -1;
  }
+
+int vhost_set_state(struct vhost_dev *hdev, bool started)
+{
+    if (hdev->vhost_ops->vhost_set_state) {
+        return hdev->vhost_ops->vhost_set_state(hdev, started);
+    }
+
+    return 0;
+}
include/hw/virtio/vhost-backend.h +9 -1
index 300b59c172..1ebe3785cf 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
      VHOST_BACKEND_TYPE_NONE = 0,
      VHOST_BACKEND_TYPE_KERNEL = 1,
      VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
  } VhostBackendType;
 
  typedef enum VhostSetConfigType {
@@ -77,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
  typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
  typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                           int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
  typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
  typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                         char *mac_addr);
@@ -112,6 +114,8 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
  typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                          struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, bool started);
+typedef int (*vhost_get_device_id_op)(struct vhost_dev *dev, uint32_t *dev_id);
  typedef struct VhostOps {
      VhostBackendType backend_type;
      vhost_backend_init vhost_backend_init;
@@ -138,6 +142,7 @@ typedef struct VhostOps {
      vhost_reset_device_op vhost_reset_device;
      vhost_get_vq_index_op vhost_get_vq_index;
      vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
      vhost_requires_shm_log_op vhost_requires_shm_log;
      vhost_migration_done_op vhost_migration_done;
      vhost_backend_can_merge_op vhost_backend_can_merge;
@@ -152,9 +157,12 @@ typedef struct VhostOps {
      vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
      vhost_get_inflight_fd_op vhost_get_inflight_fd;
      vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
+    vhost_get_device_id_op vhost_get_device_id;
  } VhostOps;
 
  extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
  int vhost_set_backend_type(struct vhost_dev *dev,
                             VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +26 -0
new file mode 100644
index 0000000000..6455663388
@@ -0,0 +1,26 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    uint32_t msg_type;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+extern int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id);
+#endif
include/hw/virtio/vhost.h +2 -0
index 085450c6f8..b682545f51 100644
@@ -124,6 +124,7 @@ int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config,
                           uint32_t config_len);
  int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *data,
                           uint32_t offset, uint32_t size, uint32_t flags);
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id);
  /* notifier callback in case vhost device config space changed
   */
  void vhost_dev_set_config_notifier(struct vhost_dev *dev,
@@ -137,4 +138,5 @@ int vhost_dev_set_inflight(struct vhost_dev *dev,
                             struct vhost_inflight *inflight);
  int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
                             struct vhost_inflight *inflight);
+int vhost_set_state(struct vhost_dev *dev, bool started);
  #endif
include/net/vhost_net.h +0 -0
index 8a6f208189..56e67fe164 100644

include/net/vhost_net.h int vhost_set_vring_ready(NetClientState *nc); · line 43

Quoted 6 lines
include/net/vhost_net.h +1 -2
@@ -40,5 +40,5 @@ int vhost_set_vring_ready(NetClientState *nc);
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
-
-#endif
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);

This should belong to vhost-vdpa header.

Thanks

Quoted 41 lines
include/net/vhost_net.h +1 -0
+endif
qemu-options.hx +15 -0
index 292d4e7c0c..c19e10ce9c 100644
@@ -2409,6 +2409,10 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
  #ifdef CONFIG_POSIX
      "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
      "                configure a vhost-user network, backed by a chardev 'dev'\n"
+#endif
+#ifdef CONFIG_POSIX
+    "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
+    "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
  #endif
      "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
      "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
@@ -2428,6 +2432,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
  #endif
  #ifdef CONFIG_POSIX
      "vhost-user|"
+#endif
+#ifdef CONFIG_POSIX
+    "vhost-vdpa|"
  #endif
      "socket][,option][,...][mac=macaddr]\n"
      "                initialize an on-board / default host NIC (using MAC address\n"
@@ -2896,6 +2903,14 @@ SRST
      hubport to another netdev with ID nd by using the ``netdev=nd``
      option.
 
+``-netdev vhost-vdpa,vhostdev=/path/to/dev ``
+    Establish a vhost-vdpa netdev.
+
+    vDPA device is a device that uses a datapath which complies with
+    the virtio specifications with a vendor specific control path.
+    vDPA devices can be both physically located on the hardware or
+    emulated by software.
+
  ``-net nic[,netdev=nd][,macaddr=mac][,model=type] [,name=name][,addr=addr][,vectors=v]``
      Legacy option to configure or create an on-board (or machine
      default) Network Interface Card(NIC) and connect it either to the
#20 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu
Quoted 1 line · Cindy Lu

From: Tiwei Bie<[email protected]>

Consider the significant modification based on the original patch.

I think you may change the other to yourslef and keep the sobs for both Tiwei and Lingshan.

Thanks

Quoted 16 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
author
#19 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Jason Wang

Hi Jason,

Quoted 163 lines · Jason Wang
Quoted 160 lines · Cindy Lu

From: Tiwei Bie <[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
---
  configure                         |  21 ++
  hw/net/vhost_net-stub.c           |   5 +
  hw/net/vhost_net.c                |  47 +++-
  hw/virtio/Makefile.objs           |   1 +
  hw/virtio/vhost-backend.c         |   5 +
  hw/virtio/vhost-vdpa.c            | 399 ++++++++++++++++++++++++++++++
  hw/virtio/vhost.c                 |  37 ++-
  include/hw/virtio/vhost-backend.h |  10 +-
  include/hw/virtio/vhost-vdpa.h    |  26 ++
  include/hw/virtio/vhost.h         |   2 +
  include/net/vhost_net.h           |   4 +-
  qemu-options.hx                   |  15 ++
  12 files changed, 566 insertions(+), 6 deletions(-)

create mode 100644 hw/virtio/vhost-vdpa.c create mode 100644 include/hw/virtio/vhost-vdpa.h

configure +21 -0
index 23b5e93752..53679ee57f 100755
@@ -1557,6 +1557,10 @@ for opt do
    ;;
    --enable-vhost-user) vhost_user="yes"
    ;;
+  --disable-vhost-vdpa) vhost_vdpa="no"
+  ;;
+  --enable-vhost-vdpa) vhost_vdpa="yes"
+  ;;
    --disable-vhost-kernel) vhost_kernel="no"
    ;;
    --enable-vhost-kernel) vhost_kernel="yes"
@@ -1846,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available:
    vhost-crypto    vhost-user-crypto backend support
    vhost-kernel    vhost kernel backend support
    vhost-user      vhost-user backend support
+  vhost-vdpa      vhost-vdpa kernel backend support
    spice           spice
    rbd             rados block device (rbd)
    libiscsi        iscsi support
@@ -2336,6 +2341,10 @@ test "$vhost_user" = "" && vhost_user=yes
  if test "$vhost_user" = "yes" && test "$mingw32" = "yes"; then
    error_exit "vhost-user isn't available on win32"
  fi
+test "$vhost_vdpa" = "" && vhost_vdpa=$linux
+if test "$vhost_vdpa" = "yes" && test "$linux" != "yes"; then
+  error_exit "vhost-vdpa is only available on Linux"
+fi
  test "$vhost_kernel" = "" && vhost_kernel=$linux
  if test "$vhost_kernel" = "yes" && test "$linux" != "yes"; then
    error_exit "vhost-kernel is only available on Linux"
@@ -2364,6 +2373,11 @@ test "$vhost_user_fs" = "" && vhost_user_fs=$vhost_user
  if test "$vhost_user_fs" = "yes" && test "$vhost_user" = "no"; then
    error_exit "--enable-vhost-user-fs requires --enable-vhost-user"
  fi
+#vhost-vdpa backends
+test "$vhost_net_vdpa" = "" && vhost_net_vdpa=$vhost_vdpa
+if test "$vhost_net_vdpa" = "yes" && test "$vhost_vdpa" = "no"; then
+  error_exit "--enable-vhost-net-vdpa requires --enable-vhost-vdpa"
+fi
 
  # OR the vhost-kernel and vhost-user values for simplicity
  if test "$vhost_net" = ""; then
@@ -6673,6 +6687,7 @@ echo "vhost-scsi support $vhost_scsi"
  echo "vhost-vsock support $vhost_vsock"
  echo "vhost-user support $vhost_user"
  echo "vhost-user-fs support $vhost_user_fs"
+echo "vhost-vdpa support $vhost_vdpa"
  echo "Trace backends    $trace_backends"
  if have_backend "simple"; then
  echo "Trace output file $trace_file-<pid>"
@@ -7170,6 +7185,9 @@ fi
  if test "$vhost_net_user" = "yes" ; then
    echo "CONFIG_VHOST_NET_USER=y" >> $config_host_mak
  fi
+if test "$vhost_net_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_NET_VDPA=y" >> $config_host_mak
+fi
  if test "$vhost_crypto" = "yes" ; then
    echo "CONFIG_VHOST_CRYPTO=y" >> $config_host_mak
  fi
@@ -7182,6 +7200,9 @@ fi
  if test "$vhost_user" = "yes" ; then
    echo "CONFIG_VHOST_USER=y" >> $config_host_mak
  fi
+if test "$vhost_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_VDPA=y" >> $config_host_mak
+fi
  if test "$vhost_user_fs" = "yes" ; then
    echo "CONFIG_VHOST_USER_FS=y" >> $config_host_mak
  fi
hw/net/vhost_net-stub.c +5 -0
index 43e93e1a9a..ab77a92a7d 100644
@@ -94,3 +94,8 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      return 0;
  }
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return 0;
+}
+
hw/net/vhost_net.c +22 -0
index e2bc7de2eb..25045cff59 100644
@@ -17,8 +17,10 @@
  #include "net/net.h"
  #include "net/tap.h"
  #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
  #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
  #include "hw/virtio/virtio-net.h"
  #include "net/vhost_net.h"
  #include "qemu/error-report.h"
@@ -85,6 +87,30 @@ static const int user_feature_bits[] = {
      VHOST_INVALID_FEATURE_BIT
  };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
Quoted 9 lines

hw/net/vhost_net.c static const int user_feature_bits[] = { · line 113

Quoted 4 lines
hw/net/vhost_net.c +4 -0
+    VIRTIO_F_RING_PACKED,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};

I think those feature bits should belong to net/vhost-vdpa.c, since it contains bits that are net specific.

Sure will move these.

Quoted 39 lines
Quoted 27 lines
hw/net/vhost_net.c +7 -1
  static const int *vhost_net_get_feature_bits(struct vhost_net *net)
  {
      const int *feature_bits = 0;
@@ -96,6 +122,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
      case NET_CLIENT_DRIVER_VHOST_USER:
          feature_bits = user_feature_bits;
          break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
      default:
          error_report("Feature bits not defined for this type: %d",
                  net->nc->info->type);
@@ -110,7 +139,10 @@ uint64_t vhost_net_get_features(struct vhost_net *net, uint64_t features)
      return vhost_get_features(&net->dev, vhost_net_get_feature_bits(net),
              features);
  }
-
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return vhost_dev_get_device_id(&net->dev, device_id);
+}
  void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
  {
      net->dev.acked_features = net->dev.backend_features;

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, · line 375

Quoted 7 lines
hw/net/vhost_net.c +4 -0
@@ -337,6 +369,11 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
      }
 
      for (i = 0; i < total_queues; i++) {
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }

So this may break vpda_sim since it call set_vq_ready() before set_vring_addr.

Quoted 4 lines

I think maybe it's better not introduce vhost_set_vring_ready() but enable virtqueue in vhost_vdpa_set_state() before setting DRIVER_OK.

will fix this

Quoted 19 lines
Quoted 19 lines
hw/net/vhost_net.c +7 -0
+
          peer = qemu_get_peer(ncs, i);
          r = vhost_net_start_one(get_vhost_net(peer), dev);
 
@@ -433,6 +470,12 @@ VHostNetState *get_vhost_net(NetClientState *nc)
          vhost_net = vhost_user_get_vhost_net(nc);
          assert(vhost_net);
          break;
+#endif
+#ifdef CONFIG_VHOST_NET_VDPA
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
  #endif
      default:
          break;
@@ -474,3 +517,5 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
Quoted 8 lines

hw/net/vhost_net.c int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) · line 521

Quoted 4 lines
hw/net/vhost_net.c +2 -0
      return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
  }
+
+

One extra newline.

will fix this

Quoted 219 lines
Quoted 219 lines
hw/virtio/Makefile.objs +1 -0
index 4e4d39a0a4..6b1b1a5fce 100644
@@ -5,6 +5,7 @@ obj-y += virtio.o
  obj-$(CONFIG_VHOST) += vhost.o vhost-backend.o
  common-obj-$(call lnot,$(CONFIG_VHOST)) += vhost-stub.o
  obj-$(CONFIG_VHOST_USER) += vhost-user.o
+obj-$(CONFIG_VHOST_VDPA) += vhost-vdpa.o
 
  common-obj-$(CONFIG_VIRTIO_RNG) += virtio-rng.o
  common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
hw/virtio/vhost-backend.c +5 -0
index 42efb4967b..420341e8c5 100644
@@ -291,6 +291,11 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
      case VHOST_BACKEND_TYPE_USER:
          dev->vhost_ops = &user_ops;
          break;
+#endif
+#ifdef CONFIG_VHOST_VDPA
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
  #endif
      default:
          error_report("Unknown vhost backend type");
hw/virtio/vhost-vdpa.c new file +184 -0
new file mode 100644
index 0000000000..2d136a8565
@@ -0,0 +1,399 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>
+#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/main-loop.h"
+#include <linux/kvm.h>
+#include "sysemu/kvm.h"
+
+
+static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings to
+            * addresses in the upper part of the 64-bit address space.  These
+            * are never accessed by the CPU and beyond the address width of
+            * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU width.
+            */
+           section->offset_within_address_space & (1ULL << 63);
+}
+
+static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
+                              void *vaddr, bool readonly)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.uaddr = (uint64_t)vaddr;
+    msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
+    msg.iotlb.type = VHOST_IOTLB_UPDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
+                                hwaddr size)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static void vhost_vdpa_listener_region_add(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
+                             vaddr, section->readonly);
+    if (ret) {
+        error_report("vhost vdpa map fail!");
+        if (memory_region_is_ram_device(section->mr)) {
+            /* Allow unexpected mappings not to be fatal for RAM devices */
+            error_report("map ram fail!");
+          return ;
+        }
+        goto fail;
+    }
+
+    return;
+
+fail:
+    if (memory_region_is_ram_device(section->mr)) {
+        error_report("failed to vdpa_dma_map. pci p2p may not work");
+        return;
+
+    }
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    error_report("vhost-vdpa: DMA mapping failed, unable to continue");
+    return;
+
+}
+
+static void vhost_vdpa_listener_region_del(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    int ret;
+    bool try_unmap = true;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    if (try_unmap) {
+        ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
+        if (ret) {
+            error_report("vhost_vdpa dma unmap error!");
Quoted 9 lines

hw/virtio/vhost-vdpa.c line 189

Quoted 5 lines
hw/virtio/vhost-vdpa.c new file +5 -0
+        }
+    }
+
+    memory_region_unref(section->mr);
+}

newline is needed here.

will fix this

Quoted 51 lines

hw/virtio/vhost-vdpa.c line 192

Quoted 4 lines
hw/virtio/vhost-vdpa.c new file +3 -0
+/* Register a new memory listener, only to get diffs from qemu,
+ * this help to reduce the tricky codes in vhost
+ * (e.g generating diffs of two rbtree as usnic did).*/

This comment needs some improvement. How about:

/* IOTLB API is used by vhost-vpda which requires incremental updating of the mapping. So we can not use generic vhost memory listener which depends on the addnop(). */

Quoted 38 lines
hw/virtio/vhost-vdpa.c new file +38 -0
+static const MemoryListener vhost_vdpa_memory_listener = {
+    .region_add = vhost_vdpa_listener_region_add,
+    .region_del = vhost_vdpa_listener_region_del,
+};
+
+static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
+                             void *arg)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    int fd = v->device_fd;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    return ioctl(fd, request, arg);
+}
+
+static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
+{
+    uint8_t s;
+
+    if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) {
+        return;
+    }
+
+    s |= status;
+
+    vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
+}
+
+static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
+{
+    struct vhost_vdpa *v;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = opaque;
+    dev->opaque =  opaque ;
+
Quoted 10 lines

hw/virtio/vhost-vdpa.c line 233

Quoted 3 lines
hw/virtio/vhost-vdpa.c new file +3 -0
+    v->listener = vhost_vdpa_memory_listener;
+    v->msg_type = VHOST_IOTLB_MSG_V2;
+    memory_listener_register(&v->listener, &address_space_memory);

Let's move the memory listener register/unregister to vhost_vdpa_set_state(). Then we can avoid lots of unnecessary vhost IOTLB transactions before DRIVER_OK which vhost-vDPA doesn't care.

sure will move this

Quoted 114 lines
Quoted 114 lines
hw/virtio/vhost-vdpa.c new file +114 -0
+
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                               VIRTIO_CONFIG_S_DRIVER);
+
+    return 0;
+}
+
+static int vhost_vdpa_cleanup(struct vhost_dev *dev)
+{
+    struct vhost_vdpa *v;
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = dev->opaque;
+    memory_listener_unregister(&v->listener);
+
+    dev->opaque = NULL;
+    return 0;
+}
+
+static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
+{
+    return INT_MAX;
+}
+
+static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+
+    if (mem->padding) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_vdpa_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    int ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
+    uint8_t status = 0;
+
+    if (ret) {
+        return ret;
+    }
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
+    vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+    return !(status & VIRTIO_CONFIG_S_FEATURES_OK);
+}
+
+int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id)
+{
+    return vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
+}
+
+static int vhost_vdpa_reset_device(struct vhost_dev *dev)
+{
+    uint8_t status = 0;
+
+    return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
+}
+
+static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
+{
+    int i;
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num = 1,
+        };
+        vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
+    }
+    return 0;
+}
+
+static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags)
+{
+    struct vhost_vdpa_config config;
+    int ret;
+    if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {
+        return -1;
+    }
+    memset(&config, 0, sizeof(struct vhost_vdpa_config));
+    config.off = 0;
+    config.len = size;
+    memcpy(&config.buf, data, size);
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, &config);
+    return ret;
+}
+
+static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
+                                   uint32_t config_len)
+{
+    struct vhost_vdpa_config v_config;
+    int ret;
+
+    memset(&v_config, 0, sizeof(struct vhost_vdpa_config));
+    if (config == NULL) {
+        return -1;
+    }
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, &v_config);
+    if ((v_config.len > config_len) || (v_config.len == 0)) {
+        return -EINVAL;
+    }
Quoted 10 lines

hw/virtio/vhost-vdpa.c line 352

Quoted 5 lines
hw/virtio/vhost-vdpa.c new file +5 -0
+    memcpy(config, &v_config.buf, config_len);
+    return ret;
+ }
+
+static int vhost_vdpa_set_state(struct vhost_dev *dev, bool started)

We probably need a better name, e.g vhost_vdpa_start()?

will fix this

Quoted 40 lines
Quoted 40 lines
hw/virtio/vhost-vdpa.c new file +40 -0
+{
+    if (started) {
+        uint8_t status = 0;
+
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
+        vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+        return !(status & VIRTIO_CONFIG_S_DRIVER_OK);
+    } else {
+        vhost_vdpa_reset_device(dev);
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                                   VIRTIO_CONFIG_S_DRIVER);
+        return 0;
+    }
+}
+
+const VhostOps vdpa_ops = {
+        .backend_type = VHOST_BACKEND_TYPE_VDPA,
+        .vhost_backend_init = vhost_vdpa_init,
+        .vhost_backend_cleanup = vhost_vdpa_cleanup,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_set_vring_endian = NULL,
+        .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
+        .vhost_set_mem_table = vhost_vdpa_set_mem_table,
+        .vhost_set_features = vhost_vdpa_set_features,
+        .vhost_reset_device = vhost_vdpa_reset_device,
+        .vhost_get_vq_index = vhost_vdpa_get_vq_index,
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
+        .vhost_get_config  = vhost_vdpa_get_config,
+        .vhost_set_config = vhost_vdpa_set_config,
+        .vhost_requires_shm_log = NULL,
+        .vhost_migration_done = NULL,
Quoted 9 lines

hw/virtio/vhost-vdpa.c line 397

Quoted 5 lines
hw/virtio/vhost-vdpa.c new file +5 -0
+        .vhost_backend_can_merge = NULL,
+        .vhost_net_set_mtu = NULL,
+        .vhost_set_iotlb_callback = NULL,
+        .vhost_send_device_iotlb_msg = NULL,
+        .vhost_set_state = vhost_vdpa_set_state,

Since it only accept boolean parameter I guess vhost_dev_start() is better?

will fix this

Quoted 17 lines

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 759

Quoted 12 lines
hw/virtio/vhost-vdpa.c new file +2 -0
+        .vhost_get_device_id = vhost_vdpa_get_device_id,
+};
hw/virtio/vhost.c +1 -0
index 01ebe12f28..b97aa02a4c 100644
@@ -756,6 +756,12 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
          .log_guest_addr = vq->used_phys,
          .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
      };
+    /*vDPA need to use the phys address here to set to hardware*/

Actually it's "IOVA" instead of "phys address".

Quoted 9 lines

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 764

Quoted 5 lines
hw/virtio/vhost.c +5 -0
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }

Let's introduce a callback here instead of such hard-coded ones.

will fix this

Quoted 23 lines
Quoted 23 lines
hw/virtio/vhost.c +8 -0
      int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
      if (r < 0) {
          VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
@@ -1506,6 +1512,14 @@ int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
      return -1;
  }
 
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id)
+{
+    assert(hdev->vhost_ops);
+    if (hdev->vhost_ops->vhost_get_device_id) {
+        return hdev->vhost_ops->vhost_get_device_id(hdev, device_id);
+    }
+    return -1;
+}
  void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
                                     const VhostDevConfigOps *ops)
  {
@@ -1661,7 +1675,13 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
          }
      }
 
Quoted 9 lines

hw/virtio/vhost.c int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) · line 1681

Quoted 5 lines
hw/virtio/vhost.c +4 -1
-    if (vhost_dev_has_iommu(hdev)) {
+    r = vhost_set_state(hdev, true);
+    if (r) {
+        goto fail_log;
+    }

Please use a separate patch for introducing vhost_set_state().

will fix this

Quoted 163 lines

hw/virtio/vhost.c int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev) · line 1684

Quoted 4 lines
hw/virtio/vhost.c +3 -0
+
+    if (vhost_dev_has_iommu(hdev) &&
+        hdev->vhost_ops->vhost_set_iotlb_callback) {

A new patch for checking vhost_set_iotlb_callback(), or just implement it for vhost-vpda that just warn about IOTLB miss.

Quoted 143 lines
hw/virtio/vhost.c +14 -1
          hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
 
          /* Update used ring information for IOTLB to work correctly,
@@ -1697,6 +1717,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
      /* should only be called after backend is connected */
      assert(hdev->vhost_ops);
 
+    vhost_set_state(hdev, false);
+
      for (i = 0; i < hdev->nvqs; ++i) {
          vhost_virtqueue_stop(hdev,
                               vdev,
@@ -1705,7 +1727,9 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
      }
 
      if (vhost_dev_has_iommu(hdev)) {
-        hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        if (hdev->vhost_ops->vhost_set_iotlb_callback) {
+            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        }
          memory_listener_unregister(&hdev->iommu_listener);
      }
      vhost_log_put(hdev, true);
@@ -1722,3 +1746,12 @@ int vhost_net_set_backend(struct vhost_dev *hdev,
 
      return -1;
  }
+
+int vhost_set_state(struct vhost_dev *hdev, bool started)
+{
+    if (hdev->vhost_ops->vhost_set_state) {
+        return hdev->vhost_ops->vhost_set_state(hdev, started);
+    }
+
+    return 0;
+}
include/hw/virtio/vhost-backend.h +9 -1
index 300b59c172..1ebe3785cf 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
      VHOST_BACKEND_TYPE_NONE = 0,
      VHOST_BACKEND_TYPE_KERNEL = 1,
      VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
  } VhostBackendType;
 
  typedef enum VhostSetConfigType {
@@ -77,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
  typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
  typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                           int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
  typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
  typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                         char *mac_addr);
@@ -112,6 +114,8 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
  typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                          struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, bool started);
+typedef int (*vhost_get_device_id_op)(struct vhost_dev *dev, uint32_t *dev_id);
  typedef struct VhostOps {
      VhostBackendType backend_type;
      vhost_backend_init vhost_backend_init;
@@ -138,6 +142,7 @@ typedef struct VhostOps {
      vhost_reset_device_op vhost_reset_device;
      vhost_get_vq_index_op vhost_get_vq_index;
      vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
      vhost_requires_shm_log_op vhost_requires_shm_log;
      vhost_migration_done_op vhost_migration_done;
      vhost_backend_can_merge_op vhost_backend_can_merge;
@@ -152,9 +157,12 @@ typedef struct VhostOps {
      vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
      vhost_get_inflight_fd_op vhost_get_inflight_fd;
      vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
+    vhost_get_device_id_op vhost_get_device_id;
  } VhostOps;
 
  extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
  int vhost_set_backend_type(struct vhost_dev *dev,
                             VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +26 -0
new file mode 100644
index 0000000000..6455663388
@@ -0,0 +1,26 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    uint32_t msg_type;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+extern int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id);
+#endif
include/hw/virtio/vhost.h +2 -0
index 085450c6f8..b682545f51 100644
@@ -124,6 +124,7 @@ int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config,
                           uint32_t config_len);
  int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *data,
                           uint32_t offset, uint32_t size, uint32_t flags);
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id);
  /* notifier callback in case vhost device config space changed
   */
  void vhost_dev_set_config_notifier(struct vhost_dev *dev,
@@ -137,4 +138,5 @@ int vhost_dev_set_inflight(struct vhost_dev *dev,
                             struct vhost_inflight *inflight);
  int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
                             struct vhost_inflight *inflight);
+int vhost_set_state(struct vhost_dev *dev, bool started);
  #endif
include/net/vhost_net.h +0 -0
index 8a6f208189..56e67fe164 100644

include/net/vhost_net.h int vhost_set_vring_ready(NetClientState *nc); · line 43

Quoted 6 lines
include/net/vhost_net.h +1 -2
@@ -40,5 +40,5 @@ int vhost_set_vring_ready(NetClientState *nc);
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
-
-#endif
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);

This should belong to vhost-vdpa header.

Quoted 2 lines

Thanks

will fix this

Quoted 43 lines
Quoted 42 lines
include/net/vhost_net.h +1 -0
+endif
qemu-options.hx +15 -0
index 292d4e7c0c..c19e10ce9c 100644
@@ -2409,6 +2409,10 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
  #ifdef CONFIG_POSIX
      "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
      "                configure a vhost-user network, backed by a chardev 'dev'\n"
+#endif
+#ifdef CONFIG_POSIX
+    "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
+    "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
  #endif
      "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
      "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
@@ -2428,6 +2432,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
  #endif
  #ifdef CONFIG_POSIX
      "vhost-user|"
+#endif
+#ifdef CONFIG_POSIX
+    "vhost-vdpa|"
  #endif
      "socket][,option][,...][mac=macaddr]\n"
      "                initialize an on-board / default host NIC (using MAC address\n"
@@ -2896,6 +2903,14 @@ SRST
      hubport to another netdev with ID nd by using the ``netdev=nd``
      option.
 
+``-netdev vhost-vdpa,vhostdev=/path/to/dev ``
+    Establish a vhost-vdpa netdev.
+
+    vDPA device is a device that uses a datapath which complies with
+    the virtio specifications with a vendor specific control path.
+    vDPA devices can be both physically located on the hardware or
+    emulated by software.
+
  ``-net nic[,netdev=nd][,macaddr=mac][,model=type] [,name=name][,addr=addr][,vectors=v]``
      Legacy option to configure or create an on-board (or machine
      default) Network Interface Card(NIC) and connect it either to the
author
#21 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Jason Wang
Quoted 6 lines · Jason Wang
Quoted 1 line · Cindy Lu

From: Tiwei Bie<[email protected]>

Quoted 8 lines

Consider the significant modification based on the original patch.

I think you may change the other to yourslef and keep the sobs for both Tiwei and Lingshan.

Thanks

Sure, Will change this

Quoted 17 lines
Quoted 16 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
#34 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

Similar for this patch, you can change the git author and keep sobs for both Tiwei and Ling Shan.

Quoted 138 lines

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is specified by a "vhostdev" parameter.

---
  include/net/vhost-vdpa.h |  19 ++++
  include/net/vhost_net.h  |   2 +-
  net/Makefile.objs        |   2 +-
  net/clients.h            |   2 +
  net/net.c                |   3 +
  net/vhost-vdpa.c         | 235 +++++++++++++++++++++++++++++++++++++++
  qapi/net.json            |  26 ++++-
  7 files changed, 285 insertions(+), 4 deletions(-)

create mode 100644 include/net/vhost-vdpa.h
create mode 100644 net/vhost-vdpa.c

include/net/vhost-vdpa.h new file +19 -0
new file mode 100644
index 0000000000..6ce0d04f72
@@ -0,0 +1,19 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -1
index 56e67fe164..0b87d3c6e9 100644
@@ -41,4 +41,4 @@ uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
  int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);
-endif
+#endif
net/Makefile.objs +1 -1
index c5d076d19c..5ab45545db 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
  tap-obj-y ?= tap-stub.o
  common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
  common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_NET_VDPA) += vhost-vdpa.o
  vde.o-libs = $(VDE_LIBS)
 
  common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
  int net_init_vhost_user(const Netdev *netdev, const char *name,
                          NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
  #endif /* QEMU_NET_CLIENTS_H */
net/net.c +3 -0
index 599fb61028..82624ea9ac 100644
@@ -965,6 +965,9 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
  #ifdef CONFIG_VHOST_NET_USER
          [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
  #endif
+#ifdef CONFIG_VHOST_NET_VDPA
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
+#endif
  #ifdef CONFIG_L2TPV3
          [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
  #endif
net/vhost-vdpa.c new file +40 -0
new file mode 100644
index 0000000000..7b98c142b5
@@ -0,0 +1,235 @@
+/*
+ * vhost-vdpa.c
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "clients.h"
+#include "net/vhost_net.h"
+#include "net/vhost-vdpa.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include <sys/ioctl.h>
+#include <err.h>
+#include "standard-headers/linux/virtio_net.h"
+#include "monitor/monitor.h"
+#include "hw/virtio/vhost.h"
+
+/* Todo:need to add the multiqueue support here */
+typedef struct VhostVDPAState {
+    NetClientState nc;
+    struct vhost_vdpa vhost_vdpa;
+    VHostNetState *vhost_net;
+    uint64_t acked_features;
+    bool started;
+} VhostVDPAState;
+
+VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->vhost_net;

net/vhost-vdpa.c line 50

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+}
+
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->acked_features;
+}
+
+static int vhost_vdpa_check_device_id(NetClientState *nc)

A better name is needed, something like "vhost_vdpa_net_check_devie_id" is better.

net/vhost-vdpa.c line 56

Quoted 6 lines
net/vhost-vdpa.c new file +6 -0
+{
+    uint32_t device_id;
+    int ret;
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    /* Get the device id from hw*/

The code explains itself, so no need for this comment.

Quoted 11 lines
net/vhost-vdpa.c new file +11 -0
+    ret = vhost_net_get_device_id(s->vhost_net, &device_id);
+    if (device_id != VIRTIO_ID_NET) {
+        return -ENOTSUP;
+    }
+    return ret;
+}
+
+static void vhost_vdpa_del(NetClientState *ncs)
+{
+    VhostVDPAState *s;
+

net/vhost-vdpa.c line 77

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    if (s->vhost_net) {
+        /* save acked features */
+        uint64_t features = vhost_net_get_acked_features(s->vhost_net);
+        if (features) {
+            s->acked_features = features;
+        }

I'm not sure I get here, is the acked_features used in the vhost_net_cleanup()?

Quoted 22 lines
net/vhost-vdpa.c new file +22 -0
+        vhost_net_cleanup(s->vhost_net);
+    }
+}
+
+static int vhost_vdpa_add(NetClientState *ncs, void *be)
+{
+    VhostNetOptions options;
+    struct vhost_net *net = NULL;
+    VhostVDPAState *s;
+    int ret;
+
+    options.backend_type = VHOST_BACKEND_TYPE_VDPA;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    options.net_backend = ncs;
+    options.opaque      = be;
+    options.busyloop_timeout = 0;
+    net = vhost_net_init(&options);
+    if (!net) {

net/vhost-vdpa.c line 109

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+        error_report("failed to init vhost_net for queue");
+        goto err;
+    }
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+    }
+    s->vhost_net = net;
+    /* check the device id for vdpa */

The comment could be removed as well.

Quoted 14 lines
net/vhost-vdpa.c new file +14 -0
+    ret = vhost_vdpa_check_device_id(ncs);
+    if (ret) {
+        goto err;
+    }
+    return 0;
+err:
+    if (net) {
+        vhost_net_cleanup(net);
+    }
+    vhost_vdpa_del(ncs);
+    return -1;
+}
+
+static void vhost_vdpa_cleanup(NetClientState *nc)

net/vhost-vdpa.c line 133

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);

Why this is needed?

Thanks

Quoted 153 lines
net/vhost-vdpa.c new file +102 -0
+}
+
+static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    return true;
+}
+
+static bool vhost_vdpa_has_ufo(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    uint64_t  features = 0;
+
+    features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
+    features = vhost_net_get_features(s->vhost_net, features);
+    return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
+
+}
+
+static NetClientInfo net_vhost_vdpa_info = {
+        .type = NET_CLIENT_DRIVER_VHOST_VDPA,
+        .size = sizeof(VhostVDPAState),
+        .cleanup = vhost_vdpa_cleanup,
+        .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
+        .has_ufo = vhost_vdpa_has_ufo,
+};
+
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev,
+                               bool has_fd, char *fd)
+{
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd = -1;
+    Error *err = NULL;
+    int ret = 0;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (has_fd) {
+        vdpa_device_fd = monitor_fd_param(cur_mon, fd, &err);
+    } else{
+        vdpa_device_fd = open(vhostdev, O_RDWR);
+    }
+
+    if (vdpa_device_fd == -1) {
+        return -errno;
+    }
+    s->vhost_vdpa.device_fd = vdpa_device_fd;
+    ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa);
+    assert(s->vhost_net);
+
+    if (ret) {
+        if (has_fd) {
+            close(vdpa_device_fd);
+        }
+    }
+    return ret;
+}
+
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
+        return -1;
+    }
+
+    return 0;
+}
+
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
+{
+    const NetdevVhostVDPAOptions *opts;
+
+    assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    opts = &netdev->u.vhost_vdpa;
+    /* verify net frontend */
+    if (qemu_opts_foreach(qemu_find_opts("device"), net_vhost_check_net,
+                          (char *)name, errp)) {
+        return -1;
+    }
+    return net_vhost_vdpa_init(peer, "vhost_vdpa", name, opts->vhostdev,
+                    opts->has_fd, opts->fd);
+}
qapi/net.json +24 -2
index cebb1b52e3..37507ce9ba 100644
@@ -428,6 +428,27 @@
      '*vhostforce':    'bool',
      '*queues':        'int' } }
 
+##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a vdpa dev path in sysfs
+#            (default path:/dev/vhost-vdpa-$ID)
+#
+# @fd: file descriptor of an already opened vdpa device
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1)
+#
+# Since: 5.1
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*fd':           'str',
+    '*queues':       'int' } }
+
  ##
  # @NetClientDriver:
  #
@@ -437,7 +458,7 @@
  ##
  { 'enum': 'NetClientDriver',
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
  ##
  # @Netdev:
@@ -465,7 +486,8 @@
      'bridge':   'NetdevBridgeOptions',
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
  ##
  # @NetLegacy:
#22 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu
Quoted 10 lines · Cindy Lu

From: Tiwei Bie<[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

Quoted 8 lines
   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

---

Btw, I don't see the how to connect the vhost_set/get_config() with the virtio_net_set/get_config().

Thanks

author
#35 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

In reply to Jason Wang

Hi Jason,

Quoted 10 lines · Jason Wang
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

Similar for this patch, you can change the git author and keep sobs for both Tiwei and Ling Shan.

Will Fix this

Quoted 144 lines
Quoted 144 lines

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is specified by a "vhostdev" parameter.

---
  include/net/vhost-vdpa.h |  19 ++++
  include/net/vhost_net.h  |   2 +-
  net/Makefile.objs        |   2 +-
  net/clients.h            |   2 +
  net/net.c                |   3 +
  net/vhost-vdpa.c         | 235 +++++++++++++++++++++++++++++++++++++++
  qapi/net.json            |  26 ++++-
  7 files changed, 285 insertions(+), 4 deletions(-)

create mode 100644 include/net/vhost-vdpa.h
create mode 100644 net/vhost-vdpa.c

include/net/vhost-vdpa.h new file +19 -0
new file mode 100644
index 0000000000..6ce0d04f72
@@ -0,0 +1,19 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -1
index 56e67fe164..0b87d3c6e9 100644
@@ -41,4 +41,4 @@ uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
  int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);
-endif
+#endif
net/Makefile.objs +1 -1
index c5d076d19c..5ab45545db 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
  tap-obj-y ?= tap-stub.o
  common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
  common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_NET_VDPA) += vhost-vdpa.o
  vde.o-libs = $(VDE_LIBS)
 
  common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
  int net_init_vhost_user(const Netdev *netdev, const char *name,
                          NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
  #endif /* QEMU_NET_CLIENTS_H */
net/net.c +3 -0
index 599fb61028..82624ea9ac 100644
@@ -965,6 +965,9 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
  #ifdef CONFIG_VHOST_NET_USER
          [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
  #endif
+#ifdef CONFIG_VHOST_NET_VDPA
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
+#endif
  #ifdef CONFIG_L2TPV3
          [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
  #endif
net/vhost-vdpa.c new file +46 -0
new file mode 100644
index 0000000000..7b98c142b5
@@ -0,0 +1,235 @@
+/*
+ * vhost-vdpa.c
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "clients.h"
+#include "net/vhost_net.h"
+#include "net/vhost-vdpa.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include <sys/ioctl.h>
+#include <err.h>
+#include "standard-headers/linux/virtio_net.h"
+#include "monitor/monitor.h"
+#include "hw/virtio/vhost.h"
+
+/* Todo:need to add the multiqueue support here */
+typedef struct VhostVDPAState {
+    NetClientState nc;
+    struct vhost_vdpa vhost_vdpa;
+    VHostNetState *vhost_net;
+    uint64_t acked_features;
+    bool started;
+} VhostVDPAState;
+
+VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->vhost_net;
+}
+
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
Quoted 9 lines

net/vhost-vdpa.c line 50

Quoted 4 lines
net/vhost-vdpa.c new file +4 -0
+    return s->acked_features;
+}
+
+static int vhost_vdpa_check_device_id(NetClientState *nc)

A better name is needed, something like "vhost_vdpa_net_check_devie_id" is better.

Sure will fix this

Quoted 11 lines

net/vhost-vdpa.c line 56

Quoted 7 lines
net/vhost-vdpa.c new file +6 -0
+{
+    uint32_t device_id;
+    int ret;
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    /* Get the device id from hw*/

The code explains itself, so no need for this comment.

will remove this

Quoted 18 lines
Quoted 18 lines
net/vhost-vdpa.c new file +17 -0
+    ret = vhost_net_get_device_id(s->vhost_net, &device_id);
+    if (device_id != VIRTIO_ID_NET) {
+        return -ENOTSUP;
+    }
+    return ret;
+}
+
+static void vhost_vdpa_del(NetClientState *ncs)
+{
+    VhostVDPAState *s;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    if (s->vhost_net) {
+        /* save acked features */
Quoted 9 lines

net/vhost-vdpa.c line 77

Quoted 4 lines
net/vhost-vdpa.c new file +4 -0
+        uint64_t features = vhost_net_get_acked_features(s->vhost_net);
+        if (features) {
+            s->acked_features = features;
+        }

I'm not sure I get here, is the acked_features used in the vhost_net_cleanup()?

I think we can remove this part, seems these bit are not using anymore

Quoted 28 lines
Quoted 28 lines
net/vhost-vdpa.c new file +27 -0
+        vhost_net_cleanup(s->vhost_net);
+    }
+}
+
+static int vhost_vdpa_add(NetClientState *ncs, void *be)
+{
+    VhostNetOptions options;
+    struct vhost_net *net = NULL;
+    VhostVDPAState *s;
+    int ret;
+
+    options.backend_type = VHOST_BACKEND_TYPE_VDPA;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    options.net_backend = ncs;
+    options.opaque      = be;
+    options.busyloop_timeout = 0;
+    net = vhost_net_init(&options);
+    if (!net) {
+        error_report("failed to init vhost_net for queue");
+        goto err;
+    }
+
+    if (s->vhost_net) {
Quoted 10 lines

net/vhost-vdpa.c line 109

Quoted 5 lines
net/vhost-vdpa.c new file +5 -0
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+    }
+    s->vhost_net = net;
+    /* check the device id for vdpa */

The comment could be removed as well.

Sure will remove this

Quoted 28 lines
Quoted 14 lines
net/vhost-vdpa.c new file +14 -0
+    ret = vhost_vdpa_check_device_id(ncs);
+    if (ret) {
+        goto err;
+    }
+    return 0;
+err:
+    if (net) {
+        vhost_net_cleanup(net);
+    }
+    vhost_vdpa_del(ncs);
+    return -1;
+}
+
+static void vhost_vdpa_cleanup(NetClientState *nc)

net/vhost-vdpa.c line 133

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);

Why this is needed?

Quoted 2 lines

Thanks

This is to clean the packet in the queue while the vdpa remove, I will double check this part

Quoted 155 lines
Quoted 154 lines
net/vhost-vdpa.c new file +102 -0
+}
+
+static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    return true;
+}
+
+static bool vhost_vdpa_has_ufo(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    uint64_t  features = 0;
+
+    features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
+    features = vhost_net_get_features(s->vhost_net, features);
+    return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
+
+}
+
+static NetClientInfo net_vhost_vdpa_info = {
+        .type = NET_CLIENT_DRIVER_VHOST_VDPA,
+        .size = sizeof(VhostVDPAState),
+        .cleanup = vhost_vdpa_cleanup,
+        .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
+        .has_ufo = vhost_vdpa_has_ufo,
+};
+
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev,
+                               bool has_fd, char *fd)
+{
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd = -1;
+    Error *err = NULL;
+    int ret = 0;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (has_fd) {
+        vdpa_device_fd = monitor_fd_param(cur_mon, fd, &err);
+    } else{
+        vdpa_device_fd = open(vhostdev, O_RDWR);
+    }
+
+    if (vdpa_device_fd == -1) {
+        return -errno;
+    }
+    s->vhost_vdpa.device_fd = vdpa_device_fd;
+    ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa);
+    assert(s->vhost_net);
+
+    if (ret) {
+        if (has_fd) {
+            close(vdpa_device_fd);
+        }
+    }
+    return ret;
+}
+
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
+        return -1;
+    }
+
+    return 0;
+}
+
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
+{
+    const NetdevVhostVDPAOptions *opts;
+
+    assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    opts = &netdev->u.vhost_vdpa;
+    /* verify net frontend */
+    if (qemu_opts_foreach(qemu_find_opts("device"), net_vhost_check_net,
+                          (char *)name, errp)) {
+        return -1;
+    }
+    return net_vhost_vdpa_init(peer, "vhost_vdpa", name, opts->vhostdev,
+                    opts->has_fd, opts->fd);
+}
qapi/net.json +24 -2
index cebb1b52e3..37507ce9ba 100644
@@ -428,6 +428,27 @@
      '*vhostforce':    'bool',
      '*queues':        'int' } }
 
+##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a vdpa dev path in sysfs
+#            (default path:/dev/vhost-vdpa-$ID)
+#
+# @fd: file descriptor of an already opened vdpa device
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1)
+#
+# Since: 5.1
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*fd':           'str',
+    '*queues':       'int' } }
+
  ##
  # @NetClientDriver:
  #
@@ -437,7 +458,7 @@
  ##
  { 'enum': 'NetClientDriver',
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
  ##
  # @Netdev:
@@ -465,7 +486,8 @@
      'bridge':   'NetdevBridgeOptions',
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
  ##
  # @NetLegacy:
author
#23 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Jason Wang
Quoted 18 lines · Jason Wang
Quoted 15 lines · Cindy Lu

From: Tiwei Bie<[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
Quoted 8 lines
Quoted 3 lines

---

Btw, I don't see the how to connect the vhost_set/get_config() with the virtio_net_set/get_config().

Sure, I will add this part

Quoted 3 lines

Thanks

#36 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu
Quoted 13 lines · Cindy Lu
Quoted 13 lines

net/vhost-vdpa.c line 133

Quoted 11 lines
net/vhost-vdpa.c new file +11 -0
+static void vhost_vdpa_cleanup(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);

Why this is needed?

Quoted 4 lines
Quoted 2 lines

Thanks

This is to clean the packet in the queue while the vdpa remove, I will double check this part

Note we don't have a software backup driver for qemu currently (we probably need one in the future).

So we can't fallback into userspace which means the packet can not be queued by qemu.

Thanks

author
#37 lore raw

Re: [RFC v3 8/8] vhost-vdpa: introduce vhost-vdpa net client

Hi Jason,

Quoted 22 lines · Jason Wang
Quoted 13 lines · Cindy Lu
Quoted 13 lines

net/vhost-vdpa.c line 133

Quoted 11 lines
net/vhost-vdpa.c new file +11 -0
+static void vhost_vdpa_cleanup(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);

Why this is needed?

Quoted 4 lines
Quoted 2 lines

Thanks

This is to clean the packet in the queue while the vdpa remove, I will double check this part

Quoted 6 lines

Note we don't have a software backup driver for qemu currently (we probably need one in the future).

So we can't fallback into userspace which means the packet can not be queued by qemu.

Got it, Thanks Jason, I will remove this part

Quoted 2 lines

Thanks

#24 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu
Quoted 568 lines · Cindy Lu

From: Tiwei Bie <[email protected]>

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
---
 configure                         |  21 ++
 hw/net/vhost_net-stub.c           |   5 +
 hw/net/vhost_net.c                |  47 +++-
 hw/virtio/Makefile.objs           |   1 +
 hw/virtio/vhost-backend.c         |   5 +
 hw/virtio/vhost-vdpa.c            | 399 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |  37 ++-
 include/hw/virtio/vhost-backend.h |  10 +-
 include/hw/virtio/vhost-vdpa.h    |  26 ++
 include/hw/virtio/vhost.h         |   2 +
 include/net/vhost_net.h           |   4 +-
 qemu-options.hx                   |  15 ++
 12 files changed, 566 insertions(+), 6 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
configure +21 -0
index 23b5e93752..53679ee57f 100755
@@ -1557,6 +1557,10 @@ for opt do
   ;;
   --enable-vhost-user) vhost_user="yes"
   ;;
+  --disable-vhost-vdpa) vhost_vdpa="no"
+  ;;
+  --enable-vhost-vdpa) vhost_vdpa="yes"
+  ;;
   --disable-vhost-kernel) vhost_kernel="no"
   ;;
   --enable-vhost-kernel) vhost_kernel="yes"
@@ -1846,6 +1850,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   vhost-crypto    vhost-user-crypto backend support
   vhost-kernel    vhost kernel backend support
   vhost-user      vhost-user backend support
+  vhost-vdpa      vhost-vdpa kernel backend support
   spice           spice
   rbd             rados block device (rbd)
   libiscsi        iscsi support
@@ -2336,6 +2341,10 @@ test "$vhost_user" = "" && vhost_user=yes
 if test "$vhost_user" = "yes" && test "$mingw32" = "yes"; then
   error_exit "vhost-user isn't available on win32"
 fi
+test "$vhost_vdpa" = "" && vhost_vdpa=$linux
+if test "$vhost_vdpa" = "yes" && test "$linux" != "yes"; then
+  error_exit "vhost-vdpa is only available on Linux"
+fi
 test "$vhost_kernel" = "" && vhost_kernel=$linux
 if test "$vhost_kernel" = "yes" && test "$linux" != "yes"; then
   error_exit "vhost-kernel is only available on Linux"
@@ -2364,6 +2373,11 @@ test "$vhost_user_fs" = "" && vhost_user_fs=$vhost_user
 if test "$vhost_user_fs" = "yes" && test "$vhost_user" = "no"; then
   error_exit "--enable-vhost-user-fs requires --enable-vhost-user"
 fi
+#vhost-vdpa backends
+test "$vhost_net_vdpa" = "" && vhost_net_vdpa=$vhost_vdpa
+if test "$vhost_net_vdpa" = "yes" && test "$vhost_vdpa" = "no"; then
+  error_exit "--enable-vhost-net-vdpa requires --enable-vhost-vdpa"
+fi
 
 # OR the vhost-kernel and vhost-user values for simplicity
 if test "$vhost_net" = ""; then
@@ -6673,6 +6687,7 @@ echo "vhost-scsi support $vhost_scsi"
 echo "vhost-vsock support $vhost_vsock"
 echo "vhost-user support $vhost_user"
 echo "vhost-user-fs support $vhost_user_fs"
+echo "vhost-vdpa support $vhost_vdpa"
 echo "Trace backends    $trace_backends"
 if have_backend "simple"; then
 echo "Trace output file $trace_file-<pid>"
@@ -7170,6 +7185,9 @@ fi
 if test "$vhost_net_user" = "yes" ; then
   echo "CONFIG_VHOST_NET_USER=y" >> $config_host_mak
 fi
+if test "$vhost_net_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_NET_VDPA=y" >> $config_host_mak
+fi
 if test "$vhost_crypto" = "yes" ; then
   echo "CONFIG_VHOST_CRYPTO=y" >> $config_host_mak
 fi
@@ -7182,6 +7200,9 @@ fi
 if test "$vhost_user" = "yes" ; then
   echo "CONFIG_VHOST_USER=y" >> $config_host_mak
 fi
+if test "$vhost_vdpa" = "yes" ; then
+  echo "CONFIG_VHOST_VDPA=y" >> $config_host_mak
+fi
 if test "$vhost_user_fs" = "yes" ; then
   echo "CONFIG_VHOST_USER_FS=y" >> $config_host_mak
 fi
hw/net/vhost_net-stub.c +5 -0
index 43e93e1a9a..ab77a92a7d 100644
@@ -94,3 +94,8 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
 }
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return 0;
+}
+
hw/net/vhost_net.c +46 -1
index e2bc7de2eb..25045cff59 100644
@@ -17,8 +17,10 @@
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
 #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
 #include "hw/virtio/virtio-net.h"
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
@@ -85,6 +87,30 @@ static const int user_feature_bits[] = {
     VHOST_INVALID_FEATURE_BIT
 };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_F_RING_PACKED,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
 static const int *vhost_net_get_feature_bits(struct vhost_net *net)
 {
     const int *feature_bits = 0;
@@ -96,6 +122,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
     case NET_CLIENT_DRIVER_VHOST_USER:
         feature_bits = user_feature_bits;
         break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
     default:
         error_report("Feature bits not defined for this type: %d",
                 net->nc->info->type);
@@ -110,7 +139,10 @@ uint64_t vhost_net_get_features(struct vhost_net *net, uint64_t features)
     return vhost_get_features(&net->dev, vhost_net_get_feature_bits(net),
             features);
 }
-
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t * device_id)
+{
+    return vhost_dev_get_device_id(&net->dev, device_id);
+}
 void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
 {
     net->dev.acked_features = net->dev.backend_features;
@@ -337,6 +369,11 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }
+
         peer = qemu_get_peer(ncs, i);
         r = vhost_net_start_one(get_vhost_net(peer), dev);
 
@@ -433,6 +470,12 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         vhost_net = vhost_user_get_vhost_net(nc);
         assert(vhost_net);
         break;
+#endif
+#ifdef CONFIG_VHOST_NET_VDPA
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
 #endif
     default:
         break;
@@ -474,3 +517,5 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
     return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
 }
+
+
hw/virtio/Makefile.objs +1 -0
index 4e4d39a0a4..6b1b1a5fce 100644
@@ -5,6 +5,7 @@ obj-y += virtio.o
 obj-$(CONFIG_VHOST) += vhost.o vhost-backend.o
 common-obj-$(call lnot,$(CONFIG_VHOST)) += vhost-stub.o
 obj-$(CONFIG_VHOST_USER) += vhost-user.o
+obj-$(CONFIG_VHOST_VDPA) += vhost-vdpa.o
 
 common-obj-$(CONFIG_VIRTIO_RNG) += virtio-rng.o
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
hw/virtio/vhost-backend.c +5 -0
index 42efb4967b..420341e8c5 100644
@@ -291,6 +291,11 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
     case VHOST_BACKEND_TYPE_USER:
         dev->vhost_ops = &user_ops;
         break;
+#endif
+#ifdef CONFIG_VHOST_VDPA
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
 #endif
     default:
         error_report("Unknown vhost backend type");
hw/virtio/vhost-vdpa.c new file +313 -0
new file mode 100644
index 0000000000..2d136a8565
@@ -0,0 +1,399 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>
+#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/main-loop.h"
+#include <linux/kvm.h>
+#include "sysemu/kvm.h"
+
+
+static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings to
+            * addresses in the upper part of the 64-bit address space.  These
+            * are never accessed by the CPU and beyond the address width of
+            * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU width.
+            */
+           section->offset_within_address_space & (1ULL << 63);
+}
+
+static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
+                              void *vaddr, bool readonly)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.uaddr = (uint64_t)vaddr;
+    msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
+    msg.iotlb.type = VHOST_IOTLB_UPDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
+                                hwaddr size)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type =  v->msg_type;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static void vhost_vdpa_listener_region_add(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
+                             vaddr, section->readonly);
+    if (ret) {
+        error_report("vhost vdpa map fail!");
+        if (memory_region_is_ram_device(section->mr)) {
+            /* Allow unexpected mappings not to be fatal for RAM devices */
+            error_report("map ram fail!");
+          return ;
+        }
+        goto fail;
+    }
+
+    return;
+
+fail:
+    if (memory_region_is_ram_device(section->mr)) {
+        error_report("failed to vdpa_dma_map. pci p2p may not work");
+        return;
+
+    }
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    error_report("vhost-vdpa: DMA mapping failed, unable to continue");
+    return;
+
+}
+
+static void vhost_vdpa_listener_region_del(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    int ret;
+    bool try_unmap = true;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    if (try_unmap) {
+        ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
+        if (ret) {
+            error_report("vhost_vdpa dma unmap error!");
+        }
+    }
+
+    memory_region_unref(section->mr);
+}
+/* Register a new memory listener, only to get diffs from qemu,
+ * this help to reduce the tricky codes in vhost
+ * (e.g generating diffs of two rbtree as usnic did).*/
+static const MemoryListener vhost_vdpa_memory_listener = {
+    .region_add = vhost_vdpa_listener_region_add,
+    .region_del = vhost_vdpa_listener_region_del,
+};
+
+static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
+                             void *arg)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    int fd = v->device_fd;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    return ioctl(fd, request, arg);
+}
+
+static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status)
+{
+    uint8_t s;
+
+    if (vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &s)) {
+        return;
+    }
+
+    s |= status;
+
+    vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &s);
+}
+
+static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
+{
+    struct vhost_vdpa *v;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = opaque;
+    dev->opaque =  opaque ;
+
+    v->listener = vhost_vdpa_memory_listener;
+    v->msg_type = VHOST_IOTLB_MSG_V2;
+    memory_listener_register(&v->listener, &address_space_memory);
+
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                               VIRTIO_CONFIG_S_DRIVER);
+
+    return 0;
+}
+
+static int vhost_vdpa_cleanup(struct vhost_dev *dev)
+{
+    struct vhost_vdpa *v;
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = dev->opaque;
+    memory_listener_unregister(&v->listener);
+
+    dev->opaque = NULL;
+    return 0;
+}
+
+static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
+{
+    return INT_MAX;
+}
+
+static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+
+    if (mem->padding) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_vdpa_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    int ret = vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
+    uint8_t status = 0;
+
+    if (ret) {
+        return ret;
+    }
+    vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
+    vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+    return !(status & VIRTIO_CONFIG_S_FEATURES_OK);
+}
+
+int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id)
+{
+    return vhost_vdpa_call(dev, VHOST_VDPA_GET_DEVICE_ID, device_id);
+}
+
+static int vhost_vdpa_reset_device(struct vhost_dev *dev)
+{
+    uint8_t status = 0;
+
+    return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &status);
+}
+
+static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
+{
+    int i;
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num = 1,
+        };
+        vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
+    }

hw/virtio/vhost-vdpa.c line 323

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    return 0;
+}
+
+static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags)
+{
+    struct vhost_vdpa_config config;
+    int ret;
+    if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {

VHOST_VDPA_MAX_CONFIG_SIZE is currently undefined.

If we want to maintain this as a stack allocation (as proposed in
https://www.mail-archive.com/[email protected]/msg701744.html) I think that the best option is to decide which is the maximum value buf can hold, and set it in vhost_vdpa_config.buf declaration.

I can only see IFCVF expecting a struct virtio_net_config there in the Linux kernel v5.7. It is assumable that its size will be the bigger data size we are going to save there through qemu configuration? It will be possible to change in future versions if needed, isn't it?.

If it could be much bigger at this moment, we should go back to heap allocation.

Any thoughts?

Thanks!

Quoted 313 lines
hw/virtio/vhost-vdpa.c new file +76 -0
+        return -1;
+    }
+    memset(&config, 0, sizeof(struct vhost_vdpa_config));
+    config.off = 0;
+    config.len = size;
+    memcpy(&config.buf, data, size);
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_SET_CONFIG, &config);
+    return ret;
+}
+
+static int vhost_vdpa_get_config(struct vhost_dev *dev, uint8_t *config,
+                                   uint32_t config_len)
+{
+    struct vhost_vdpa_config v_config;
+    int ret;
+
+    memset(&v_config, 0, sizeof(struct vhost_vdpa_config));
+    if (config == NULL) {
+        return -1;
+    }
+    ret = vhost_vdpa_call(dev, VHOST_VDPA_GET_CONFIG, &v_config);
+    if ((v_config.len > config_len) || (v_config.len == 0)) {
+        return -EINVAL;
+    }
+    memcpy(config, &v_config.buf, config_len);
+    return ret;
+ }
+
+static int vhost_vdpa_set_state(struct vhost_dev *dev, bool started)
+{
+    if (started) {
+        uint8_t status = 0;
+
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
+        vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status);
+
+        return !(status & VIRTIO_CONFIG_S_DRIVER_OK);
+    } else {
+        vhost_vdpa_reset_device(dev);
+        vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
+                                   VIRTIO_CONFIG_S_DRIVER);
+        return 0;
+    }
+}
+
+const VhostOps vdpa_ops = {
+        .backend_type = VHOST_BACKEND_TYPE_VDPA,
+        .vhost_backend_init = vhost_vdpa_init,
+        .vhost_backend_cleanup = vhost_vdpa_cleanup,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_set_vring_endian = NULL,
+        .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
+        .vhost_set_mem_table = vhost_vdpa_set_mem_table,
+        .vhost_set_features = vhost_vdpa_set_features,
+        .vhost_reset_device = vhost_vdpa_reset_device,
+        .vhost_get_vq_index = vhost_vdpa_get_vq_index,
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
+        .vhost_get_config  = vhost_vdpa_get_config,
+        .vhost_set_config = vhost_vdpa_set_config,
+        .vhost_requires_shm_log = NULL,
+        .vhost_migration_done = NULL,
+        .vhost_backend_can_merge = NULL,
+        .vhost_net_set_mtu = NULL,
+        .vhost_set_iotlb_callback = NULL,
+        .vhost_send_device_iotlb_msg = NULL,
+        .vhost_set_state = vhost_vdpa_set_state,
+        .vhost_get_device_id = vhost_vdpa_get_device_id,
+};
hw/virtio/vhost.c +35 -2
index 01ebe12f28..b97aa02a4c 100644
@@ -756,6 +756,12 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
+    /*vDPA need to use the phys address here to set to hardware*/
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }
     int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
         VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
@@ -1506,6 +1512,14 @@ int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *data,
     return -1;
 }
 
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id)
+{
+    assert(hdev->vhost_ops);
+    if (hdev->vhost_ops->vhost_get_device_id) {
+        return hdev->vhost_ops->vhost_get_device_id(hdev, device_id);
+    }
+    return -1;
+}
 void vhost_dev_set_config_notifier(struct vhost_dev *hdev,
                                    const VhostDevConfigOps *ops)
 {
@@ -1661,7 +1675,13 @@ int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
         }
     }
 
-    if (vhost_dev_has_iommu(hdev)) {
+    r = vhost_set_state(hdev, true);
+    if (r) {
+        goto fail_log;
+    }
+
+    if (vhost_dev_has_iommu(hdev) &&
+        hdev->vhost_ops->vhost_set_iotlb_callback) {
         hdev->vhost_ops->vhost_set_iotlb_callback(hdev, true);
 
         /* Update used ring information for IOTLB to work correctly,
@@ -1697,6 +1717,8 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     /* should only be called after backend is connected */
     assert(hdev->vhost_ops);
 
+    vhost_set_state(hdev, false);
+
     for (i = 0; i < hdev->nvqs; ++i) {
         vhost_virtqueue_stop(hdev,
                              vdev,
@@ -1705,7 +1727,9 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
 
     if (vhost_dev_has_iommu(hdev)) {
-        hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        if (hdev->vhost_ops->vhost_set_iotlb_callback) {
+            hdev->vhost_ops->vhost_set_iotlb_callback(hdev, false);
+        }
         memory_listener_unregister(&hdev->iommu_listener);
     }
     vhost_log_put(hdev, true);
@@ -1722,3 +1746,12 @@ int vhost_net_set_backend(struct vhost_dev *hdev,
 
     return -1;
 }
+
+int vhost_set_state(struct vhost_dev *hdev, bool started)
+{
+    if (hdev->vhost_ops->vhost_set_state) {
+        return hdev->vhost_ops->vhost_set_state(hdev, started);
+    }
+
+    return 0;
+}
include/hw/virtio/vhost-backend.h +9 -1
index 300b59c172..1ebe3785cf 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
     VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
 } VhostBackendType;
 
 typedef enum VhostSetConfigType {
@@ -77,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
 typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
 typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                          int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
 typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
 typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                        char *mac_addr);
@@ -112,6 +114,8 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
 typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                         struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, bool started);
+typedef int (*vhost_get_device_id_op)(struct vhost_dev *dev, uint32_t *dev_id);
 typedef struct VhostOps {
     VhostBackendType backend_type;
     vhost_backend_init vhost_backend_init;
@@ -138,6 +142,7 @@ typedef struct VhostOps {
     vhost_reset_device_op vhost_reset_device;
     vhost_get_vq_index_op vhost_get_vq_index;
     vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
     vhost_requires_shm_log_op vhost_requires_shm_log;
     vhost_migration_done_op vhost_migration_done;
     vhost_backend_can_merge_op vhost_backend_can_merge;
@@ -152,9 +157,12 @@ typedef struct VhostOps {
     vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
     vhost_get_inflight_fd_op vhost_get_inflight_fd;
     vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
+    vhost_get_device_id_op vhost_get_device_id;
 } VhostOps;
 
 extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
 int vhost_set_backend_type(struct vhost_dev *dev,
                            VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +26 -0
new file mode 100644
index 0000000000..6455663388
@@ -0,0 +1,26 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017-2018 Intel Corporation.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    uint32_t msg_type;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+extern int vhost_vdpa_get_device_id(struct vhost_dev *dev,
+                                   uint32_t *device_id);
+#endif
include/hw/virtio/vhost.h +2 -0
index 085450c6f8..b682545f51 100644
@@ -124,6 +124,7 @@ int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config,
                          uint32_t config_len);
 int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *data,
                          uint32_t offset, uint32_t size, uint32_t flags);
+int vhost_dev_get_device_id(struct vhost_dev *hdev, uint32_t *device_id);
 /* notifier callback in case vhost device config space changed
  */
 void vhost_dev_set_config_notifier(struct vhost_dev *dev,
@@ -137,4 +138,5 @@ int vhost_dev_set_inflight(struct vhost_dev *dev,
                            struct vhost_inflight *inflight);
 int vhost_dev_get_inflight(struct vhost_dev *dev, uint16_t queue_size,
                            struct vhost_inflight *inflight);
+int vhost_set_state(struct vhost_dev *dev, bool started);
 #endif
include/net/vhost_net.h +2 -2
index 8a6f208189..56e67fe164 100644
@@ -40,5 +40,5 @@ int vhost_set_vring_ready(NetClientState *nc);
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
-
-#endif
+int vhost_net_get_device_id(struct vhost_net *net, uint32_t *device_id);
+endif
qemu-options.hx +15 -1
index 292d4e7c0c..c19e10ce9c 100644
@@ -2409,6 +2409,10 @@ DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
 #ifdef CONFIG_POSIX
     "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
     "                configure a vhost-user network, backed by a chardev 'dev'\n"
+#endif
+#ifdef CONFIG_POSIX
+    "-netdev vhost-vdpa,id=str,vhostdev=/path/to/dev\n"
+    "                configure a vhost-vdpa network,Establish a vhost-vdpa netdev\n"
 #endif
     "-netdev hubport,id=str,hubid=n[,netdev=nd]\n"
     "                configure a hub port on the hub with ID 'n'\n", QEMU_ARCH_ALL)
@@ -2428,6 +2432,9 @@ DEF("nic", HAS_ARG, QEMU_OPTION_nic,
 #endif
 #ifdef CONFIG_POSIX
     "vhost-user|"
+#endif
+#ifdef CONFIG_POSIX
+    "vhost-vdpa|"
 #endif
     "socket][,option][,...][mac=macaddr]\n"
     "                initialize an on-board / default host NIC (using MAC address\n"
@@ -2896,6 +2903,14 @@ SRST
     hubport to another netdev with ID nd by using the ``netdev=nd``
     option.
 
+``-netdev vhost-vdpa,vhostdev=/path/to/dev ``
+    Establish a vhost-vdpa netdev.
+
+    vDPA device is a device that uses a datapath which complies with
+    the virtio specifications with a vendor specific control path.
+    vDPA devices can be both physically located on the hardware or
+    emulated by software.
+
 ``-net nic[,netdev=nd][,macaddr=mac][,model=type] [,name=name][,addr=addr][,vectors=v]``
     Legacy option to configure or create an on-board (or machine
     default) Network Interface Card(NIC) and connect it either to the
--

2.21.1

#25 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

Quoted 10 lines · Eugenio Perez Martin

hw/virtio/vhost-vdpa.c line 323

Quoted 7 lines
hw/virtio/vhost-vdpa.c new file +7 -0
+static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags)
+{
+    struct vhost_vdpa_config config;
+    int ret;
+    if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {

VHOST_VDPA_MAX_CONFIG_SIZE is currently undefined.

Quoted 4 lines

If we want to maintain this as a stack allocation (as proposed in
https://www.mail-archive.com/[email protected]/msg701744.html) I think that the best option is to decide which is the maximum value buf can hold, and set it in vhost_vdpa_config.buf declaration.

That depends on device features. qemu has logic to figure out config size based on that and set config_size accordingly. Why not reuse it? Sending more should be ok and extra data just ignored.

Signature · 1 line
MST
author
#26 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

Quoted 17 lines · Michael S. Tsirkin
Quoted 10 lines · Eugenio Perez Martin

hw/virtio/vhost-vdpa.c line 323

Quoted 7 lines
hw/virtio/vhost-vdpa.c new file +7 -0
+static int vhost_vdpa_set_config(struct vhost_dev *dev, const uint8_t *data,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags)
+{
+    struct vhost_vdpa_config config;
+    int ret;
+    if ((size > VHOST_VDPA_MAX_CONFIG_SIZE) || (data == NULL)) {

VHOST_VDPA_MAX_CONFIG_SIZE is currently undefined.

Quoted 4 lines

If we want to maintain this as a stack allocation (as proposed in
https://www.mail-archive.com/[email protected]/msg701744.html) I think that the best option is to decide which is the maximum value buf can hold, and set it in vhost_vdpa_config.buf declaration.

Quoted 8 lines

That depends on device features. qemu has logic to figure out config size based on that and set config_size accordingly. Why not reuse it? Sending more should be ok and extra data just ignored.

--
MST

Thanks Michael and Eugenio for your suggestion,I‘m rewriting this part

#27 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Cindy Lu
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

The original author is Tiwei Bie...

Quoted 9 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

Quoted 8 lines
   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

---

...but there is no S-o-b here. Also, Co-Authored-By is an unusual tag; it's just as easy to spell it Signed-off-by even for co-authors.

[Pardon my delicacy in wording my response; I unfortunately lack enough cultural context to know a preferred name or even gender-correct pronouns for referring to the authors in shorthand]

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
author
#28 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

Quoted 24 lines · Eric Blake
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

The original author is Tiwei Bie...

Quoted 9 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

Quoted 8 lines
   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

---

Quoted 7 lines

...but there is no S-o-b here. Also, Co-Authored-By is an unusual tag; it's just as easy to spell it Signed-off-by even for co-authors.

[Pardon my delicacy in wording my response; I unfortunately lack enough cultural context to know a preferred name or even gender-correct pronouns for referring to the authors in shorthand]

Thanks Eric for pointing that out :-), I will fix this soon.

Thanks
Cindy

Quoted 5 lines

--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org

#2 lore raw

Re: [RFC v3 1/8] net: introduce qemu_get_peer

In reply to Cindy Lu
Quoted 25 lines · Cindy Lu

This is a small function that can get the peer from given NetClientState and queue_index

---
 include/net/net.h | 1 +
 net/net.c         | 6 ++++++
 2 files changed, 7 insertions(+)
include/net/net.h +1 -0
index 39085d9444..e7ef42d62b 100644
@@ -176,6 +176,7 @@ void hmp_info_network(Monitor *mon, const QDict *qdict);
 void net_socket_rs_init(SocketReadState *rs,
                         SocketReadStateFinalize *finalize,
                         bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
 /* NIC info */
 
net/net.c +0 -0
index 38778e831d..599fb61028 100644

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 326

Quoted 3 lines
net/net.c +0 -0
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
     return nic->opaque;
 }

To be consistent with the style of the file, you should add a blank line here.

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 335

Quoted 10 lines
net/net.c +6 -0
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    assert(nc != NULL);
+    NetClientState *ncs = nc + queue_index;
+    return ncs->peer;
+}
 
 static void qemu_cleanup_net_client(NetClientState *nc)
 {

Thanks,
Laurent

author
#3 lore raw

Re: [RFC v3 1/8] net: introduce qemu_get_peer

Quoted 28 lines · Laurent Vivier
Quoted 26 lines · Cindy Lu

This is a small function that can get the peer from given NetClientState and queue_index

---
 include/net/net.h | 1 +
 net/net.c         | 6 ++++++
 2 files changed, 7 insertions(+)
include/net/net.h +1 -0
index 39085d9444..e7ef42d62b 100644
@@ -176,6 +176,7 @@ void hmp_info_network(Monitor *mon, const QDict *qdict);
 void net_socket_rs_init(SocketReadState *rs,
                         SocketReadStateFinalize *finalize,
                         bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
 /* NIC info */
 
net/net.c +0 -0
index 38778e831d..599fb61028 100644
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
Quoted 6 lines

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 326

Quoted 2 lines
net/net.c +0 -0
     return nic->opaque;
 }

To be consistent with the style of the file, you should add a blank line here.

Thanks Laurent, I will fix this

Quoted 14 lines

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 335

Quoted 10 lines
net/net.c +6 -0
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    assert(nc != NULL);
+    NetClientState *ncs = nc + queue_index;
+    return ncs->peer;
+}
 
 static void qemu_cleanup_net_client(NetClientState *nc)
 {

Thanks,
Laurent

#29 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Eric Blake
Quoted 21 lines · Eric Blake
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

The original author is Tiwei Bie...

Quoted 16 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration
interface for setting up a vhost HW accelerator, this patch set introduces
a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Quoted 7 lines
Quoted 3 lines

---

...but there is no S-o-b here.  Also, Co-Authored-By is an unusual tag; it's just as easy to spell it Signed-off-by even for co-authors.

Normally the tag to use in this case is "Co-developed-by".

https://www.kernel.org/doc/html/v4.17/process/submitting-patches.html#when-to-use-acked-by-cc-and-co-developed-by

"A Co-Developed-by: states that the patch was also created by another developer along with the original author. This is useful at times when multiple people work on a single patch. Note, this person also needs to have a Signed-off-by: line in the patch as well."

Thanks,
Laurent

#5 lore raw

Re: [RFC v3 2/8] vhost_net: use the function qemu_get_peer

In reply to Cindy Lu
Quoted 44 lines · Cindy Lu

user the qemu_get_peer to replace the old process

---
 hw/net/vhost_net.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
hw/net/vhost_net.c +6 -3
index 6b82803fa7..d1d421e3d9 100644
@@ -306,7 +306,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
+    struct vhost_net *net;
     int r, e, i;
+    NetClientState *peer;
 
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
@@ -314,9 +316,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        struct vhost_net *net;
 
-        net = get_vhost_net(ncs[i].peer);
+        peer = qemu_get_peer(ncs, i);
+        net = get_vhost_net(peer);
         vhost_net_set_vq_index(net, i * 2);
 
         /* Suppress the masking guest notifiers on vhost user
@@ -335,7 +337,8 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs, i);
+        r = vhost_net_start_one(get_vhost_net(peer), dev);
 
         if (r < 0) {
             goto err_start;

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, · line 347

Quoted 2 lines
hw/net/vhost_net.c +0 -0
@@ -343,7 +346,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
         if (ncs[i].peer->vring_enable) {

You can replace this "ncs[i].peer->vring_enable" by "peer->vring_enable"... and you do this later in PATCH 5/8.

Thanks,
Laurent

#7 lore raw

Re: [RFC v3 3/8] virtio-bus: introduce queue_enabled method

In reply to Cindy Lu
Quoted 7 lines · Cindy Lu

From: Jason Wang <[email protected]>

This patch introduces queue_enabled() method which allows the transport to implement its own way to report whether or not a queue is enabled.

Cindy, you must add your signed-off-by on all the patch you send, after all the existing S-o-b.

Quoted 2 lines

0005-virtio-bus-introduce-queue_enabled-method.patch

bad cut&paste?

Thanks,
Laurent

#10 lore raw

Re: [RFC v3 4/8] virtio-pci: implement queue_enabled method

In reply to Cindy Lu
Quoted 6 lines · Cindy Lu

From: Jason Wang <[email protected]>

With version 1, we can detect whether a queue is enabled via queue_enabled.

Add your S-o-b.

Quoted 12 lines
---
 hw/virtio/virtio-pci.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
hw/virtio/virtio-pci.c +0 -0
index 4cb784389c..2c82ed5246 100644
@@ -1107,6 +1107,18 @@ static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
     return pci_get_address_space(dev);
 }
 

hw/virtio/virtio-pci.c static AddressSpace *virtio_pci_get_dma_as(DeviceState *d) · line 1119

Quoted 10 lines
hw/virtio/virtio-pci.c +10 -0
+static bool virtio_pci_queue_enabled(DeviceState *d, int n)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return proxy->vqs[vdev->queue_sel].enabled;
+    }
+
+    return virtio_queue_get_desc_addr(vdev, n) != 0;

I think it would be clearer/cleaner to use here:

  return virtio_queue_enabled(vdev, n);

Thanks,
Laurent

#12 lore raw

Re: [RFC v3 5/8] vhost: introduce vhost_set_vring_ready method

In reply to Cindy Lu
Quoted 7 lines · Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Quoted 6 lines

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

Add your S-o-b.

Quoted 11 lines
---
 hw/net/vhost_net-stub.c |  4 ++++
 hw/net/vhost_net.c      | 11 ++++++++++-
 include/net/vhost_net.h |  1 +
 3 files changed, 15 insertions(+), 1 deletion(-)
hw/net/vhost_net-stub.c +0 -0
index aac0e98228..43e93e1a9a 100644

hw/net/vhost_net-stub.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 92

Quoted 7 lines
hw/net/vhost_net-stub.c +4 -0
@@ -86,6 +86,10 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}

Add a blank line here.

Quoted 8 lines
hw/net/vhost_net-stub.c +0 -0
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
hw/net/vhost_net.c +0 -0
index d1d421e3d9..e2bc7de2eb 100644

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, · line 349

Quoted 7 lines
hw/net/vhost_net.c +1 -1
@@ -344,7 +344,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
             goto err_start;
         }
 
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
             /* restore vring enable state */
             r = vhost_set_vring_enable(peer, peer->vring_enable);

Move this part to PATCH 2/8

hw/net/vhost_net.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 462

Quoted 9 lines
hw/net/vhost_net.c +5 -0
@@ -455,6 +455,15 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {

The structure VhostOps doesn't declare the vhost_set_vring_ready field. Your patch is missing something and it could be not built.

It is defined in PATCH 7/8. If you want to keep this patch you should move the declaration of "vhost_set_vring_ready_op vhost_set_vring_ready" (and related) to this patch.

hw/net/vhost_net.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 466

Quoted 4 lines
hw/net/vhost_net.c +4 -0
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+    return 0;
+}

Add a blank line.

Quoted 9 lines
hw/net/vhost_net.c +0 -0
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     const VhostOps *vhost_ops = net->dev.vhost_ops;
include/net/vhost_net.h +0 -0
index 77e47398c4..8a6f208189 100644

include/net/vhost_net.h int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr); · line 41

Quoted 7 lines
include/net/vhost_net.h +1 -0
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
 
 int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState *nc);
 
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 

Thanks,
Laurent

#15 lore raw

Re: [RFC v3 6/8] vhost-backend: export the vhost backend helper

In reply to Cindy Lu
Quoted 13 lines · Cindy Lu

export the helper then we can reuse some of them in vhost-vdpa

---
 hw/virtio/vhost-backend.c         | 34 ++++++++++++++++++-------------
 include/hw/virtio/vhost-backend.h | 28 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 14 deletions(-)
hw/virtio/vhost-backend.c +0 -0
index 48905383f8..42efb4967b 100644

hw/virtio/vhost-backend.c line 17

Quoted 5 lines
hw/virtio/vhost-backend.c +1 -1
@@ -14,7 +14,7 @@
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "standard-headers/linux/vhost_types.h"
-
+#include "hw/virtio/vhost-vdpa.h"

You can't include this file because it is created in the next patch.

Quoted 11 lines
hw/virtio/vhost-backend.c +0 -4
 #ifdef CONFIG_VHOST_KERNEL
 #include <linux/vhost.h>
 #include <sys/ioctl.h>
@@ -22,10 +22,16 @@
 static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
                              void *arg)
 {
-    int fd = (uintptr_t) dev->opaque;
-
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
-

hw/virtio/vhost-backend.c line 34

Quoted 10 lines
hw/virtio/vhost-backend.c +10 -0
+    int fd = -1;
+    struct vhost_vdpa *v = NULL;
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL) {
+        fd  = (uintptr_t) dev->opaque;
+    }
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        v = dev->opaque;
+        fd = v->device_fd;
+    }
+    assert(fd != -1);

A switch would be cleaner:

    switch (dev->vhost_ops->backend_type) {
    case VHOST_BACKEND_TYPE_KERNEL:
        fd  = (uintptr_t)dev->opaque;
        break;
    case VHOST_BACKEND_TYPE_VDPA:
        fd = ((struct vhost_vdpa *)dev->opaque)->device_fd;
        break;
    default:
        g_assert_not_reached()
    }

hw/virtio/vhost-backend.c line 37

Quoted 3 lines
hw/virtio/vhost-backend.c +0 -0
     return ioctl(fd, request, arg);
 }
 

Thanks,
Laurent

author
#30 lore raw

Re: [RFC v3 7/8] vhost-vdpa: introduce vhost-vdpa backend

In reply to Laurent Vivier
Quoted 35 lines · Laurent Vivier
Quoted 21 lines · Eric Blake
Quoted 1 line · Cindy Lu

From: Tiwei Bie <[email protected]>

The original author is Tiwei Bie...

Quoted 16 lines

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration
interface for setting up a vhost HW accelerator, this patch set introduces
a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \
Quoted 7 lines
Quoted 3 lines

---

...but there is no S-o-b here. Also, Co-Authored-By is an unusual tag; it's just as easy to spell it Signed-off-by even for co-authors.

Normally the tag to use in this case is "Co-developed-by".

https://www.kernel.org/doc/html/v4.17/process/submitting-patches.html#when-to-use-acked-by-cc-and-co-developed-by

Quoted 8 lines

"A Co-Developed-by: states that the patch was also created by another developer along with the original author. This is useful at times when multiple people work on a single patch. Note, this person also needs to have a Signed-off-by: line in the patch as well."

Thanks,
Laurent

Thanks Laurent,I will fix this

author
#13 lore raw

Re: [RFC v3 5/8] vhost: introduce vhost_set_vring_ready method

In reply to Laurent Vivier
Quoted 9 lines · Laurent Vivier
Quoted 7 lines · Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Quoted 9 lines
Quoted 6 lines

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

Add your S-o-b.

will fix this

Quoted 14 lines
Quoted 14 lines
---
 hw/net/vhost_net-stub.c |  4 ++++
 hw/net/vhost_net.c      | 11 ++++++++++-
 include/net/vhost_net.h |  1 +
 3 files changed, 15 insertions(+), 1 deletion(-)
hw/net/vhost_net-stub.c +0 -0
index aac0e98228..43e93e1a9a 100644
@@ -86,6 +86,10 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
Quoted 7 lines

hw/net/vhost_net-stub.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 92

Quoted 4 lines
hw/net/vhost_net-stub.c +4 -0
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}

Add a blank line here.

will fix this

Quoted 11 lines
Quoted 11 lines
hw/net/vhost_net-stub.c +0 -0
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
hw/net/vhost_net.c +0 -0
index d1d421e3d9..e2bc7de2eb 100644
@@ -344,7 +344,7 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
             goto err_start;
         }
 
Quoted 7 lines

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, · line 349

Quoted 4 lines
hw/net/vhost_net.c +1 -1
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
             /* restore vring enable state */
             r = vhost_set_vring_enable(peer, peer->vring_enable);

Move this part to PATCH 2/8

will fix this

Quoted 13 lines

hw/net/vhost_net.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 462

Quoted 9 lines
hw/net/vhost_net.c +5 -0
@@ -455,6 +455,15 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {

The structure VhostOps doesn't declare the vhost_set_vring_ready field. Your patch is missing something and it could be not built.

Quoted 4 lines

It is defined in PATCH 7/8. If you want to keep this patch you should move the declaration of "vhost_set_vring_ready_op vhost_set_vring_ready" (and related) to this patch.

Thanks Laurent, I will fix this

Quoted 7 lines

hw/net/vhost_net.c int vhost_set_vring_enable(NetClientState *nc, int enable) · line 466

Quoted 4 lines
hw/net/vhost_net.c +4 -0
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+    return 0;
+}

Add a blank line.

sure will fix this

Quoted 20 lines
Quoted 9 lines
hw/net/vhost_net.c +0 -0
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     const VhostOps *vhost_ops = net->dev.vhost_ops;
include/net/vhost_net.h +0 -0
index 77e47398c4..8a6f208189 100644

include/net/vhost_net.h int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr); · line 41

Quoted 7 lines
include/net/vhost_net.h +1 -0
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
 
 int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState *nc);
 
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 

Thanks,
Laurent

author
#8 lore raw

Re: [RFC v3 3/8] virtio-bus: introduce queue_enabled method

In reply to Laurent Vivier
Quoted 4 lines · Laurent Vivier
Quoted 2 lines · Cindy Lu

From: Jason Wang <[email protected]>

Quoted 9 lines
Quoted 5 lines

This patch introduces queue_enabled() method which allows the transport to implement its own way to report whether or not a queue is enabled.

Cindy, you must add your signed-off-by on all the patch you send, after all the existing S-o-b.

sure will fix this

Quoted 3 lines
Quoted 2 lines

0005-virtio-bus-introduce-queue_enabled-method.patch

will remove this part

Quoted 5 lines

bad cut&paste?

Thanks,
Laurent

author
#16 lore raw

Re: [RFC v3 6/8] vhost-backend: export the vhost backend helper

In reply to Laurent Vivier
Quoted 47 lines · Laurent Vivier
Quoted 13 lines · Cindy Lu

export the helper then we can reuse some of them in vhost-vdpa

---
 hw/virtio/vhost-backend.c         | 34 ++++++++++++++++++-------------
 include/hw/virtio/vhost-backend.h | 28 +++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 14 deletions(-)
hw/virtio/vhost-backend.c +0 -0
index 48905383f8..42efb4967b 100644

hw/virtio/vhost-backend.c line 17

Quoted 5 lines
hw/virtio/vhost-backend.c +1 -1
@@ -14,7 +14,7 @@
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "standard-headers/linux/vhost_types.h"
-
+#include "hw/virtio/vhost-vdpa.h"

You can't include this file because it is created in the next patch.

Quoted 11 lines
hw/virtio/vhost-backend.c +0 -4
 #ifdef CONFIG_VHOST_KERNEL
 #include <linux/vhost.h>
 #include <sys/ioctl.h>
@@ -22,10 +22,16 @@
 static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
                              void *arg)
 {
-    int fd = (uintptr_t) dev->opaque;
-
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
-

hw/virtio/vhost-backend.c line 34

Quoted 10 lines
hw/virtio/vhost-backend.c +10 -0
+    int fd = -1;
+    struct vhost_vdpa *v = NULL;
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL) {
+        fd  = (uintptr_t) dev->opaque;
+    }
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        v = dev->opaque;
+        fd = v->device_fd;
+    }
+    assert(fd != -1);

A switch would be cleaner:

Quoted 15 lines
    switch (dev->vhost_ops->backend_type) {
    case VHOST_BACKEND_TYPE_KERNEL:
        fd  = (uintptr_t)dev->opaque;
        break;
    case VHOST_BACKEND_TYPE_VDPA:
        fd = ((struct vhost_vdpa *)dev->opaque)->device_fd;
        break;
    default:
        g_assert_not_reached()
    }
Quoted 3 lines
hw/virtio/vhost-backend.c +0 -0
     return ioctl(fd, request, arg);
 }
 

Thanks Laurent , will fix this

Quoted 3 lines

Thanks,
Laurent