Skip to contentSkip to conversation

vDPA support in qemu

RFC Superseded Cindy Lu 5 participants 28 messages Apr 20, 2020 – May 7, 2020 Networking

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Cindy Lu

author
#0 lore raw

cover vDPA support in qemu

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. This RFC introduce the vDPA support in qemu

Cindy Lu (3):
net: Introduce qemu_get_peer
vhost-vdpa: introduce vhost-vdpa net client
vhost-vdpa: implement vhost-vdpa backend

Jason Wang (1):
vhost: introduce vhost_set_vring_ready method

 hw/net/vhost_net-stub.c           |   5 +
 hw/net/vhost_net.c                |  75 +++++-
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 376 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 hw/virtio/virtio-pci.c            |  13 ++
 hw/virtio/virtio.c                |   6 +
 include/hw/virtio/vhost-backend.h |   8 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 include/hw/virtio/virtio-bus.h    |   4 +
 include/net/net.h                 |   1 +
 include/net/vhost-vdpa.h          |  18 ++
 include/net/vhost_net.h           |   2 +
 net/Makefile.objs                 |   2 +-
 net/clients.h                     |   2 +
 net/net.c                         |   7 +
 net/vhost-vdpa.c                  | 211 +++++++++++++++++
 qapi/net.json                     |  21 +-
 20 files changed, 773 insertions(+), 11 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
Signature · 1 line
2.21.1
author
#1 lore raw

RFC 1/4 net: Introduce qemu_get_peer

This is a small function that can get the peer from given NetClientState and queue_index

---
 hw/net/vhost_net.c | 16 ++++++++++------
 include/net/net.h  |  1 +
 net/net.c          |  6 ++++++
 3 files changed, 17 insertions(+), 6 deletions(-)
hw/net/vhost_net.c +10 -6
index 6b82803fa7..4096d64aaf 100644
@@ -306,7 +306,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
+    struct vhost_net *net;
     int r, e, i;
+    NetClientState *peer;
 
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
@@ -314,9 +316,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        struct vhost_net *net;
 
-        net = get_vhost_net(ncs[i].peer);
+        peer = qemu_get_peer(ncs, i);
+        net = get_vhost_net(peer);
         vhost_net_set_vq_index(net, i * 2);
 
         /* Suppress the masking guest notifiers on vhost user
@@ -335,15 +337,16 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
     }
 
     for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs, i);
+        r = vhost_net_start_one(get_vhost_net(peer), dev);
 
         if (r < 0) {
             goto err_start;
         }
 
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
             /* restore vring enable state */
-            r = vhost_set_vring_enable(ncs[i].peer, ncs[i].peer->vring_enable);
+            r = vhost_set_vring_enable(peer, peer->vring_enable);
 
             if (r < 0) {
                 goto err_start;
@@ -355,7 +358,8 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
 err_start:
     while (--i >= 0) {
-        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs , i);
+        vhost_net_stop_one(get_vhost_net(peer), dev);
     }
     e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
     if (e < 0) {
include/net/net.h +1 -0
index e175ba9677..0a74324ccd 100644
@@ -175,6 +175,7 @@ void hmp_info_network(Monitor *mon, const QDict *qdict);
 void net_socket_rs_init(SocketReadState *rs,
                         SocketReadStateFinalize *finalize,
                         bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
 /* NIC info */
 
net/net.c +6 -0
index 84aa6d8d00..ac5080dda1 100644
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
     return nic->opaque;
 }
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    NetClientState *ncs  =  nc + queue_index;
+    assert(ncs != NULL);
+    return ncs->peer;
+}
 
 static void qemu_cleanup_net_client(NetClientState *nc)
 {
Signature · 1 line
2.21.1
#2 lore raw

Re: [RFC v1 1/4] net: Introduce qemu_get_peer

Quoted 1 line · Cindy Lu

This is a small function that can get the peer from given NetClientState and queue_index

Unnecessary space  between 'function' and 'that'.

Quoted 2 lines

Please split this patch into two parts:

1) introduce the function
2) the actual user for this fucntion

Quoted 79 lines
---
  hw/net/vhost_net.c | 16 ++++++++++------
  include/net/net.h  |  1 +
  net/net.c          |  6 ++++++
  3 files changed, 17 insertions(+), 6 deletions(-)
hw/net/vhost_net.c +10 -6
index 6b82803fa7..4096d64aaf 100644
@@ -306,7 +306,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
      BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
      VirtioBusState *vbus = VIRTIO_BUS(qbus);
      VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
+    struct vhost_net *net;
      int r, e, i;
+    NetClientState *peer;
 
      if (!k->set_guest_notifiers) {
          error_report("binding does not support guest notifiers");
@@ -314,9 +316,9 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
      }
 
      for (i = 0; i < total_queues; i++) {
-        struct vhost_net *net;
 
-        net = get_vhost_net(ncs[i].peer);
+        peer = qemu_get_peer(ncs, i);
+        net = get_vhost_net(peer);
          vhost_net_set_vq_index(net, i * 2);
 
          /* Suppress the masking guest notifiers on vhost user
@@ -335,15 +337,16 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
      }
 
      for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs, i);
+        r = vhost_net_start_one(get_vhost_net(peer), dev);
 
          if (r < 0) {
              goto err_start;
          }
 
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
              /* restore vring enable state */
-            r = vhost_set_vring_enable(ncs[i].peer, ncs[i].peer->vring_enable);
+            r = vhost_set_vring_enable(peer, peer->vring_enable);
 
              if (r < 0) {
                  goto err_start;
@@ -355,7 +358,8 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
 
  err_start:
      while (--i >= 0) {
-        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs , i);
+        vhost_net_stop_one(get_vhost_net(peer), dev);
      }
      e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
      if (e < 0) {
include/net/net.h +1 -0
index e175ba9677..0a74324ccd 100644
@@ -175,6 +175,7 @@ void hmp_info_network(Monitor *mon, const QDict *qdict);
  void net_socket_rs_init(SocketReadState *rs,
                          SocketReadStateFinalize *finalize,
                          bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
  /* NIC info */
 
net/net.c +0 -0
index 84aa6d8d00..ac5080dda1 100644

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 329

Quoted 6 lines
net/net.c +3 -0
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
      return nic->opaque;
  }
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    NetClientState *ncs  =  nc + queue_index;

Unnecessary space around '='.

Thanks

Quoted 6 lines
net/net.c +3 -0
+    assert(ncs != NULL);
+    return ncs->peer;
+}
 
  static void qemu_cleanup_net_client(NetClientState *nc)
  {
author
#3 lore raw

Re: [RFC v1 1/4] net: Introduce qemu_get_peer

Quoted 129 lines · Jason Wang
Quoted 1 line · Cindy Lu

This is a small function that can get the peer from given

NetClientState and queue_index

Unnecessary space between 'function' and 'that'.

Quoted 2 lines

Please split this patch into two parts:

1) introduce the function
2) the actual user for this fucntion

sure, I will fix this problem.

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev,

Quoted 11 lines
---
  hw/net/vhost_net.c | 16 ++++++++++------
  include/net/net.h  |  1 +
  net/net.c          |  6 ++++++
  3 files changed, 17 insertions(+), 6 deletions(-)
hw/net/vhost_net.c +0 -0
index 6b82803fa7..4096d64aaf 100644
@@ -306,7 +306,9 @@ int vhost_net_start(VirtIODevice *dev,

NetClientState *ncs,

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev,

Quoted 10 lines
hw/net/vhost_net.c +2 -0
      BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(dev)));
      VirtioBusState *vbus = VIRTIO_BUS(qbus);
      VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
+    struct vhost_net *net;
      int r, e, i;
+    NetClientState *peer;
 
      if (!k->set_guest_notifiers) {
          error_report("binding does not support guest notifiers");
@@ -314,9 +316,9 @@ int vhost_net_start(VirtIODevice *dev,

NetClientState *ncs,

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev,

Quoted 12 lines
hw/net/vhost_net.c +2 -2
      }
 
      for (i = 0; i < total_queues; i++) {
-        struct vhost_net *net;
 
-        net = get_vhost_net(ncs[i].peer);
+        peer = qemu_get_peer(ncs, i);
+        net = get_vhost_net(peer);
          vhost_net_set_vq_index(net, i * 2);
 
          /* Suppress the masking guest notifiers on vhost user
@@ -335,15 +337,16 @@ int vhost_net_start(VirtIODevice *dev,

NetClientState *ncs,

Quoted 5 lines
hw/net/vhost_net.c +1 -1
      }
 
      for (i = 0; i < total_queues; i++) {
-        r = vhost_net_start_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs, i);

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,

Quoted 10 lines
hw/net/vhost_net.c +2 -2
+        r = vhost_net_start_one(get_vhost_net(peer), dev);
 
          if (r < 0) {
              goto err_start;
          }
 
-        if (ncs[i].peer->vring_enable) {
+        if (peer->vring_enable) {
              /* restore vring enable state */
-            r = vhost_set_vring_enable(ncs[i].peer,

ncs[i].peer->vring_enable);

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev,

Quoted 5 lines
hw/net/vhost_net.c +1 -0
+            r = vhost_set_vring_enable(peer, peer->vring_enable);
 
              if (r < 0) {
                  goto err_start;
@@ -355,7 +358,8 @@ int vhost_net_start(VirtIODevice *dev,

NetClientState *ncs,

include/net/net.h void hmp_info_network(Monitor *mon, const QDict

Quoted 14 lines
hw/net/vhost_net.c +2 -1
  err_start:
      while (--i >= 0) {
-        vhost_net_stop_one(get_vhost_net(ncs[i].peer), dev);
+        peer = qemu_get_peer(ncs , i);
+        vhost_net_stop_one(get_vhost_net(peer), dev);
      }
      e = k->set_guest_notifiers(qbus->parent, total_queues * 2, false);
      if (e < 0) {
include/net/net.h +0 -0
index e175ba9677..0a74324ccd 100644
@@ -175,6 +175,7 @@ void hmp_info_network(Monitor *mon, const QDict

*qdict);

Quoted 12 lines
include/net/net.h +1 -0
  void net_socket_rs_init(SocketReadState *rs,
                          SocketReadStateFinalize *finalize,
                          bool vnet_hdr);
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index);
 
  /* NIC info */
 
net/net.c +0 -0
index 84aa6d8d00..ac5080dda1 100644

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 329

Quoted 6 lines
net/net.c +3 -0
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
      return nic->opaque;
  }
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    NetClientState *ncs  =  nc + queue_index;

Unnecessary space around '='.

Thanks

Will correct this

+ assert(ncs != NULL);

Quoted 5 lines
net/net.c +2 -0
+    return ncs->peer;
+}
 
  static void qemu_cleanup_net_client(NetClientState *nc)
  {
#4 lore raw

Re: [RFC v1 1/4] net: Introduce qemu_get_peer

In reply to Cindy Lu
Quoted 9 lines · Cindy Lu

This is a small function that can get the peer from given NetClientState and queue_index

---
 hw/net/vhost_net.c | 16 ++++++++++------
 include/net/net.h  |  1 +
 net/net.c          |  6 ++++++
 3 files changed, 17 insertions(+), 6 deletions(-)

...

net/net.c void *qemu_get_nic_opaque(NetClientState *nc) · line 330

Quoted 12 lines
net/net.c +4 -0
index 84aa6d8d00..ac5080dda1 100644
@@ -324,6 +324,12 @@ void *qemu_get_nic_opaque(NetClientState *nc)
 
     return nic->opaque;
 }
+NetClientState *qemu_get_peer(NetClientState *nc, int queue_index)
+{
+    NetClientState *ncs  =  nc + queue_index;
+    assert(ncs != NULL);

This will not assert if nc is NULL and queue_index != 0.

You should check value of nc and then calculate ncs.

Thanks,
Laurent

author
#5 lore raw

RFC 2/4 vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is svhostdevpecified by a "vhostdev" parameter.

Author: Tiwei Bie

---
 include/net/vhost-vdpa.h |  18 ++++
 include/net/vhost_net.h  |   1 +
 net/Makefile.objs        |   2 +-
 net/clients.h            |   2 +
 net/net.c                |   1 +
 net/vhost-vdpa.c         | 211 +++++++++++++++++++++++++++++++++++++++
 qapi/net.json            |  21 +++-
 7 files changed, 253 insertions(+), 3 deletions(-)
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
include/net/vhost-vdpa.h new file +18 -0
new file mode 100644
index 0000000000..9ddd538dad
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -0
index 77e47398c4..6f3a624cf7 100644
@@ -39,5 +39,6 @@ int vhost_set_vring_enable(NetClientState * nc, int enable);
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
+int vhost_set_state(NetClientState *nc, int state);
 
 #endif
net/Makefile.objs +1 -1
index c5d076d19c..da459cfc19 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
 tap-obj-y ?= tap-stub.o
 common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
 common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_KERNEL) += vhost-vdpa.o
 vde.o-libs = $(VDE_LIBS)
 
 common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
 int net_init_vhost_user(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
 #endif /* QEMU_NET_CLIENTS_H */
net/net.c +1 -0
index ac5080dda1..2beb95388a 100644
@@ -964,6 +964,7 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
         [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
 #ifdef CONFIG_VHOST_NET_USER
         [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
 #endif
 #ifdef CONFIG_L2TPV3
         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
net/vhost-vdpa.c new file +211 -0

211 lines not on this page · raw message

qapi/net.json +19 -2
index 335295be50..35a5ccee39 100644
@@ -441,6 +441,22 @@
     '*queues':        'int' } }
 
 ##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a mdev dev path in sysfs
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1) (Since 2.11)
+#
+# Since: 2.11
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*queues':       'int' } }
+##
 # @NetClientDriver:
 #
 # Available netdev drivers.
@@ -451,7 +467,7 @@
 ##
 { 'enum': 'NetClientDriver',
   'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
 ##
 # @Netdev:
@@ -479,7 +495,8 @@
     'bridge':   'NetdevBridgeOptions',
     'hubport':  'NetdevHubPortOptions',
     'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
 ##
 # @NetLegacy:
Signature · 1 line
2.21.1
#6 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

Quoted 2 lines · Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is svhostdevpecified

looks like you pasted 'vhostdev' in the middle of 'specified'

Quoted 3 lines

by a "vhostdev" parameter.

Author: Tiwei Bie

Should this be a 'Signed-off-by' tag?

include/net/vhost-vdpa.h line 6

Quoted 11 lines

---

+++ b/include/net/vhost-vdpa.h

include/net/vhost-vdpa.h new file +6 -0
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

Claiming "All rights reserved" is at odds with claiming "GPL", which specifically requires that you are NOT reserving all rights but are instead granting various rights to others insofar as they preserve this code as free software. I can overlook it on BSD licenses (as there are lots of examples of bad copy-and-paste on various templates that uses the misleading term), but not on GPL licenses. Also, you may want to consider if adding 2020 to your copyright date is appropriate.

net/vhost-vdpa.c line 7

Quoted 9 lines
net/vhost-vdpa.c new file +8 -0
+++ b/net/vhost-vdpa.c
@@ -0,0 +1,211 @@
+/*
+ * vhost-vdpa.c
+ *
+ * Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

Another inconsistent license.

qapi/net.json line 451

Quoted 13 lines
qapi/net.json +9 -0
+++ b/qapi/net.json
@@ -441,6 +441,22 @@
      '*queues':        'int' } }
 
  ##
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a mdev dev path in sysfs
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1) (Since 2.11)

There's no need to add a 'since' tag for an individual member if...

Review line 453

Quoted 2 lines
(quoted diff) +2 -0
+#
+# Since: 2.11

...the struct itself was introduced in the same release. However, using 2.11 as the release is wrong; the next release will be 5.1.

qapi/net.json line 468

Quoted 12 lines
qapi/net.json +6 -0
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*queues':       'int' } }
+##
  # @NetClientDriver:
  #
  # Available netdev drivers.
@@ -451,7 +467,7 @@
  ##
  { 'enum': 'NetClientDriver',

Missing documentation that 'vhost-vdpa' was added in 5.1.

Quoted 17 lines
qapi/net.json +3 -2
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
  ##
  # @Netdev:
@@ -479,7 +495,8 @@
      'bridge':   'NetdevBridgeOptions',
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
  ##
  # @NetLegacy:
Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
#7 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu
Quoted 2 lines · Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is svhostdevpecified

typo.

Quoted 3 lines

by a "vhostdev" parameter.

Author: Tiwei Bie

Please keep the sobs from the original patch.

Quoted 10 lines
---
  include/net/vhost-vdpa.h |  18 ++++
  include/net/vhost_net.h  |   1 +
  net/Makefile.objs        |   2 +-
  net/clients.h            |   2 +
  net/net.c                |   1 +
  net/vhost-vdpa.c         | 211 +++++++++++++++++++++++++++++++++++++++
  qapi/net.json            |  21 +++-
  7 files changed, 253 insertions(+), 3 deletions(-)
Quoted 2 lines

create mode 100644 include/net/vhost-vdpa.h
create mode 100644 net/vhost-vdpa.c

I think this patch should come after patch 3.

And it's better to make this as a module like vhost-user.

include/net/vhost-vdpa.h line 4

Quoted 11 lines
include/net/vhost-vdpa.h new file +4 -0
new file mode 100644
index 0000000000..9ddd538dad
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.

2020 and please add Red Hat here as well.

Quoted 19 lines
include/net/vhost-vdpa.h new file +14 -0
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +0 -0
index 77e47398c4..6f3a624cf7 100644

include/net/vhost_net.h int vhost_set_vring_enable(NetClientState * nc, int enable); · line 42

Quoted 4 lines
include/net/vhost_net.h +1 -0
@@ -39,5 +39,6 @@ int vhost_set_vring_enable(NetClientState * nc, int enable);
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
+int vhost_set_state(NetClientState *nc, int state);

This function is not used in this patch.

Quoted 51 lines
include/net/vhost_net.h +0 -0
  #endif
net/Makefile.objs +1 -1
index c5d076d19c..da459cfc19 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
  tap-obj-y ?= tap-stub.o
  common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
  common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_KERNEL) += vhost-vdpa.o
  vde.o-libs = $(VDE_LIBS)
 
  common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
  int net_init_vhost_user(const Netdev *netdev, const char *name,
                          NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
  #endif /* QEMU_NET_CLIENTS_H */
net/net.c +1 -0
index ac5080dda1..2beb95388a 100644
@@ -964,6 +964,7 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
          [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
  #ifdef CONFIG_VHOST_NET_USER
          [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
  #endif
  #ifdef CONFIG_L2TPV3
          [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
net/vhost-vdpa.c new file +7 -0
new file mode 100644
index 0000000000..5daeba0b76
@@ -0,0 +1,211 @@
+/*
+ * vhost-vdpa.c
+ *
+ * Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

net/vhost-vdpa.c line 17

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "clients.h"
+#include "net/vhost_net.h"
+#include "net/vhost-vdpa.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "chardev/char-fe.h"

I don't think we use charfe in this file.

net/vhost-vdpa.c line 22

Quoted 5 lines
net/vhost-vdpa.c new file +5 -0
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "trace.h"

I think we don't use tracing in this file.

net/vhost-vdpa.c line 30

Quoted 8 lines
net/vhost-vdpa.c new file +8 -0
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+#include <err.h>
+#include <linux/virtio_net.h>
+
+
+typedef struct VhostVDPAState {
+    NetClientState nc;

This may not work for the case of multiqueue, you can either fix this or just leave a comment for TODO.

net/vhost-vdpa.c line 31

Quoted 1 line
net/vhost-vdpa.c new file +1 -0
+    struct vhost_vdpa vhost_vdpa;

This explains why patch 3 should come first.

Quoted 29 lines
net/vhost-vdpa.c new file +29 -0
+    VHostNetState *vhost_net;
+    uint64_t acked_features;
+    bool started;
+} VhostVDPAState;
+
+VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->vhost_net;
+}
+
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->acked_features;
+}
+
+static void vhost_vdpa_stop(NetClientState *ncs)
+{
+    VhostVDPAState *s;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    if (s->vhost_net) {
+        /* save acked features */

net/vhost-vdpa.c line 70

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+        uint64_t features = vhost_net_get_acked_features(s->vhost_net);
+        if (features) {
+            s->acked_features = features;
+         }
+        vhost_net_cleanup(s->vhost_net);
+    }
+}
+
+static int vhost_vdpa_start(NetClientState *ncs, void *be)
+{

The name of this function is confusing, we don't start vhost_vdpa actually.

Quoted 83 lines
net/vhost-vdpa.c new file +83 -0
+    VhostNetOptions options;
+    struct vhost_net *net = NULL;
+    VhostVDPAState *s;
+
+    options.backend_type = VHOST_BACKEND_TYPE_VDPA;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    options.net_backend = ncs;
+    options.opaque      = be;
+    options.busyloop_timeout = 0;
+    net = vhost_net_init(&options);
+    if (!net) {
+        error_report("failed to init vhost_net for queue");
+        goto err;
+     }
+
+     if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+     }
+     s->vhost_net = net;
+
+    return 0;
+
+err:
+    if (net) {
+        vhost_net_cleanup(net);
+    }
+    vhost_vdpa_stop(ncs);
+    return -1;
+}
+static void vhost_vdpa_cleanup(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);
+}
+
+static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    return true;
+}
+
+static bool vhost_vdpa_has_ufo(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    uint64_t  features = 0;
+
+    features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
+    features = vhost_net_get_features(s->vhost_net, features);
+    return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
+
+}
+
+static NetClientInfo net_vhost_vdpa_info = {
+        .type = NET_CLIENT_DRIVER_VHOST_VDPA,
+        .size = sizeof(VhostVDPAState),
+        .cleanup = vhost_vdpa_cleanup,
+        .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
+        .has_ufo = vhost_vdpa_has_ufo,
+};
+
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev)
+{
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);

net/vhost-vdpa.c line 163

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    vdpa_device_fd = open(vhostdev, O_RDWR);
+    if (vdpa_device_fd == -1) {
+        return -errno;
+    }
+    s->vhost_vdpa.device_fd = vdpa_device_fd;

Need to add a step to verify the device type and fail if it was not a networking device.

Quoted 35 lines
net/vhost-vdpa.c new file +35 -0
+    vhost_vdpa_start(nc, (void *)&s->vhost_vdpa);
+
+    assert(s->vhost_net);
+
+    return 0;
+}
+
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");
+
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
+        return -1;
+    }
+
+    return 0;
+}
+
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
+{
+    const NetdevVhostVDPAOptions *vhost_vdpa_opts;
+
+    assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    vhost_vdpa_opts = &netdev->u.vhost_vdpa;

net/vhost-vdpa.c line 208

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+
+    /* verify net frontend */
+    if (qemu_opts_foreach(qemu_find_opts("device"), net_vhost_check_net,
+                          (char *)name, errp)) {
+        return -1;
+    }
+
+
+    return net_vhost_vdpa_init(peer, "vhost_vdpa", name,
+                               vhost_vdpa_opts->vhostdev);

Please add the support for passing fd via command line.

Quoted 11 lines
net/vhost-vdpa.c new file +3 -0
+
+    return 0;
+}
qapi/net.json +0 -0
index 335295be50..35a5ccee39 100644
@@ -441,6 +441,22 @@
      '*queues':        'int' } }
 
  ##

qapi/net.json line 453

Quoted 10 lines
qapi/net.json +10 -0
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a mdev dev path in sysfs
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1) (Since 2.11)
+#
+# Since: 2.11

The version number is wrong, I guess it's probably 5.1.

Thanks

Quoted 28 lines
qapi/net.json +9 -2
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*queues':       'int' } }
+##
  # @NetClientDriver:
  #
  # Available netdev drivers.
@@ -451,7 +467,7 @@
  ##
  { 'enum': 'NetClientDriver',
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ] }
 
  ##
  # @Netdev:
@@ -479,7 +495,8 @@
      'bridge':   'NetdevBridgeOptions',
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
  ##
  # @NetLegacy:
author
#8 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

Quoted 385 lines · Jason Wang
Quoted 2 lines · Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is

svhostdevpecified

typo.

Will fix this

Quoted 3 lines

by a "vhostdev" parameter.

Author: Tiwei Bie

Please keep the sobs from the original patch.

Will fix this

Quoted 10 lines
---
  include/net/vhost-vdpa.h |  18 ++++
  include/net/vhost_net.h  |   1 +
  net/Makefile.objs        |   2 +-
  net/clients.h            |   2 +
  net/net.c                |   1 +
  net/vhost-vdpa.c         | 211 +++++++++++++++++++++++++++++++++++++++
  qapi/net.json            |  21 +++-
  7 files changed, 253 insertions(+), 3 deletions(-)
Quoted 2 lines

create mode 100644 include/net/vhost-vdpa.h
create mode 100644 net/vhost-vdpa.c

I think this patch should come after patch 3.

And it's better to make this as a module like vhost-user.

Got it, Will fix this

include/net/vhost-vdpa.h line 4

Quoted 11 lines
include/net/vhost-vdpa.h new file +4 -0
new file mode 100644
index 0000000000..9ddd538dad
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.

2020 and please add Red Hat here as well.

Will fix this

include/net/vhost-vdpa.h line 6

Quoted 2 lines
include/net/vhost-vdpa.h new file +2 -0
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or

later.

include/net/vhost_net.h int vhost_set_vring_enable(NetClientState * nc, int

Quoted 17 lines
include/net/vhost-vdpa.h new file +12 -0
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +0 -0
index 77e47398c4..6f3a624cf7 100644
@@ -39,5 +39,6 @@ int vhost_set_vring_enable(NetClientState * nc, int

enable);

include/net/vhost_net.h int vhost_set_vring_enable(NetClientState * nc, int enable); · line 42

Quoted 4 lines
include/net/vhost_net.h +1 -0
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
+int vhost_set_state(NetClientState *nc, int state);

This function is not used in this patch.

Will fix this

net/clients.h int net_init_netmap(const Netdev *netdev, const char

Quoted 20 lines
include/net/vhost_net.h +0 -0
  #endif
net/Makefile.objs +1 -1
index c5d076d19c..da459cfc19 100644
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
  tap-obj-y ?= tap-stub.o
  common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
  common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_KERNEL) += vhost-vdpa.o
  vde.o-libs = $(VDE_LIBS)
 
  common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +0 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char

*name,

net/net.c static int (* const

Quoted 11 lines
net/clients.h +2 -0
  int net_init_vhost_user(const Netdev *netdev, const char *name,
                          NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
  #endif /* QEMU_NET_CLIENTS_H */
net/net.c +0 -0
index ac5080dda1..2beb95388a 100644
@@ -964,6 +964,7 @@ static int (* const

net_client_init_fun[NET_CLIENT_DRIVER__MAX])(

Quoted 13 lines
net/net.c +1 -0
          [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
  #ifdef CONFIG_VHOST_NET_USER
          [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
  #endif
  #ifdef CONFIG_L2TPV3
          [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
net/vhost-vdpa.c new file +0 -0
new file mode 100644
index 0000000000..5daeba0b76

net/vhost-vdpa.c line 7

Quoted 7 lines
net/vhost-vdpa.c new file +7 -0
@@ -0,0 +1,211 @@
+/*
+ * vhost-vdpa.c
+ *
+ * Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ * Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or

later.

net/vhost-vdpa.c line 17

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "clients.h"
+#include "net/vhost_net.h"
+#include "net/vhost-vdpa.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "chardev/char-fe.h"

I don't think we use charfe in this file.

net/vhost-vdpa.c line 22

Quoted 5 lines
net/vhost-vdpa.c new file +5 -0
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "qemu/option.h"
+#include "qapi/error.h"
+#include "trace.h"

I think we don't use tracing in this file.

Will fix this

net/vhost-vdpa.c line 30

Quoted 8 lines
net/vhost-vdpa.c new file +8 -0
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+#include <err.h>
+#include <linux/virtio_net.h>
+
+
+typedef struct VhostVDPAState {
+    NetClientState nc;

This may not work for the case of multiqueue, you can either fix this or just leave a comment for TODO.

Will fix this

net/vhost-vdpa.c line 31

Quoted 1 line
net/vhost-vdpa.c new file +1 -0
+    struct vhost_vdpa vhost_vdpa;

This explains why patch 3 should come first.

Quoted 29 lines
net/vhost-vdpa.c new file +29 -0
+    VHostNetState *vhost_net;
+    uint64_t acked_features;
+    bool started;
+} VhostVDPAState;
+
+VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->vhost_net;
+}
+
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    return s->acked_features;
+}
+
+static void vhost_vdpa_stop(NetClientState *ncs)
+{
+    VhostVDPAState *s;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    if (s->vhost_net) {
+        /* save acked features */

net/vhost-vdpa.c line 70

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+        uint64_t features = vhost_net_get_acked_features(s->vhost_net);
+        if (features) {
+            s->acked_features = features;
+         }
+        vhost_net_cleanup(s->vhost_net);
+    }
+}
+
+static int vhost_vdpa_start(NetClientState *ncs, void *be)
+{

The name of this function is confusing, we don't start vhost_vdpa actually.

ok Got it , I will change it

Quoted 83 lines
net/vhost-vdpa.c new file +83 -0
+    VhostNetOptions options;
+    struct vhost_net *net = NULL;
+    VhostVDPAState *s;
+
+    options.backend_type = VHOST_BACKEND_TYPE_VDPA;
+
+    assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    s = DO_UPCAST(VhostVDPAState, nc, ncs);
+
+    options.net_backend = ncs;
+    options.opaque      = be;
+    options.busyloop_timeout = 0;
+    net = vhost_net_init(&options);
+    if (!net) {
+        error_report("failed to init vhost_net for queue");
+        goto err;
+     }
+
+     if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+     }
+     s->vhost_net = net;
+
+    return 0;
+
+err:
+    if (net) {
+        vhost_net_cleanup(net);
+    }
+    vhost_vdpa_stop(ncs);
+    return -1;
+}
+static void vhost_vdpa_cleanup(NetClientState *nc)
+{
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+
+    qemu_purge_queued_packets(nc);
+}
+
+static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+
+    return true;
+}
+
+static bool vhost_vdpa_has_ufo(NetClientState *nc)
+{
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
+    uint64_t  features = 0;
+
+    features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
+    features = vhost_net_get_features(s->vhost_net, features);
+    return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
+
+}
+
+static NetClientInfo net_vhost_vdpa_info = {
+        .type = NET_CLIENT_DRIVER_VHOST_VDPA,
+        .size = sizeof(VhostVDPAState),
+        .cleanup = vhost_vdpa_cleanup,
+        .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
+        .has_ufo = vhost_vdpa_has_ufo,
+};
+
+static int net_vhost_vdpa_init(NetClientState *peer, const char *device,
+                               const char *name, const char *vhostdev)
+{
+    NetClientState *nc = NULL;
+    VhostVDPAState *s;
+    int vdpa_device_fd;
+    assert(name);
+
+    nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device, name);

net/vhost-vdpa.c line 163

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-vdpa");
+    nc->queue_index = 0;
+
+    s = DO_UPCAST(VhostVDPAState, nc, nc);
+
+    vdpa_device_fd = open(vhostdev, O_RDWR);
+    if (vdpa_device_fd == -1) {
+        return -errno;
+    }
+    s->vhost_vdpa.device_fd = vdpa_device_fd;

Need to add a step to verify the device type and fail if it was not a networking device.

sure will add the check .

net/vhost-vdpa.c

Quoted 8 lines
net/vhost-vdpa.c new file +8 -0
+    vhost_vdpa_start(nc, (void *)&s->vhost_vdpa);
+
+    assert(s->vhost_net);
+
+    return 0;
+}
+
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error

**errp)

Quoted 4 lines
net/vhost-vdpa.c new file +4 -0
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+

net/vhost-vdpa.c

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");
+
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver

virtio-net-*");

Quoted 18 lines
net/vhost-vdpa.c new file +18 -0
+        return -1;
+    }
+
+    return 0;
+}
+
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
+{
+    const NetdevVhostVDPAOptions *vhost_vdpa_opts;
+
+    assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
+    vhost_vdpa_opts = &netdev->u.vhost_vdpa;
+
+    /* verify net frontend */
+    if (qemu_opts_foreach(qemu_find_opts("device"), net_vhost_check_net,
+                          (char *)name, errp)) {
+        return -1;
Quoted 9 lines

net/vhost-vdpa.c line 208

Quoted 5 lines
net/vhost-vdpa.c new file +5 -0
+    }
+
+
+    return net_vhost_vdpa_init(peer, "vhost_vdpa", name,
+                               vhost_vdpa_opts->vhostdev);

Please add the support for passing fd via command line.

Will add the support

Quoted 63 lines

+

Quoted 10 lines
net/vhost-vdpa.c new file +2 -0
+    return 0;
+}
qapi/net.json +0 -0
index 335295be50..35a5ccee39 100644
@@ -441,6 +441,22 @@
      '*queues':        'int' } }
 
  ##

qapi/net.json line 453

Quoted 10 lines
qapi/net.json +10 -0
+# @NetdevVhostVDPAOptions:
+#
+# Vhost-vdpa network backend
+#
+# @vhostdev: name of a mdev dev path in sysfs
+#
+# @queues: number of queues to be created for multiqueue vhost-vdpa
+#          (default: 1) (Since 2.11)
+#
+# Since: 2.11

The version number is wrong, I guess it's probably 5.1.

Thanks

Will fix this

Quoted 10 lines
qapi/net.json +6 -0
+##
+{ 'struct': 'NetdevVhostVDPAOptions',
+  'data': {
+    '*vhostdev':     'str',
+    '*queues':       'int' } }
+##
  # @NetClientDriver:
  #
  # Available netdev drivers.

qapi/net.json line 470

Quoted 5 lines
qapi/net.json +1 -1
@@ -451,7 +467,7 @@
  ##
  { 'enum': 'NetClientDriver',
    'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde',
-            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+            'bridge', 'hubport', 'netmap', 'vhost-user', 'vhost-vdpa' ]

}

Quoted 13 lines
qapi/net.json +2 -1
  ##
  # @Netdev:
@@ -479,7 +495,8 @@
      'bridge':   'NetdevBridgeOptions',
      'hubport':  'NetdevHubPortOptions',
      'netmap':   'NetdevNetmapOptions',
-    'vhost-user': 'NetdevVhostUserOptions' } }
+    'vhost-user': 'NetdevVhostUserOptions',
+    'vhost-vdpa': 'NetdevVhostVDPAOptions' } }
 
  ##
  # @NetLegacy:
#9 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

Quoted 19 lines · Cindy Lu

On Tue, Apr 21, 2020 at 11:40 AM Jason Wang <[email protected] <mailto:[email protected]>> wrote:

On 2020/4/20 下午5:32, Cindy Lu wrote:
> This patch set introduces a new net client type: vhost-vdpa. > vhost-vdpa net client will set up a vDPA device which is svhostdevpecified

typo.

Will fix this

Quoted 2 lines

by a "vhostdev" parameter.

Quoted 7 lines
Quoted 1 line

Author: Tiwei Bie

Please keep the sobs from the original patch.

Will fix this

If you want to set Tiwei Bie as the original author of the patch you can use "git commit --author="Tiwei Bie <XXXX>" --amend" to update your patch.

When you will send the series, git-send-email will add the line "From:
Tiwei Bie <XXXX>" at the beginning of the email to identify the original author. Don't remove his Sob and don't forget yo add yours.

Thanks,
Laurent

#10 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

In reply to Cindy Lu
Quoted 58 lines · Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is svhostdevpecified by a "vhostdev" parameter.

Author: Tiwei Bie

---
 include/net/vhost-vdpa.h |  18 ++++
 include/net/vhost_net.h  |   1 +
 net/Makefile.objs        |   2 +-
 net/clients.h            |   2 +
 net/net.c                |   1 +
 net/vhost-vdpa.c         | 211 +++++++++++++++++++++++++++++++++++++++
 qapi/net.json            |  21 +++-
 7 files changed, 253 insertions(+), 3 deletions(-)
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
include/net/vhost-vdpa.h new file +18 -0
new file mode 100644
index 0000000000..9ddd538dad
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -0
index 77e47398c4..6f3a624cf7 100644
@@ -39,5 +39,6 @@ int vhost_set_vring_enable(NetClientState * nc, int enable);
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
+int vhost_set_state(NetClientState *nc, int state);
 
 #endif
net/Makefile.objs +0 -0
index c5d076d19c..da459cfc19 100644

net/Makefile.objs tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o · line 29

Quoted 5 lines
net/Makefile.objs +1 -1
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
 tap-obj-y ?= tap-stub.o
 common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
 common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_KERNEL) += vhost-vdpa.o

should it be CONFIG_VHOST_NET_USER as you use net_init_vhost_vdpa() below inside a "#ifdef CONFIG_VHOST_NET_USER"?

Why don't you define a CONFIG_VHOST_VDPA?

Quoted 19 lines
net/Makefile.objs +0 -0
 vde.o-libs = $(VDE_LIBS)
 
 common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
 int net_init_vhost_user(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
 #endif /* QEMU_NET_CLIENTS_H */
net/net.c +0 -0
index ac5080dda1..2beb95388a 100644

net/net.c static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])( · line 965

Quoted 2 lines
net/net.c +0 -0
@@ -964,6 +964,7 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
         [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
 #ifdef CONFIG_VHOST_NET_USER          ^^^^^^^^^^^^^^^^^^^^^

here

Quoted 10 lines
net/net.c +1 -0
         [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
 #endif
 #ifdef CONFIG_L2TPV3
         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
net/vhost-vdpa.c new file +0 -0
new file mode 100644
index 0000000000..5daeba0b76

...

Quoted 8 lines
net/vhost-vdpa.c new file +7 -0
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");

net/vhost-vdpa.c line 187

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
+        return -1;
+    }

So perhaps you can build the file only if CONFIG_VIRTIO_NET is set?

Thanks,
Laurent

author
#11 lore raw

Re: [RFC v1 2/4] vhost-vdpa: introduce vhost-vdpa net client

Quoted 69 lines · Laurent Vivier
Quoted 58 lines · Cindy Lu

This patch set introduces a new net client type: vhost-vdpa. vhost-vdpa net client will set up a vDPA device which is svhostdevpecified by a "vhostdev" parameter.

Author: Tiwei Bie

---
 include/net/vhost-vdpa.h |  18 ++++
 include/net/vhost_net.h  |   1 +
 net/Makefile.objs        |   2 +-
 net/clients.h            |   2 +
 net/net.c                |   1 +
 net/vhost-vdpa.c         | 211 +++++++++++++++++++++++++++++++++++++++
 qapi/net.json            |  21 +++-
 7 files changed, 253 insertions(+), 3 deletions(-)
 create mode 100644 include/net/vhost-vdpa.h
 create mode 100644 net/vhost-vdpa.c
include/net/vhost-vdpa.h new file +18 -0
new file mode 100644
index 0000000000..9ddd538dad
@@ -0,0 +1,18 @@
+/*
+ * vhost-vdpa.h
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef VHOST_VDPA_H
+#define VHOST_VDPA_H
+
+struct vhost_net;
+struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc);
+uint64_t vhost_vdpa_get_acked_features(NetClientState *nc);
+
+#endif /* VHOST_VDPA_H */
include/net/vhost_net.h +1 -0
index 77e47398c4..6f3a624cf7 100644
@@ -39,5 +39,6 @@ int vhost_set_vring_enable(NetClientState * nc, int enable);
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu);
+int vhost_set_state(NetClientState *nc, int state);
 
 #endif
net/Makefile.objs +0 -0
index c5d076d19c..da459cfc19 100644

net/Makefile.objs tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o · line 29

Quoted 5 lines
net/Makefile.objs +1 -1
@@ -26,7 +26,7 @@ tap-obj-$(CONFIG_SOLARIS) = tap-solaris.o
 tap-obj-y ?= tap-stub.o
 common-obj-$(CONFIG_POSIX) += tap.o $(tap-obj-y)
 common-obj-$(CONFIG_WIN32) += tap-win32.o
-
+common-obj-$(CONFIG_VHOST_KERNEL) += vhost-vdpa.o

should it be CONFIG_VHOST_NET_USER as you use net_init_vhost_vdpa() below inside a "#ifdef CONFIG_VHOST_NET_USER"?

Quoted 2 lines

Why don't you define a CONFIG_VHOST_VDPA?

Thanks Laurent for point it out, There is no dependence between CONFIG_VHOST_NET_USER and vDPA,
So I will implement a new macro specific for vDPA

Quoted 55 lines
Quoted 19 lines
net/Makefile.objs +0 -0
 vde.o-libs = $(VDE_LIBS)
 
 common-obj-$(CONFIG_CAN_BUS) += can/
net/clients.h +2 -0
index a6ef267e19..92f9b59aed 100644
@@ -61,4 +61,6 @@ int net_init_netmap(const Netdev *netdev, const char *name,
 int net_init_vhost_user(const Netdev *netdev, const char *name,
                         NetClientState *peer, Error **errp);
 
+int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
 #endif /* QEMU_NET_CLIENTS_H */
net/net.c +0 -0
index ac5080dda1..2beb95388a 100644

net/net.c static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])( · line 965

Quoted 2 lines
net/net.c +0 -0
@@ -964,6 +964,7 @@ static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
         [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
 #ifdef CONFIG_VHOST_NET_USER          ^^^^^^^^^^^^^^^^^^^^^

here

Quoted 10 lines
net/net.c +1 -0
         [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_VDPA] = net_init_vhost_vdpa,
 #endif
 #ifdef CONFIG_L2TPV3
         [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
net/vhost-vdpa.c new file +0 -0
new file mode 100644
index 0000000000..5daeba0b76

...

Quoted 8 lines
net/vhost-vdpa.c new file +7 -0
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
+{
+    const char *name = opaque;
+    const char *driver, *netdev;
+
+    driver = qemu_opt_get(opts, "driver");
+    netdev = qemu_opt_get(opts, "netdev");

net/vhost-vdpa.c line 187

Quoted 10 lines
net/vhost-vdpa.c new file +10 -0
+
+    if (!driver || !netdev) {
+        return 0;
+    }
+
+    if (strcmp(netdev, name) == 0 &&
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
+        return -1;
+    }

So perhaps you can build the file only if CONFIG_VIRTIO_NET is set?

Quoted 3 lines

Thanks,
Laurent

Thanks, There will be a new macro specific for vDPA

author
#12 lore raw

RFC 3/4 vhost-vdpa: implement vhost-vdpa backend

In reply to Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

---
 hw/net/vhost_net.c                |  43 ++++
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 include/hw/virtio/vhost-backend.h |   6 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 8 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
hw/net/vhost_net.c +43 -0
index 4096d64aaf..0d13fda2fc 100644
@@ -17,8 +17,10 @@
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
 #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
 #include "hw/virtio/virtio-net.h"
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
@@ -85,6 +87,29 @@ static const int user_feature_bits[] = {
     VHOST_INVALID_FEATURE_BIT
 };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
 static const int *vhost_net_get_feature_bits(struct vhost_net *net)
 {
     const int *feature_bits = 0;
@@ -96,6 +121,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
     case NET_CLIENT_DRIVER_VHOST_USER:
         feature_bits = user_feature_bits;
         break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
     default:
         error_report("Feature bits not defined for this type: %d",
                 net->nc->info->type);
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         assert(vhost_net);
         break;
 #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
     default:
         break;
     }
@@ -465,3 +497,14 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
     return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
 }
+int vhost_set_state(NetClientState *nc, int state)
+{
+    struct vhost_net *net = get_vhost_net(nc);
+    struct vhost_dev *hdev = &net->dev;
+    if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
+        if (hdev->vhost_ops->vhost_set_state) {
+                return hdev->vhost_ops->vhost_set_state(hdev, state);
+             }
+        }
+    return 0;
+}
hw/net/virtio-net.c +9 -0
index db3d7c38e6..bbecd7ab96 100644
@@ -206,6 +206,9 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
     NetClientState *nc = qemu_get_queue(n->nic);
     int queues = n->multiqueue ? n->max_queues : 1;
+    NetClientState *peer = nc->peer;
+    uint8_t status_set  = vdev->status ;
+    uint8_t vhost_started_pre = n->vhost_started;
 
     if (!get_vhost_net(nc->peer)) {
         return;
@@ -245,6 +248,7 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
                 return;
             }
         }
+        status_set = status_set | VIRTIO_CONFIG_S_DRIVER_OK;
 
         n->vhost_started = 1;
         r = vhost_net_start(vdev, n->nic->ncs, queues);
@@ -252,11 +256,16 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
             error_report("unable to start vhost net: %d: "
                          "falling back on userspace virtio", -r);
             n->vhost_started = 0;
+            status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;
         }
     } else {
         vhost_net_stop(vdev, n->nic->ncs, queues);
+        status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;
         n->vhost_started = 0;
     }
+    if (vhost_started_pre != n->vhost_started) {
+            vhost_set_state(peer, status_set);
+    }
 }
 
 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
hw/virtio/Makefile.objs +1 -1
index e2f70fbb89..17361d959e 100644
@@ -2,7 +2,7 @@ ifeq ($(CONFIG_VIRTIO),y)
 common-obj-y += virtio-bus.o
 obj-y += virtio.o
 
-obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o vhost-backend.o
+obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o vhost-backend.o vhost-vdpa.o
 common-obj-$(call lnot,$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL))) += vhost-stub.o
 obj-$(CONFIG_VHOST_USER) += vhost-user.o
 
hw/virtio/vhost-backend.c +3 -0
index 48905383f8..935cd9e561 100644
@@ -286,6 +286,9 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
         dev->vhost_ops = &user_ops;
         break;
 #endif
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
     default:
         error_report("Unknown vhost backend type");
         r = -1;
hw/virtio/vhost-vdpa.c new file +379 -0

379 lines not on this page · raw message

hw/virtio/vhost.c +5 -0
index 4da0d5a6c5..d1f2c4add7 100644
@@ -746,6 +746,11 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }
     int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
         VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
include/hw/virtio/vhost-backend.h +5 -1
index 6f6670783f..d81bd9885f 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
     VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
 } VhostBackendType;
 
 typedef enum VhostSetConfigType {
@@ -112,6 +113,7 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
 typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                         struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, int state);
 typedef struct VhostOps {
     VhostBackendType backend_type;
     vhost_backend_init vhost_backend_init;
@@ -152,9 +154,11 @@ typedef struct VhostOps {
     vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
     vhost_get_inflight_fd_op vhost_get_inflight_fd;
     vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
 } VhostOps;
 
 extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
 int vhost_set_backend_type(struct vhost_dev *dev,
                            VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +14 -0
new file mode 100644
index 0000000000..889c1a4410
@@ -0,0 +1,14 @@
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+
+#endif
Signature · 1 line
2.21.1
#13 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

Quoted 6 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Quoted 8 lines

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

Another questionable authorship line; should this be Signed-off-by? (Do we have permission from Tiwei Bie to include this code?)

hw/virtio/vhost-vdpa.c line 7

Quoted 12 lines

---

+++ b/hw/virtio/vhost-vdpa.c

hw/virtio/vhost-vdpa.c new file +7 -0
@@ -0,0 +1,379 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.

Another questionable "All rights reserved"

include/hw/virtio/vhost-vdpa.h line 4

Quoted 7 lines
include/hw/virtio/vhost-vdpa.h new file +4 -0
@@ -0,0 +1,14 @@
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+

All new files should include a copyright and license, even if they are short.

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
#14 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

In reply to Cindy Lu
Quoted 6 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Quoted 6 lines

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Actually, this part should belongs to patch 2.

And we probably need to add a comment that vIOMMU is not supported right now.

Quoted 102 lines

Author: Tiwei Bie

---
  hw/net/vhost_net.c                |  43 ++++
  hw/net/virtio-net.c               |   9 +
  hw/virtio/Makefile.objs           |   2 +-
  hw/virtio/vhost-backend.c         |   3 +
  hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
  hw/virtio/vhost.c                 |   5 +
  include/hw/virtio/vhost-backend.h |   6 +-
  include/hw/virtio/vhost-vdpa.h    |  14 ++
  8 files changed, 459 insertions(+), 2 deletions(-)

create mode 100644 hw/virtio/vhost-vdpa.c create mode 100644 include/hw/virtio/vhost-vdpa.h

hw/net/vhost_net.c +43 -0
index 4096d64aaf..0d13fda2fc 100644
@@ -17,8 +17,10 @@
  #include "net/net.h"
  #include "net/tap.h"
  #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
  #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
  #include "hw/virtio/virtio-net.h"
  #include "net/vhost_net.h"
  #include "qemu/error-report.h"
@@ -85,6 +87,29 @@ static const int user_feature_bits[] = {
      VHOST_INVALID_FEATURE_BIT
  };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
  static const int *vhost_net_get_feature_bits(struct vhost_net *net)
  {
      const int *feature_bits = 0;
@@ -96,6 +121,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
      case NET_CLIENT_DRIVER_VHOST_USER:
          feature_bits = user_feature_bits;
          break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
      default:
          error_report("Feature bits not defined for this type: %d",
                  net->nc->info->type);
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
          assert(vhost_net);
          break;
  #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
      default:
          break;
      }
@@ -465,3 +497,14 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
      return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
  }
+int vhost_set_state(NetClientState *nc, int state)
+{
+    struct vhost_net *net = get_vhost_net(nc);
+    struct vhost_dev *hdev = &net->dev;
+    if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
+        if (hdev->vhost_ops->vhost_set_state) {
+                return hdev->vhost_ops->vhost_set_state(hdev, state);
+             }
+        }
+    return 0;
+}
hw/net/virtio-net.c +0 -0
index db3d7c38e6..bbecd7ab96 100644

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) · line 209

Quoted 4 lines
hw/net/virtio-net.c +1 -0
@@ -206,6 +206,9 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
      VirtIODevice *vdev = VIRTIO_DEVICE(n);
      NetClientState *nc = qemu_get_queue(n->nic);
      int queues = n->multiqueue ? n->max_queues : 1;
+    NetClientState *peer = nc->peer;

qemu_get_peer()?

Quoted 18 lines
hw/net/virtio-net.c +4 -0
+    uint8_t status_set  = vdev->status ;
+    uint8_t vhost_started_pre = n->vhost_started;
 
      if (!get_vhost_net(nc->peer)) {
          return;
@@ -245,6 +248,7 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
                  return;
              }
          }
+        status_set = status_set | VIRTIO_CONFIG_S_DRIVER_OK;
 
          n->vhost_started = 1;
          r = vhost_net_start(vdev, n->nic->ncs, queues);
@@ -252,11 +256,16 @@ static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
              error_report("unable to start vhost net: %d: "
                           "falling back on userspace virtio", -r);
              n->vhost_started = 0;
+            status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) · line 269

Quoted 10 lines
hw/net/virtio-net.c +4 -0
          }
      } else {
          vhost_net_stop(vdev, n->nic->ncs, queues);
+        status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;
          n->vhost_started = 0;
      }
+    if (vhost_started_pre != n->vhost_started) {
+            vhost_set_state(peer, status_set);
+    }
  }

I think this deserves an independent patch.

Quoted 71 lines
hw/net/virtio-net.c +0 -0
  static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
hw/virtio/Makefile.objs +1 -1
index e2f70fbb89..17361d959e 100644
@@ -2,7 +2,7 @@ ifeq ($(CONFIG_VIRTIO),y)
  common-obj-y += virtio-bus.o
  obj-y += virtio.o
 
-obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o vhost-backend.o
+obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o vhost-backend.o vhost-vdpa.o
  common-obj-$(call lnot,$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL))) += vhost-stub.o
  obj-$(CONFIG_VHOST_USER) += vhost-user.o
 
hw/virtio/vhost-backend.c +3 -0
index 48905383f8..935cd9e561 100644
@@ -286,6 +286,9 @@ int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
          dev->vhost_ops = &user_ops;
          break;
  #endif
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
      default:
          error_report("Unknown vhost backend type");
          r = -1;
hw/virtio/vhost-vdpa.c new file +36 -0
new file mode 100644
index 0000000000..213b327600
@@ -0,0 +1,379 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>
+#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/main-loop.h"
+#include <linux/kvm.h>
+#include "sysemu/kvm.h"
+
+
+static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section)
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings to
+            * addresses in the upper part of the 64-bit address space.  These
+            * are never accessed by the CPU and beyond the address width of
+            * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU width.
+            */
+           section->offset_within_address_space & (1ULL << 63);

hw/virtio/vhost-vdpa.c line 46

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+}
+
+static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr size,
+                              void *vaddr, bool readonly)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type = VHOST_IOTLB_MSG_V2;

Since V2 of the message is used here, I believe we need a kernel patch to allow the querying of backend capability.

Quoted 134 lines
hw/virtio/vhost-vdpa.c new file +134 -0
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.uaddr = (uint64_t)vaddr;
+    msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
+    msg.iotlb.type = VHOST_IOTLB_UPDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
+                                hwaddr size)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type = VHOST_IOTLB_MSG_V2;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static void vhost_vdpa_listener_region_add(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
+                             vaddr, section->readonly);
+    if (ret) {
+        error_report("vhost vdpa map fail!");
+        if (memory_region_is_ram_device(section->mr)) {
+            /* Allow unexpected mappings not to be fatal for RAM devices */
+            error_report("map ram fail!");
+          return ;
+        }
+        goto fail;
+    }
+
+    return;
+
+fail:
+    if (memory_region_is_ram_device(section->mr)) {
+        error_report("failed to vdpa_dma_map. pci p2p may not work");
+        return;
+
+    }
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    error_report("vhost-vdpa: DMA mapping failed, unable to continue");
+    return;
+
+}
+
+static void vhost_vdpa_listener_region_del(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa, listener);
+    hwaddr iova;
+    Int128 llend, llsize;
+    int ret;
+    bool try_unmap = true;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    llsize = int128_sub(llend, int128_make64(iova));
+

hw/virtio/vhost-vdpa.c line 190

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    if (try_unmap) {
+        ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
+        if (ret) {
+            error_report("vhost_vdpa dma unmap error!");
+        }
+    }
+
+    memory_region_unref(section->mr);
+}
+

I think it's better to add comment to explain why vhost-vdpa use a different listener other than the one used by other vhost backends (e.g kernel or user).

Quoted 36 lines
hw/virtio/vhost-vdpa.c new file +36 -0
+static const MemoryListener vhost_vdpa_memory_listener = {
+    .region_add = vhost_vdpa_listener_region_add,
+    .region_del = vhost_vdpa_listener_region_del,
+};
+
+
+static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int request,
+                             void *arg)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    int fd = v->device_fd;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    return ioctl(fd, request, arg);
+}
+
+
+
+static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
+{
+    struct vhost_vdpa *v;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = opaque;
+    dev->opaque =  opaque ;
+
+    v->listener = vhost_vdpa_memory_listener;
+    memory_listener_register(&v->listener, &address_space_memory);
+
+    return 0;
+}
+
+static int vhost_vdpa_cleanup(struct vhost_dev *dev)
+{

hw/virtio/vhost-vdpa.c line 236

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    struct vhost_vdpa *v;
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = dev->opaque;
+    memory_listener_unregister(&v->listener);
+
+    dev->opaque = NULL;
+    return 0;
+}
+

A comment here is need to explain why INT_MAX is used.

hw/virtio/vhost-vdpa.c line 246

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
+{
+    return INT_MAX;
+}
+
+static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                   struct vhost_log *log)
+{
+    return 0;
+}

I think we should fail this function since we don't support dirty page tracking now.

And it's not guarantee to use dirty page bitmap in the future.

hw/virtio/vhost-vdpa.c line 256

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+
+static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+
+    if (mem->padding) {
+        return -1;
+    }
+
+    return 0;

A comment is need to explain why mem table is not used. (E.g we used IOTLB API instead).

Quoted 34 lines
hw/virtio/vhost-vdpa.c new file +34 -0
+}
+
+static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
+                                     struct vhost_vring_addr *addr)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
+}
+
+static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
+                                    struct vhost_vring_state *ring)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
+}
+
+static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+
+    return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
+}
+
+static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,

hw/virtio/vhost-vdpa.c line 300

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+                                     struct vhost_vring_file *file)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
+static int vhost_vdpa_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+
+    features |= (1ULL << VIRTIO_F_IOMMU_PLATFORM);

This seems tricky, I don't think we need this actually.

Quoted 30 lines
hw/virtio/vhost-vdpa.c new file +30 -0
+    return vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
+
+}
+
+static int vhost_vdpa_get_features(struct vhost_dev *dev,
+                                   uint64_t *features)
+{
+    return vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
+}
+
+static int vhost_vdpa_set_owner(struct vhost_dev *dev)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
+}
+
+static int vhost_vdpa_reset_device(struct vhost_dev *dev)
+{
+    return vhost_vdpa_call(dev, VHOST_RESET_OWNER, NULL);
+}
+
+static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
+static int vhost_vdpa_set_vring_enable(struct vhost_dev *dev, int enable)
+{
+    int i;

hw/virtio/vhost-vdpa.c line 340

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num   = enable,
+        };
+
+        state.num = 1;
+
+        vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);

Please make sure patch 4 comes first then we don't need to fix this in patch 4.

Quoted 44 lines
hw/virtio/vhost-vdpa.c new file +39 -0
+    }
+
+    return 0;
+}
+
+static int vhost_vdpa_set_state(struct vhost_dev *dev, int state)
+{
+    return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &state);
+}
+
+
+const VhostOps vdpa_ops = {
+        .backend_type = VHOST_BACKEND_TYPE_VDPA,
+        .vhost_backend_init = vhost_vdpa_init,
+        .vhost_backend_cleanup = vhost_vdpa_cleanup,
+        .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
+        .vhost_set_log_base = vhost_vdpa_set_log_base,
+        .vhost_set_mem_table = vhost_vdpa_set_mem_table,
+        .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
+        .vhost_set_vring_endian = NULL,
+        .vhost_set_vring_num = vhost_vdpa_set_vring_num,
+        .vhost_set_vring_base = vhost_vdpa_set_vring_base,
+        .vhost_get_vring_base = vhost_vdpa_get_vring_base,
+        .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
+        .vhost_set_vring_call = vhost_vdpa_set_vring_call,
+        .vhost_set_features = vhost_vdpa_set_features,
+        .vhost_get_features = vhost_vdpa_get_features,
+        .vhost_set_owner = vhost_vdpa_set_owner,
+        .vhost_reset_device = vhost_vdpa_reset_device,
+        .vhost_get_vq_index = vhost_vdpa_get_vq_index,
+        .vhost_set_vring_enable = vhost_vdpa_set_vring_enable,
+        .vhost_requires_shm_log = NULL,
+        .vhost_migration_done = NULL,
+        .vhost_backend_can_merge = NULL,
+        .vhost_net_set_mtu = NULL,
+        .vhost_set_iotlb_callback = NULL,
+        .vhost_send_device_iotlb_msg = NULL,
+        .vhost_set_state = vhost_vdpa_set_state,
+};
hw/virtio/vhost.c +0 -0
index 4da0d5a6c5..d1f2c4add7 100644

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 753

Quoted 8 lines
hw/virtio/vhost.c +5 -0
@@ -746,6 +746,11 @@ static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
          .log_guest_addr = vq->used_phys,
          .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
      };
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }

Comment is needed to explain why vDPA differs from others.

Thanks

Quoted 57 lines
hw/virtio/vhost.c +0 -0
      int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
      if (r < 0) {
          VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
include/hw/virtio/vhost-backend.h +5 -1
index 6f6670783f..d81bd9885f 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
      VHOST_BACKEND_TYPE_NONE = 0,
      VHOST_BACKEND_TYPE_KERNEL = 1,
      VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
  } VhostBackendType;
 
  typedef enum VhostSetConfigType {
@@ -112,6 +113,7 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
  typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                          struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, int state);
  typedef struct VhostOps {
      VhostBackendType backend_type;
      vhost_backend_init vhost_backend_init;
@@ -152,9 +154,11 @@ typedef struct VhostOps {
      vhost_backend_mem_section_filter_op vhost_backend_mem_section_filter;
      vhost_get_inflight_fd_op vhost_get_inflight_fd;
      vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
  } VhostOps;
 
  extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
  int vhost_set_backend_type(struct vhost_dev *dev,
                             VhostBackendType backend_type);
include/hw/virtio/vhost-vdpa.h new file +14 -0
new file mode 100644
index 0000000000..889c1a4410
@@ -0,0 +1,14 @@
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+
+#endif
author
#15 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

Quoted 754 lines · Jason Wang
Quoted 3 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific

configuration

Quoted 1 line

interface for setting up a vhost HW accelerator, this patch set

introduces

Quoted 8 lines

a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

   qemu-system-x86_64 -cpu host -enable-kvm \
     ......
   -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
   -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Actually, this part should belongs to patch 2.

And we probably need to add a comment that vIOMMU is not supported right now.

Will fix this problem

hw/net/vhost_net.c static const int *vhost_net_get_feature_bits(struct

Quoted 62 lines

Author: Tiwei Bie

---
  hw/net/vhost_net.c                |  43 ++++
  hw/net/virtio-net.c               |   9 +
  hw/virtio/Makefile.objs           |   2 +-
  hw/virtio/vhost-backend.c         |   3 +
  hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
  hw/virtio/vhost.c                 |   5 +
  include/hw/virtio/vhost-backend.h |   6 +-
  include/hw/virtio/vhost-vdpa.h    |  14 ++
  8 files changed, 459 insertions(+), 2 deletions(-)

create mode 100644 hw/virtio/vhost-vdpa.c create mode 100644 include/hw/virtio/vhost-vdpa.h

hw/net/vhost_net.c +25 -0
index 4096d64aaf..0d13fda2fc 100644
@@ -17,8 +17,10 @@
  #include "net/net.h"
  #include "net/tap.h"
  #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
  #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
  #include "hw/virtio/virtio-net.h"
  #include "net/vhost_net.h"
  #include "qemu/error-report.h"
@@ -85,6 +87,29 @@ static const int user_feature_bits[] = {
      VHOST_INVALID_FEATURE_BIT
  };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
  static const int *vhost_net_get_feature_bits(struct vhost_net *net)
  {
      const int *feature_bits = 0;
@@ -96,6 +121,9 @@ static const int *vhost_net_get_feature_bits(struct

vhost_net *net)

hw/net/vhost_net.c int vhost_net_set_mtu(struct vhost_net *net,

Quoted 21 lines
hw/net/vhost_net.c +7 -0
      case NET_CLIENT_DRIVER_VHOST_USER:
          feature_bits = user_feature_bits;
          break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
      default:
          error_report("Feature bits not defined for this type: %d",
                  net->nc->info->type);
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
          assert(vhost_net);
          break;
  #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
      default:
          break;
      }
@@ -465,3 +497,14 @@ int vhost_net_set_mtu(struct vhost_net *net,

uint16_t mtu)

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n,

Quoted 19 lines
hw/net/vhost_net.c +11 -0
      return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
  }
+int vhost_set_state(NetClientState *nc, int state)
+{
+    struct vhost_net *net = get_vhost_net(nc);
+    struct vhost_dev *hdev = &net->dev;
+    if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
+        if (hdev->vhost_ops->vhost_set_state) {
+                return hdev->vhost_ops->vhost_set_state(hdev, state);
+             }
+        }
+    return 0;
+}
hw/net/virtio-net.c +0 -0
index db3d7c38e6..bbecd7ab96 100644
@@ -206,6 +206,9 @@ static void virtio_net_vhost_status(VirtIONet *n,

uint8_t status)

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) · line 209

Quoted 4 lines
hw/net/virtio-net.c +1 -0
      VirtIODevice *vdev = VIRTIO_DEVICE(n);
      NetClientState *nc = qemu_get_queue(n->nic);
      int queues = n->multiqueue ? n->max_queues : 1;
+    NetClientState *peer = nc->peer;

qemu_get_peer()?

will fix it

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n,

Quoted 6 lines
hw/net/virtio-net.c +2 -0
+    uint8_t status_set  = vdev->status ;
+    uint8_t vhost_started_pre = n->vhost_started;
 
      if (!get_vhost_net(nc->peer)) {
          return;
@@ -245,6 +248,7 @@ static void virtio_net_vhost_status(VirtIONet *n,

uint8_t status)

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n,

Quoted 8 lines
hw/net/virtio-net.c +1 -0
                  return;
              }
          }
+        status_set = status_set | VIRTIO_CONFIG_S_DRIVER_OK;
 
          n->vhost_started = 1;
          r = vhost_net_start(vdev, n->nic->ncs, queues);
@@ -252,11 +256,16 @@ static void virtio_net_vhost_status(VirtIONet *n,

uint8_t status)

Quoted 4 lines
hw/net/virtio-net.c +1 -0
              error_report("unable to start vhost net: %d: "
                           "falling back on userspace virtio", -r);
              n->vhost_started = 0;
+            status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;

hw/net/virtio-net.c static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) · line 269

Quoted 10 lines
hw/net/virtio-net.c +4 -0
          }
      } else {
          vhost_net_stop(vdev, n->nic->ncs, queues);
+        status_set = status_set & ~VIRTIO_CONFIG_S_DRIVER_OK;
          n->vhost_started = 0;
      }
+    if (vhost_started_pre != n->vhost_started) {
+            vhost_set_state(peer, status_set);
+    }
  }

I think this deserves an independent patch.

will fix it

hw/virtio/Makefile.objs ifeq ($(CONFIG_VIRTIO),y) · line 5

Quoted 11 lines
hw/net/virtio-net.c +0 -0
  static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
hw/virtio/Makefile.objs +0 -1
index e2f70fbb89..17361d959e 100644
@@ -2,7 +2,7 @@ ifeq ($(CONFIG_VIRTIO),y)
  common-obj-y += virtio-bus.o
  obj-y += virtio.o
 
-obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o

vhost-backend.o

hw/virtio/Makefile.objs ifeq ($(CONFIG_VIRTIO),y) · line 5

Quoted 1 line
hw/virtio/Makefile.objs +1 -0
+obj-$(call lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL)) += vhost.o

vhost-backend.o vhost-vdpa.o

hw/virtio/Makefile.objs ifeq ($(CONFIG_VIRTIO),y) · line 6

Quoted 1 line
hw/virtio/Makefile.objs +0 -0
  common-obj-$(call lnot,$(call

lor,$(CONFIG_VHOST_USER),$(CONFIG_VHOST_KERNEL))) += vhost-stub.o

hw/virtio/vhost-backend.c int vhost_set_backend_type(struct vhost_dev *dev,

Quoted 7 lines
hw/virtio/Makefile.objs +0 -0
  obj-$(CONFIG_VHOST_USER) += vhost-user.o
 
hw/virtio/vhost-backend.c +0 -0
index 48905383f8..935cd9e561 100644
@@ -286,6 +286,9 @@ int vhost_set_backend_type(struct vhost_dev *dev,

VhostBackendType backend_type)

Quoted 15 lines
hw/virtio/vhost-backend.c +3 -0
          dev->vhost_ops = &user_ops;
          break;
  #endif
+    case VHOST_BACKEND_TYPE_VDPA:
+        dev->vhost_ops = &vdpa_ops;
+        break;
      default:
          error_report("Unknown vhost backend type");
          r = -1;
hw/virtio/vhost-vdpa.c new file +0 -0
new file mode 100644
index 0000000000..213b327600

hw/virtio/vhost-vdpa.c line 7

Quoted 7 lines
hw/virtio/vhost-vdpa.c new file +7 -0
@@ -0,0 +1,379 @@
+/*
+ * vhost-vdpa
+ *
+ *  Copyright(c) 2017-2018 Intel Corporation. All rights reserved.
+ *  Copyright(c) 2020 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or

later.

Quoted 9 lines
hw/virtio/vhost-vdpa.c new file +9 -0
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <linux/vfio.h>
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+#include "hw/virtio/vhost.h"
+#include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/vhost-vdpa.h"
+#include "qemu/main-loop.h"
+#include <linux/kvm.h>
+#include "sysemu/kvm.h"
+
+
+static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection

*section)

hw/virtio/vhost-vdpa.c

Quoted 5 lines
hw/virtio/vhost-vdpa.c new file +5 -0
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings

to

hw/virtio/vhost-vdpa.c line 32

Quoted 1 line
hw/virtio/vhost-vdpa.c new file +1 -0
+            * addresses in the upper part of the 64-bit address space.

These

hw/virtio/vhost-vdpa.c line 33

Quoted 1 line
hw/virtio/vhost-vdpa.c new file +1 -0
+            * are never accessed by the CPU and beyond the address

width of

hw/virtio/vhost-vdpa.c line 34

Quoted 1 line
hw/virtio/vhost-vdpa.c new file +1 -0
+            * some IOMMU hardware.  TODO: VDPA should tell us the IOMMU

width.

hw/virtio/vhost-vdpa.c

Quoted 5 lines
hw/virtio/vhost-vdpa.c new file +5 -0
+            */
+           section->offset_within_address_space & (1ULL << 63);
+}
+
+static int vhost_vdpa_dma_map(struct vhost_vdpa *v, hwaddr iova, hwaddr

size,

hw/virtio/vhost-vdpa.c line 46

Quoted 7 lines
hw/virtio/vhost-vdpa.c new file +7 -0
+                              void *vaddr, bool readonly)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type = VHOST_IOTLB_MSG_V2;

Since V2 of the message is used here, I believe we need a kernel patch to allow the querying of backend capability.

Sure, I will provide another patch for kernel

Quoted 30 lines
hw/virtio/vhost-vdpa.c new file +30 -0
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.uaddr = (uint64_t)vaddr;
+    msg.iotlb.perm = readonly ? VHOST_ACCESS_RO : VHOST_ACCESS_RW;
+    msg.iotlb.type = VHOST_IOTLB_UPDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static int vhost_vdpa_dma_unmap(struct vhost_vdpa *v, hwaddr iova,
+                                hwaddr size)
+{
+    struct vhost_msg_v2 msg;
+    int fd = v->device_fd;
+    int ret = 0;
+
+    msg.type = VHOST_IOTLB_MSG_V2;
+    msg.iotlb.iova = iova;
+    msg.iotlb.size = size;
+    msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
+
+    if (write(fd, &msg, sizeof(msg)) != sizeof(msg)) {
+        error_report("failed to write, fd=%d, errno=%d (%s)",
+            fd, errno, strerror(errno));

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+        return -EIO ;
+    }
+
+    return ret;
+}
+
+static void vhost_vdpa_listener_region_add(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa,

listener);

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    hwaddr iova;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space &

~TARGET_PAGE_MASK) !=

Quoted 20 lines
hw/virtio/vhost-vdpa.c new file +20 -0
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+            (iova - section->offset_within_address_space);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vhost_vdpa_dma_map(v, iova, int128_get64(llsize),
+                             vaddr, section->readonly);
+    if (ret) {
+        error_report("vhost vdpa map fail!");
+        if (memory_region_is_ram_device(section->mr)) {
+            /* Allow unexpected mappings not to be fatal for RAM

devices */

Quoted 18 lines
hw/virtio/vhost-vdpa.c new file +18 -0
+            error_report("map ram fail!");
+          return ;
+        }
+        goto fail;
+    }
+
+    return;
+
+fail:
+    if (memory_region_is_ram_device(section->mr)) {
+        error_report("failed to vdpa_dma_map. pci p2p may not work");
+        return;
+
+    }
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+     */
+    error_report("vhost-vdpa: DMA mapping failed, unable to continue");
+    return;
+
+}
+
+static void vhost_vdpa_listener_region_del(MemoryListener *listener,
+                                           MemoryRegionSection *section)
+{
+    struct vhost_vdpa *v = container_of(listener, struct vhost_vdpa,

listener);

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    hwaddr iova;
+    Int128 llend, llsize;
+    int ret;
+    bool try_unmap = true;
+
+    if (vhost_vdpa_listener_skipped_section(section)) {
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space &

~TARGET_PAGE_MASK) !=

Quoted 16 lines
hw/virtio/vhost-vdpa.c new file +16 -0
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+
+    llsize = int128_sub(llend, int128_make64(iova));
+

hw/virtio/vhost-vdpa.c line 190

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    if (try_unmap) {
+        ret = vhost_vdpa_dma_unmap(v, iova, int128_get64(llsize));
+        if (ret) {
+            error_report("vhost_vdpa dma unmap error!");
+        }
+    }
+
+    memory_region_unref(section->mr);
+}
+

I think it's better to add comment to explain why vhost-vdpa use a different listener other than the one used by other vhost backends (e.g kernel or user).

will fix it

hw/virtio/vhost-vdpa.c

Quoted 7 lines
hw/virtio/vhost-vdpa.c new file +7 -0
+static const MemoryListener vhost_vdpa_memory_listener = {
+    .region_add = vhost_vdpa_listener_region_add,
+    .region_del = vhost_vdpa_listener_region_del,
+};
+
+
+static int vhost_vdpa_call(struct vhost_dev *dev, unsigned long int

request,

Quoted 29 lines
hw/virtio/vhost-vdpa.c new file +29 -0
+                             void *arg)
+{
+    struct vhost_vdpa *v = dev->opaque;
+    int fd = v->device_fd;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    return ioctl(fd, request, arg);
+}
+
+
+
+static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque)
+{
+    struct vhost_vdpa *v;
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = opaque;
+    dev->opaque =  opaque ;
+
+    v->listener = vhost_vdpa_memory_listener;
+    memory_listener_register(&v->listener, &address_space_memory);
+
+    return 0;
+}
+
+static int vhost_vdpa_cleanup(struct vhost_dev *dev)
+{

hw/virtio/vhost-vdpa.c line 236

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+    struct vhost_vdpa *v;
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA);
+
+    v = dev->opaque;
+    memory_listener_unregister(&v->listener);
+
+    dev->opaque = NULL;
+    return 0;
+}
+

A comment here is need to explain why INT_MAX is used.

will do

hw/virtio/vhost-vdpa.c line 246

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+static int vhost_vdpa_memslots_limit(struct vhost_dev *dev)
+{
+    return INT_MAX;
+}
+
+static int vhost_vdpa_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                   struct vhost_log *log)
+{
+    return 0;
+}

I think we should fail this function since we don't support dirty page tracking now.

And it's not guarantee to use dirty page bitmap in the future.

hw/virtio/vhost-vdpa.c line 256

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+
+static int vhost_vdpa_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+
+    if (mem->padding) {
+        return -1;
+    }
+
+    return 0;

A comment is need to explain why mem table is not used. (E.g we used IOTLB API instead).

will do

Quoted 34 lines
hw/virtio/vhost-vdpa.c new file +34 -0
+}
+
+static int vhost_vdpa_set_vring_addr(struct vhost_dev *dev,
+                                     struct vhost_vring_addr *addr)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_ADDR, addr);
+}
+
+static int vhost_vdpa_set_vring_num(struct vhost_dev *dev,
+                                    struct vhost_vring_state *ring)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_NUM, ring);
+}
+
+static int vhost_vdpa_set_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+
+    return vhost_vdpa_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_vdpa_set_vring_kick(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_KICK, file);
+}
+
+static int vhost_vdpa_set_vring_call(struct vhost_dev *dev,

hw/virtio/vhost-vdpa.c line 300

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+                                     struct vhost_vring_file *file)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
+static int vhost_vdpa_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+
+    features |= (1ULL << VIRTIO_F_IOMMU_PLATFORM);

This seems tricky, I don't think we need this actually.

I will double check for this problem

Quoted 18 lines
hw/virtio/vhost-vdpa.c new file +18 -0
+    return vhost_vdpa_call(dev, VHOST_SET_FEATURES, &features);
+
+}
+
+static int vhost_vdpa_get_features(struct vhost_dev *dev,
+                                   uint64_t *features)
+{
+    return vhost_vdpa_call(dev, VHOST_GET_FEATURES, features);
+}
+
+static int vhost_vdpa_set_owner(struct vhost_dev *dev)
+{
+    return vhost_vdpa_call(dev, VHOST_SET_OWNER, NULL);
+}
+
+static int vhost_vdpa_reset_device(struct vhost_dev *dev)
+{
+    return vhost_vdpa_call(dev, VHOST_RESET_OWNER, NULL);

hw/virtio/vhost-vdpa.c

Quoted 10 lines
hw/virtio/vhost-vdpa.c new file +10 -0
+}
+
+static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
+static int vhost_vdpa_set_vring_enable(struct vhost_dev *dev, int

enable)

hw/virtio/vhost-vdpa.c line 340

Quoted 12 lines
hw/virtio/vhost-vdpa.c new file +12 -0
+{
+    int i;
+
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num   = enable,
+        };
+
+        state.num = 1;
+
+        vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);

Please make sure patch 4 comes first then we don't need to fix this in patch 4.

will do

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct

Quoted 44 lines
hw/virtio/vhost-vdpa.c new file +39 -0
+    }
+
+    return 0;
+}
+
+static int vhost_vdpa_set_state(struct vhost_dev *dev, int state)
+{
+    return vhost_vdpa_call(dev, VHOST_VDPA_SET_STATUS, &state);
+}
+
+
+const VhostOps vdpa_ops = {
+        .backend_type = VHOST_BACKEND_TYPE_VDPA,
+        .vhost_backend_init = vhost_vdpa_init,
+        .vhost_backend_cleanup = vhost_vdpa_cleanup,
+        .vhost_backend_memslots_limit = vhost_vdpa_memslots_limit,
+        .vhost_set_log_base = vhost_vdpa_set_log_base,
+        .vhost_set_mem_table = vhost_vdpa_set_mem_table,
+        .vhost_set_vring_addr = vhost_vdpa_set_vring_addr,
+        .vhost_set_vring_endian = NULL,
+        .vhost_set_vring_num = vhost_vdpa_set_vring_num,
+        .vhost_set_vring_base = vhost_vdpa_set_vring_base,
+        .vhost_get_vring_base = vhost_vdpa_get_vring_base,
+        .vhost_set_vring_kick = vhost_vdpa_set_vring_kick,
+        .vhost_set_vring_call = vhost_vdpa_set_vring_call,
+        .vhost_set_features = vhost_vdpa_set_features,
+        .vhost_get_features = vhost_vdpa_get_features,
+        .vhost_set_owner = vhost_vdpa_set_owner,
+        .vhost_reset_device = vhost_vdpa_reset_device,
+        .vhost_get_vq_index = vhost_vdpa_get_vq_index,
+        .vhost_set_vring_enable = vhost_vdpa_set_vring_enable,
+        .vhost_requires_shm_log = NULL,
+        .vhost_migration_done = NULL,
+        .vhost_backend_can_merge = NULL,
+        .vhost_net_set_mtu = NULL,
+        .vhost_set_iotlb_callback = NULL,
+        .vhost_send_device_iotlb_msg = NULL,
+        .vhost_set_state = vhost_vdpa_set_state,
+};
hw/virtio/vhost.c +0 -0
index 4da0d5a6c5..d1f2c4add7 100644
@@ -746,6 +746,11 @@ static int vhost_virtqueue_set_addr(struct

vhost_dev *dev,

hw/virtio/vhost.c static int vhost_virtqueue_set_addr(struct vhost_dev *dev, · line 753

Quoted 8 lines
hw/virtio/vhost.c +5 -0
          .log_guest_addr = vq->used_phys,
          .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
      };
+    if (dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA) {
+        addr.desc_user_addr = (uint64_t)(unsigned long)vq->desc_phys;
+        addr.avail_user_addr = (uint64_t)(unsigned long)vq->avail_phys;
+        addr.used_user_addr = (uint64_t)(unsigned long)vq->used_phys;
+    }

Comment is needed to explain why vDPA differs from others.

Thanks

will do

Review

Quoted 4 lines
hw/virtio/vhost.c +0 -0
      int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
      if (r < 0) {
          VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
(quoted diff) +0 -0

b/include/hw/virtio/vhost-backend.h

include/hw/virtio/vhost-backend.h typedef int (*vhost_get_inflight_fd_op)(struct

Quoted 14 lines

index 6f6670783f..d81bd9885f 100644

include/hw/virtio/vhost-backend.h +2 -1
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
      VHOST_BACKEND_TYPE_NONE = 0,
      VHOST_BACKEND_TYPE_KERNEL = 1,
      VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
  } VhostBackendType;
 
  typedef enum VhostSetConfigType {
@@ -112,6 +113,7 @@ typedef int (*vhost_get_inflight_fd_op)(struct

vhost_dev *dev,

include/hw/virtio/vhost-backend.h typedef int (*vhost_get_inflight_fd_op)(struct · line 114

Quoted 2 lines
include/hw/virtio/vhost-backend.h +0 -0
  typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                          struct vhost_inflight

*inflight);

include/hw/virtio/vhost-backend.h typedef struct VhostOps { · line 154

Quoted 7 lines
include/hw/virtio/vhost-backend.h +1 -0
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, int state);
  typedef struct VhostOps {
      VhostBackendType backend_type;
      vhost_backend_init vhost_backend_init;
@@ -152,9 +154,11 @@ typedef struct VhostOps {
      vhost_backend_mem_section_filter_op

vhost_backend_mem_section_filter;

Review

Quoted 11 lines
include/hw/virtio/vhost-backend.h +2 -0
      vhost_get_inflight_fd_op vhost_get_inflight_fd;
      vhost_set_inflight_fd_op vhost_set_inflight_fd;
+    vhost_set_state_op vhost_set_state;
  } VhostOps;
 
  extern const VhostOps user_ops;
+extern const VhostOps vdpa_ops;
 
  int vhost_set_backend_type(struct vhost_dev *dev,
                             VhostBackendType backend_type);
(quoted diff) +0 -0

b/include/hw/virtio/vhost-vdpa.h

Quoted 19 lines

new file mode 100644
index 0000000000..889c1a4410

include/hw/virtio/vhost-vdpa.h new file +14 -0
@@ -0,0 +1,14 @@
+
+#ifndef HW_VIRTIO_VHOST_VDPA_H
+#define HW_VIRTIO_VHOST_VDPA_H
+
+#include "hw/virtio/virtio.h"
+
+typedef struct vhost_vdpa {
+    int device_fd;
+    MemoryListener listener;
+} VhostVDPA;
+
+extern AddressSpace address_space_memory;
+
+#endif
#16 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

In reply to Cindy Lu
Quoted 6 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Quoted 8 lines

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

Use "git commit --author" to set that.

Quoted 18 lines
---
 hw/net/vhost_net.c                |  43 ++++
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 include/hw/virtio/vhost-backend.h |   6 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 8 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
hw/net/vhost_net.c +0 -0
index 4096d64aaf..0d13fda2fc 100644

...

hw/net/vhost_net.c VHostNetState *get_vhost_net(NetClientState *nc) · line 468

Quoted 8 lines
hw/net/vhost_net.c +4 -0
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         assert(vhost_net);
         break;
 #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;

This should be inside a "#ifdef".

Thanks,
Laurent

author
#17 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

Quoted 10 lines · Laurent Vivier
Quoted 8 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

Quoted 8 lines
Quoted 6 lines
  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

Use "git commit --author" to set that.

Thanks, I will fix this

Quoted 19 lines
Quoted 18 lines
---
 hw/net/vhost_net.c                |  43 ++++
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 include/hw/virtio/vhost-backend.h |   6 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 8 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
hw/net/vhost_net.c +0 -0
index 4096d64aaf..0d13fda2fc 100644

...

Quoted 11 lines

hw/net/vhost_net.c VHostNetState *get_vhost_net(NetClientState *nc) · line 468

Quoted 8 lines
hw/net/vhost_net.c +4 -0
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         assert(vhost_net);
         break;
 #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;

This should be inside a "#ifdef".

Thanks Laurent, I will add a new macro for vDPA

Quoted 3 lines

Thanks,
Laurent

#18 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

In reply to Cindy Lu
Quoted 95 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

---
 hw/net/vhost_net.c                |  43 ++++
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 include/hw/virtio/vhost-backend.h |   6 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 8 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
hw/net/vhost_net.c +32 -0
index 4096d64aaf..0d13fda2fc 100644
@@ -17,8 +17,10 @@
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
 #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
 #include "hw/virtio/virtio-net.h"
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
@@ -85,6 +87,29 @@ static const int user_feature_bits[] = {
     VHOST_INVALID_FEATURE_BIT
 };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
 static const int *vhost_net_get_feature_bits(struct vhost_net *net)
 {
     const int *feature_bits = 0;
@@ -96,6 +121,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
     case NET_CLIENT_DRIVER_VHOST_USER:
         feature_bits = user_feature_bits;
         break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
     default:
         error_report("Feature bits not defined for this type: %d",
                 net->nc->info->type);
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         assert(vhost_net);
         break;
 #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
     default:
         break;
     }

hw/net/vhost_net.c int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) · line 504

Quoted 8 lines
hw/net/vhost_net.c +5 -0
@@ -465,3 +497,14 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
     return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
 }
+int vhost_set_state(NetClientState *nc, int state)
+{
+    struct vhost_net *net = get_vhost_net(nc);
+    struct vhost_dev *hdev = &net->dev;
+    if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {

Maybe checking the vhost_set_state callback is implemented is enough, and it is not need to restrict that to Vhost-vDPA?

Quoted 6 lines
hw/net/vhost_net.c +6 -0
+        if (hdev->vhost_ops->vhost_set_state) {
+                return hdev->vhost_ops->vhost_set_state(hdev, state);
+             }
+        }
+    return 0;
+}
author
#19 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

Quoted 100 lines · Maxime Coquelin
Quoted 96 lines · Cindy Lu

Currently we have 2 types of vhost backends in QEMU: vhost kernel and vhost-user. The above patch provides a generic device for vDPA purpose, this vDPA device exposes to user space a non-vendor-specific configuration interface for setting up a vhost HW accelerator, this patch set introduces a third vhost backend called vhost-vdpa based on the vDPA interface.

Vhost-vdpa usage:

  qemu-system-x86_64 -cpu host -enable-kvm \
    ......
  -netdev type=vhost-vdpa,vhostdev=/dev/vhost-vdpa-id,id=vhost-vdpa0 \
  -device virtio-net-pci,netdev=vhost-vdpa0,page-per-vq=on \

Author: Tiwei Bie

---
 hw/net/vhost_net.c                |  43 ++++
 hw/net/virtio-net.c               |   9 +
 hw/virtio/Makefile.objs           |   2 +-
 hw/virtio/vhost-backend.c         |   3 +
 hw/virtio/vhost-vdpa.c            | 379 ++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |   5 +
 include/hw/virtio/vhost-backend.h |   6 +-
 include/hw/virtio/vhost-vdpa.h    |  14 ++
 8 files changed, 459 insertions(+), 2 deletions(-)
 create mode 100644 hw/virtio/vhost-vdpa.c
 create mode 100644 include/hw/virtio/vhost-vdpa.h
hw/net/vhost_net.c +32 -0
index 4096d64aaf..0d13fda2fc 100644
@@ -17,8 +17,10 @@
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
+#include "net/vhost-vdpa.h"
 
 #include "standard-headers/linux/vhost_types.h"
+#include "linux-headers/linux/vhost.h"
 #include "hw/virtio/virtio-net.h"
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
@@ -85,6 +87,29 @@ static const int user_feature_bits[] = {
     VHOST_INVALID_FEATURE_BIT
 };
 
+static const int vdpa_feature_bits[] = {
+    VIRTIO_F_NOTIFY_ON_EMPTY,
+    VIRTIO_RING_F_INDIRECT_DESC,
+    VIRTIO_RING_F_EVENT_IDX,
+    VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
+    VIRTIO_NET_F_CSUM,
+    VIRTIO_NET_F_GUEST_CSUM,
+    VIRTIO_NET_F_GSO,
+    VIRTIO_NET_F_GUEST_TSO4,
+    VIRTIO_NET_F_GUEST_TSO6,
+    VIRTIO_NET_F_GUEST_ECN,
+    VIRTIO_NET_F_GUEST_UFO,
+    VIRTIO_NET_F_HOST_TSO4,
+    VIRTIO_NET_F_HOST_TSO6,
+    VIRTIO_NET_F_HOST_ECN,
+    VIRTIO_NET_F_HOST_UFO,
+    VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_NET_F_MTU,
+    VIRTIO_F_IOMMU_PLATFORM,
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
+    VHOST_INVALID_FEATURE_BIT
+};
 static const int *vhost_net_get_feature_bits(struct vhost_net *net)
 {
     const int *feature_bits = 0;
@@ -96,6 +121,9 @@ static const int *vhost_net_get_feature_bits(struct vhost_net *net)
     case NET_CLIENT_DRIVER_VHOST_USER:
         feature_bits = user_feature_bits;
         break;
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        feature_bits = vdpa_feature_bits;
+        break;
     default:
         error_report("Feature bits not defined for this type: %d",
                 net->nc->info->type);
@@ -434,6 +462,10 @@ VHostNetState *get_vhost_net(NetClientState *nc)
         assert(vhost_net);
         break;
 #endif
+    case NET_CLIENT_DRIVER_VHOST_VDPA:
+        vhost_net = vhost_vdpa_get_vhost_net(nc);
+        assert(vhost_net);
+        break;
     default:
         break;
     }
@@ -465,3 +497,14 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 
Quoted 10 lines

hw/net/vhost_net.c int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) · line 504

Quoted 7 lines
hw/net/vhost_net.c +5 -0
     return vhost_ops->vhost_net_set_mtu(&net->dev, mtu);
 }
+int vhost_set_state(NetClientState *nc, int state)
+{
+    struct vhost_net *net = get_vhost_net(nc);
+    struct vhost_dev *hdev = &net->dev;
+    if (nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {

Maybe checking the vhost_set_state callback is implemented is enough, and it is not need to restrict that to Vhost-vDPA?

Sure, Will remove this

Quoted 7 lines
Quoted 6 lines
hw/net/vhost_net.c +6 -0
+        if (hdev->vhost_ops->vhost_set_state) {
+                return hdev->vhost_ops->vhost_set_state(hdev, state);
+             }
+        }
+    return 0;
+}
#20 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

In reply to Cindy Lu
Quoted 15 lines · Cindy Lu
include/hw/virtio/vhost-backend.h +2 -1
index 6f6670783f..d81bd9885f 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
     VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
 } VhostBackendType;
 
 typedef enum VhostSetConfigType {

include/hw/virtio/vhost-backend.h typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev, · line 116

Quoted 4 lines
include/hw/virtio/vhost-backend.h +1 -0
@@ -112,6 +113,7 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
 typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                         struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, int state);

I think state should be of type uint8_t.

author
#21 lore raw

Re: [RFC v1 3/4] vhost-vdpa: implement vhost-vdpa backend

Quoted 17 lines · Maxime Coquelin
Quoted 13 lines · Cindy Lu
include/hw/virtio/vhost-backend.h +2 -1
index 6f6670783f..d81bd9885f 100644
@@ -17,7 +17,8 @@ typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
     VHOST_BACKEND_TYPE_KERNEL = 1,
     VHOST_BACKEND_TYPE_USER = 2,
-    VHOST_BACKEND_TYPE_MAX = 3,
+    VHOST_BACKEND_TYPE_VDPA = 3,
+    VHOST_BACKEND_TYPE_MAX = 4,
 } VhostBackendType;
 
Quoted 9 lines

include/hw/virtio/vhost-backend.h typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev, · line 116

Quoted 6 lines
include/hw/virtio/vhost-backend.h +1 -0
 typedef enum VhostSetConfigType {
@@ -112,6 +113,7 @@ typedef int (*vhost_get_inflight_fd_op)(struct vhost_dev *dev,
 typedef int (*vhost_set_inflight_fd_op)(struct vhost_dev *dev,
                                         struct vhost_inflight *inflight);
 
+typedef int (*vhost_set_state_op)(struct vhost_dev *dev, int state);

I think state should be of type uint8_t.

ok, I will change this to uint8_t

author
#22 lore raw

RFC 4/4 vhost: introduce vhost_set_vring_ready method

In reply to Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

Author: Jason Wang <[email protected]>

---
 hw/net/vhost_net-stub.c           |  5 +++++
 hw/net/vhost_net.c                | 16 ++++++++++++++++
 hw/virtio/vhost-vdpa.c            |  9 +++------
 hw/virtio/virtio-pci.c            | 13 +++++++++++++
 hw/virtio/virtio.c                |  6 ++++++
 include/hw/virtio/vhost-backend.h |  2 ++
 include/hw/virtio/virtio-bus.h    |  4 ++++
 include/net/vhost_net.h           |  1 +
 8 files changed, 50 insertions(+), 6 deletions(-)
hw/net/vhost_net-stub.c +5 -0
index aac0e98228..f5ef1e3055 100644
@@ -86,6 +86,11 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}
+
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     return 0;
hw/net/vhost_net.c +16 -0
index 0d13fda2fc..463e333531 100644
@@ -380,6 +380,10 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
                 goto err_start;
             }
         }
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }
     }
 
     return 0;
@@ -487,6 +491,18 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
     return 0;
 }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+
+    return 0;
+}
+
 int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
 {
     const VhostOps *vhost_ops = net->dev.vhost_ops;
hw/virtio/vhost-vdpa.c +3 -6
index 213b327600..49224ef9f8 100644
@@ -325,18 +325,15 @@ static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
     return idx - dev->vq_index;
 }
 
-static int vhost_vdpa_set_vring_enable(struct vhost_dev *dev, int enable)
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
 {
     int i;
 
     for (i = 0; i < dev->nvqs; ++i) {
         struct vhost_vring_state state = {
             .index = dev->vq_index + i,
-            .num   = enable,
+            .num = 1,
         };
-
-        state.num = 1;
-
         vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
     }
 
@@ -368,7 +365,7 @@ const VhostOps vdpa_ops = {
         .vhost_set_owner = vhost_vdpa_set_owner,
         .vhost_reset_device = vhost_vdpa_reset_device,
         .vhost_get_vq_index = vhost_vdpa_get_vq_index,
-        .vhost_set_vring_enable = vhost_vdpa_set_vring_enable,
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
         .vhost_requires_shm_log = NULL,
         .vhost_migration_done = NULL,
         .vhost_backend_can_merge = NULL,
hw/virtio/virtio-pci.c +13 -0
index c6b47a9c73..4aaf5d953e 100644
@@ -1103,6 +1103,18 @@ static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
     return pci_get_address_space(dev);
 }
 
+static bool virtio_pci_queue_enabled(DeviceState *d, int n)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return proxy->vqs[vdev->queue_sel].enabled;
+    }
+
+    return virtio_queue_get_desc_addr(vdev, n) != 0;
+}
+
 static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
                                    struct virtio_pci_cap *cap)
 {
@@ -2053,6 +2065,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
     k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
     k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
     k->get_dma_as = virtio_pci_get_dma_as;
+    k->queue_enabled = virtio_pci_queue_enabled;
 }
 
 static const TypeInfo virtio_pci_bus_info = {
hw/virtio/virtio.c +6 -0
index 04716b5f6c..09732a8836 100644
@@ -3169,6 +3169,12 @@ hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 
 bool virtio_queue_enabled(VirtIODevice *vdev, int n)
 {
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (k->queue_enabled)
+        return k->queue_enabled(qbus->parent, n);
+
     return virtio_queue_get_desc_addr(vdev, n) != 0;
 }
 
include/hw/virtio/vhost-backend.h +2 -0
index d81bd9885f..ce8de6d308 100644
@@ -78,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
 typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
 typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                          int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
 typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
 typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                        char *mac_addr);
@@ -140,6 +141,7 @@ typedef struct VhostOps {
     vhost_reset_device_op vhost_reset_device;
     vhost_get_vq_index_op vhost_get_vq_index;
     vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
     vhost_requires_shm_log_op vhost_requires_shm_log;
     vhost_migration_done_op vhost_migration_done;
     vhost_backend_can_merge_op vhost_backend_can_merge;
include/hw/virtio/virtio-bus.h +4 -0
index 38c9399cd4..0f6f215925 100644
@@ -83,6 +83,10 @@ typedef struct VirtioBusClass {
      */
     int (*ioeventfd_assign)(DeviceState *d, EventNotifier *notifier,
                             int n, bool assign);
+    /*
+     * Whether queue number n is enabled.
+     */
+    bool (*queue_enabled)(DeviceState *d, int n);
     /*
      * Does the transport have variable vring alignment?
      * (ie can it ever call virtio_queue_set_align()?)
include/net/vhost_net.h +1 -0
index 6f3a624cf7..db473ff4d2 100644
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
 
 int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState * nc);
 
 uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
Signature · 1 line
2.21.1
#23 lore raw

Re: [RFC v1 4/4] vhost: introduce vhost_set_vring_ready method

Quoted 11 lines · Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

This patch seems to mix fours things, please use dedicated patches for:

1) introduce queue_enabled method for virtio-bus
2) implement queue_enabled for virtio-pci
3) introduce vhost_set_vring_ready method for vhost ops
4) implement vhost_set_vring_ready for vdpa (need to be squashed into the patch of vhost-vdpa).

Thanks

Quoted 193 lines

Author: Jason Wang <[email protected]>

---
  hw/net/vhost_net-stub.c           |  5 +++++
  hw/net/vhost_net.c                | 16 ++++++++++++++++
  hw/virtio/vhost-vdpa.c            |  9 +++------
  hw/virtio/virtio-pci.c            | 13 +++++++++++++
  hw/virtio/virtio.c                |  6 ++++++
  include/hw/virtio/vhost-backend.h |  2 ++
  include/hw/virtio/virtio-bus.h    |  4 ++++
  include/net/vhost_net.h           |  1 +
  8 files changed, 50 insertions(+), 6 deletions(-)
hw/net/vhost_net-stub.c +5 -0
index aac0e98228..f5ef1e3055 100644
@@ -86,6 +86,11 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
      return 0;
  }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}
+
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      return 0;
hw/net/vhost_net.c +16 -0
index 0d13fda2fc..463e333531 100644
@@ -380,6 +380,10 @@ int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
                  goto err_start;
              }
          }
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }
      }
 
      return 0;
@@ -487,6 +491,18 @@ int vhost_set_vring_enable(NetClientState *nc, int enable)
      return 0;
  }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+
+    return 0;
+}
+
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      const VhostOps *vhost_ops = net->dev.vhost_ops;
hw/virtio/vhost-vdpa.c +3 -6
index 213b327600..49224ef9f8 100644
@@ -325,18 +325,15 @@ static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)
      return idx - dev->vq_index;
  }
 
-static int vhost_vdpa_set_vring_enable(struct vhost_dev *dev, int enable)
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
  {
      int i;
 
      for (i = 0; i < dev->nvqs; ++i) {
          struct vhost_vring_state state = {
              .index = dev->vq_index + i,
-            .num   = enable,
+            .num = 1,
          };
-
-        state.num = 1;
-
          vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
      }
 
@@ -368,7 +365,7 @@ const VhostOps vdpa_ops = {
          .vhost_set_owner = vhost_vdpa_set_owner,
          .vhost_reset_device = vhost_vdpa_reset_device,
          .vhost_get_vq_index = vhost_vdpa_get_vq_index,
-        .vhost_set_vring_enable = vhost_vdpa_set_vring_enable,
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
          .vhost_requires_shm_log = NULL,
          .vhost_migration_done = NULL,
          .vhost_backend_can_merge = NULL,
hw/virtio/virtio-pci.c +13 -0
index c6b47a9c73..4aaf5d953e 100644
@@ -1103,6 +1103,18 @@ static AddressSpace *virtio_pci_get_dma_as(DeviceState *d)
      return pci_get_address_space(dev);
  }
 
+static bool virtio_pci_queue_enabled(DeviceState *d, int n)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return proxy->vqs[vdev->queue_sel].enabled;
+    }
+
+    return virtio_queue_get_desc_addr(vdev, n) != 0;
+}
+
  static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
                                     struct virtio_pci_cap *cap)
  {
@@ -2053,6 +2065,7 @@ static void virtio_pci_bus_class_init(ObjectClass *klass, void *data)
      k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
      k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
      k->get_dma_as = virtio_pci_get_dma_as;
+    k->queue_enabled = virtio_pci_queue_enabled;
  }
 
  static const TypeInfo virtio_pci_bus_info = {
hw/virtio/virtio.c +6 -0
index 04716b5f6c..09732a8836 100644
@@ -3169,6 +3169,12 @@ hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 
  bool virtio_queue_enabled(VirtIODevice *vdev, int n)
  {
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (k->queue_enabled)
+        return k->queue_enabled(qbus->parent, n);
+
      return virtio_queue_get_desc_addr(vdev, n) != 0;
  }
 
include/hw/virtio/vhost-backend.h +2 -0
index d81bd9885f..ce8de6d308 100644
@@ -78,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
  typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
  typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                           int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
  typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
  typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                         char *mac_addr);
@@ -140,6 +141,7 @@ typedef struct VhostOps {
      vhost_reset_device_op vhost_reset_device;
      vhost_get_vq_index_op vhost_get_vq_index;
      vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
      vhost_requires_shm_log_op vhost_requires_shm_log;
      vhost_migration_done_op vhost_migration_done;
      vhost_backend_can_merge_op vhost_backend_can_merge;
include/hw/virtio/virtio-bus.h +4 -0
index 38c9399cd4..0f6f215925 100644
@@ -83,6 +83,10 @@ typedef struct VirtioBusClass {
       */
      int (*ioeventfd_assign)(DeviceState *d, EventNotifier *notifier,
                              int n, bool assign);
+    /*
+     * Whether queue number n is enabled.
+     */
+    bool (*queue_enabled)(DeviceState *d, int n);
      /*
       * Does the transport have variable vring alignment?
       * (ie can it ever call virtio_queue_set_align()?)
include/net/vhost_net.h +1 -0
index 6f3a624cf7..db473ff4d2 100644
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
  VHostNetState *get_vhost_net(NetClientState *nc);
 
  int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState * nc);
 
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
author
#24 lore raw

Re: [RFC v1 4/4] vhost: introduce vhost_set_vring_ready method

Quoted 235 lines · Jason Wang
Quoted 11 lines · Cindy Lu

From: Jason Wang <[email protected]>

Vhost-vdpa introduces VHOST_VDPA_SET_VRING_ENABLE which complies the semantic of queue_enable defined in virtio spec. This method can be used for preventing device from executing request for a specific virtqueue. This patch introduces the vhost_ops for this.

Note that, we've already had vhost_set_vring_enable which has different semantic which allows to enable or disable a specific virtqueue for some kinds of vhost backends. E.g vhost-user use this to changes the number of active queue pairs.

This patch seems to mix fours things, please use dedicated patches for:

1) introduce queue_enabled method for virtio-bus
2) implement queue_enabled for virtio-pci
3) introduce vhost_set_vring_ready method for vhost ops
4) implement vhost_set_vring_ready for vdpa (need to be squashed into the patch of vhost-vdpa).

Thanks

Sure, Will fix this

hw/net/vhost_net-stub.c int vhost_set_vring_enable(NetClientState *nc, int

Quoted 19 lines

Author: Jason Wang <[email protected]>

---
  hw/net/vhost_net-stub.c           |  5 +++++
  hw/net/vhost_net.c                | 16 ++++++++++++++++
  hw/virtio/vhost-vdpa.c            |  9 +++------
  hw/virtio/virtio-pci.c            | 13 +++++++++++++
  hw/virtio/virtio.c                |  6 ++++++
  include/hw/virtio/vhost-backend.h |  2 ++
  include/hw/virtio/virtio-bus.h    |  4 ++++
  include/net/vhost_net.h           |  1 +
  8 files changed, 50 insertions(+), 6 deletions(-)
hw/net/vhost_net-stub.c +0 -0
index aac0e98228..f5ef1e3055 100644
@@ -86,6 +86,11 @@ int vhost_set_vring_enable(NetClientState *nc, int

enable)

hw/net/vhost_net.c int vhost_net_start(VirtIODevice *dev,

Quoted 16 lines
hw/net/vhost_net-stub.c +5 -0
      return 0;
  }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    return 0;
+}
+
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      return 0;
hw/net/vhost_net.c +0 -0
index 0d13fda2fc..463e333531 100644
@@ -380,6 +380,10 @@ int vhost_net_start(VirtIODevice *dev,

NetClientState *ncs,

hw/net/vhost_net.c int vhost_set_vring_enable(NetClientState *nc, int

Quoted 11 lines
hw/net/vhost_net.c +4 -0
                  goto err_start;
              }
          }
+
+        if (virtio_queue_enabled(dev, i)) {
+            vhost_set_vring_ready(peer);
+        }
      }
 
      return 0;
@@ -487,6 +491,18 @@ int vhost_set_vring_enable(NetClientState *nc, int

enable)

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vq_index(struct

Quoted 23 lines
hw/net/vhost_net.c +12 -0
      return 0;
  }
 
+int vhost_set_vring_ready(NetClientState *nc)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    if (vhost_ops && vhost_ops->vhost_set_vring_ready) {
+        return vhost_ops->vhost_set_vring_ready(&net->dev);
+    }
+
+    return 0;
+}
+
  int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu)
  {
      const VhostOps *vhost_ops = net->dev.vhost_ops;
hw/virtio/vhost-vdpa.c +0 -0
index 213b327600..49224ef9f8 100644
@@ -325,18 +325,15 @@ static int vhost_vdpa_get_vq_index(struct

vhost_dev *dev, int idx)

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vq_index(struct vhost_dev *dev, int idx)

Quoted 4 lines
hw/virtio/vhost-vdpa.c +0 -1
      return idx - dev->vq_index;
  }
 
-static int vhost_vdpa_set_vring_enable(struct vhost_dev *dev, int

enable)

hw/virtio/virtio-pci.c static AddressSpace

Quoted 30 lines
hw/virtio/vhost-vdpa.c +3 -5
+static int vhost_vdpa_set_vring_ready(struct vhost_dev *dev)
  {
      int i;
 
      for (i = 0; i < dev->nvqs; ++i) {
          struct vhost_vring_state state = {
              .index = dev->vq_index + i,
-            .num   = enable,
+            .num = 1,
          };
-
-        state.num = 1;
-
          vhost_vdpa_call(dev, VHOST_VDPA_SET_VRING_ENABLE, &state);
      }
 
@@ -368,7 +365,7 @@ const VhostOps vdpa_ops = {
          .vhost_set_owner = vhost_vdpa_set_owner,
          .vhost_reset_device = vhost_vdpa_reset_device,
          .vhost_get_vq_index = vhost_vdpa_get_vq_index,
-        .vhost_set_vring_enable = vhost_vdpa_set_vring_enable,
+        .vhost_set_vring_ready = vhost_vdpa_set_vring_ready,
          .vhost_requires_shm_log = NULL,
          .vhost_migration_done = NULL,
          .vhost_backend_can_merge = NULL,
hw/virtio/virtio-pci.c +0 -0
index c6b47a9c73..4aaf5d953e 100644
@@ -1103,6 +1103,18 @@ static AddressSpace

*virtio_pci_get_dma_as(DeviceState *d)

hw/virtio/virtio-pci.c static void virtio_pci_bus_class_init(ObjectClass

Quoted 19 lines
hw/virtio/virtio-pci.c +12 -0
      return pci_get_address_space(dev);
  }
 
+static bool virtio_pci_queue_enabled(DeviceState *d, int n)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return proxy->vqs[vdev->queue_sel].enabled;
+    }
+
+    return virtio_queue_get_desc_addr(vdev, n) != 0;
+}
+
  static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
                                     struct virtio_pci_cap *cap)
  {
@@ -2053,6 +2065,7 @@ static void virtio_pci_bus_class_init(ObjectClass

*klass, void *data)

hw/virtio/virtio.c hwaddr virtio_queue_get_desc_addr(VirtIODevice

Quoted 12 lines
hw/virtio/virtio-pci.c +1 -0
      k->ioeventfd_enabled = virtio_pci_ioeventfd_enabled;
      k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
      k->get_dma_as = virtio_pci_get_dma_as;
+    k->queue_enabled = virtio_pci_queue_enabled;
  }
 
  static const TypeInfo virtio_pci_bus_info = {
hw/virtio/virtio.c +0 -0
index 04716b5f6c..09732a8836 100644
@@ -3169,6 +3169,12 @@ hwaddr virtio_queue_get_desc_addr(VirtIODevice

*vdev, int n)

Review

Quoted 13 lines
hw/virtio/virtio.c +6 -0
  bool virtio_queue_enabled(VirtIODevice *vdev, int n)
  {
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (k->queue_enabled)
+        return k->queue_enabled(qbus->parent, n);
+
      return virtio_queue_get_desc_addr(vdev, n) != 0;
  }
 
(quoted diff) +0 -0

b/include/hw/virtio/vhost-backend.h

include/hw/virtio/vhost-backend.h typedef int (*vhost_reset_device_op)(struct vhost_dev

Quoted 4 lines

index d81bd9885f..ce8de6d308 100644

include/hw/virtio/vhost-backend.h +0 -0
@@ -78,6 +78,7 @@ typedef int (*vhost_reset_device_op)(struct vhost_dev

*dev);

Review

Quoted 16 lines
include/hw/virtio/vhost-backend.h +2 -0
  typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
  typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
                                           int enable);
+typedef int (*vhost_set_vring_ready_op)(struct vhost_dev *dev);
  typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
  typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
                                         char *mac_addr);
@@ -140,6 +141,7 @@ typedef struct VhostOps {
      vhost_reset_device_op vhost_reset_device;
      vhost_get_vq_index_op vhost_get_vq_index;
      vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_set_vring_ready_op vhost_set_vring_ready;
      vhost_requires_shm_log_op vhost_requires_shm_log;
      vhost_migration_done_op vhost_migration_done;
      vhost_backend_can_merge_op vhost_backend_can_merge;
(quoted diff) +0 -0

b/include/hw/virtio/virtio-bus.h

include/net/vhost_net.h int vhost_net_notify_migration_done(VHostNetState

Quoted 19 lines

index 38c9399cd4..0f6f215925 100644

include/hw/virtio/virtio-bus.h +4 -0
@@ -83,6 +83,10 @@ typedef struct VirtioBusClass {
       */
      int (*ioeventfd_assign)(DeviceState *d, EventNotifier *notifier,
                              int n, bool assign);
+    /*
+     * Whether queue number n is enabled.
+     */
+    bool (*queue_enabled)(DeviceState *d, int n);
      /*
       * Does the transport have variable vring alignment?
       * (ie can it ever call virtio_queue_set_align()?)
include/net/vhost_net.h +0 -0
index 6f3a624cf7..db473ff4d2 100644
@@ -35,6 +35,7 @@ int vhost_net_notify_migration_done(VHostNetState

*net, char* mac_addr);

Quoted 7 lines
include/net/vhost_net.h +1 -0
  VHostNetState *get_vhost_net(NetClientState *nc);
 
  int vhost_set_vring_enable(NetClientState * nc, int enable);
+int vhost_set_vring_ready(NetClientState * nc);
 
  uint64_t vhost_net_get_acked_features(VHostNetState *net);
 
#25 lore raw
In reply to Cindy Lu
Quoted 4 lines · Cindy Lu

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. This RFC introduce the vDPA support in qemu

Looks good overall, see comments inline.

And I think we need reorder the patches and split some patches into smaller ones.

E.g

1) qemu_get_peer
2) queue_enabled method for virtio-bus
3) queue_enabled for virtio-pci
4) set_vring_ready for vhost_ops
5) vhost_set_state for vhost_ops
6) call vhost_set_state in virtio_net_vhost_status()
7) generic vdpa support
8) vhost-vdpa net backend

Thanks

Quoted 35 lines

Cindy Lu (3):
net: Introduce qemu_get_peer
vhost-vdpa: introduce vhost-vdpa net client
vhost-vdpa: implement vhost-vdpa backend

Jason Wang (1):
vhost: introduce vhost_set_vring_ready method

  hw/net/vhost_net-stub.c           |   5 +
  hw/net/vhost_net.c                |  75 +++++-
  hw/net/virtio-net.c               |   9 +
  hw/virtio/Makefile.objs           |   2 +-
  hw/virtio/vhost-backend.c         |   3 +
  hw/virtio/vhost-vdpa.c            | 376 ++++++++++++++++++++++++++++++
  hw/virtio/vhost.c                 |   5 +
  hw/virtio/virtio-pci.c            |  13 ++
  hw/virtio/virtio.c                |   6 +
  include/hw/virtio/vhost-backend.h |   8 +-
  include/hw/virtio/vhost-vdpa.h    |  14 ++
  include/hw/virtio/virtio-bus.h    |   4 +
  include/net/net.h                 |   1 +
  include/net/vhost-vdpa.h          |  18 ++
  include/net/vhost_net.h           |   2 +
  net/Makefile.objs                 |   2 +-
  net/clients.h                     |   2 +
  net/net.c                         |   7 +
  net/vhost-vdpa.c                  | 211 +++++++++++++++++
  qapi/net.json                     |  21 +-
  20 files changed, 773 insertions(+), 11 deletions(-)

create mode 100644 hw/virtio/vhost-vdpa.c create mode 100644 include/hw/virtio/vhost-vdpa.h create mode 100644 include/net/vhost-vdpa.h create mode 100644 net/vhost-vdpa.c

#26 lore raw
In reply to Cindy Lu
Quoted 4 lines · Cindy Lu

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. This RFC introduce the vDPA support in qemu

And please mention the TODO list. E.g:

1) vIOMMU support
2) software assisted live migration

Thanks

author
#27 lore raw

Thanks Jason, I will fix these problems and send new version soon

Quoted 16 lines · Jason Wang
Quoted 4 lines · Cindy Lu

vDPA device is a device that uses a datapath which complies with the virtio specifications with vendor specific control path. vDPA devices can be both physically located on the hardware or emulated by software. This RFC introduce the vDPA support in qemu

And please mention the TODO list. E.g:

1) vIOMMU support
2) software assisted live migration

Thanks