vhost: support inflight share memory protocol feature

This patch introduces two new messages VHOST_USER_GET_INFLIGHT_FD and VHOST_USER_SET_INFLIGHT_FD to support transferring a shared buffer between qemu and backend.

Firstly, qemu uses VHOST_USER_GET_INFLIGHT_FD to get the shared buffer from backend. Then qemu should send it back through VHOST_USER_SET_INFLIGHT_FD each time we start vhost-user.

This shared buffer is used to process inflight I/O when backend reconnect.

Signed-off-by Lin Li <[email protected]>

Signed-off-by Xun Ni <[email protected]>

Signed-off-by Yu Zhang <[email protected]>

Signed-off-by JinYu <[email protected]>

---
v1 - specify the APIs are split-ring only
v2 - fix APIs and judge split or packed
v3 - Add rte_vhost_ prefix and fix one issue.
v4 - add the packed ring support
v5 - revise get_vring_base func depend on Tiwei's suggestion

---
 lib/librte_vhost/rte_vhost.h           | 255 ++++++++++++++-
 lib/librte_vhost/rte_vhost_version.map |  12 +
 lib/librte_vhost/vhost.c               | 396 +++++++++++++++++++++-
 lib/librte_vhost/vhost.h               |  61 ++--
 lib/librte_vhost/vhost_user.c          | 437 ++++++++++++++++++++++++-
 lib/librte_vhost/vhost_user.h          |  13 +-
 6 files changed, 1128 insertions(+), 46 deletions(-)

lib/librte_vhost/rte_vhost.h +7 -0

index 0226b3eff..3f01429b1 100644
@@ -11,6 +11,7 @@
  */
 
 #include <stdint.h>
+#include <stdbool.h>
 #include <sys/eventfd.h>
 
 #include <rte_memory.h>

@@ -71,6 +72,10 @@ extern "C" {
 #define VHOST_USER_PROTOCOL_F_HOST_NOTIFIER 11
 #endif
 
+#ifndef VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
+#define VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD 12
+#endif
+
 /** Indicate whether protocol features negotiation is supported. */
 #ifndef VHOST_USER_F_PROTOCOL_FEATURES
 #define VHOST_USER_F_PROTOCOL_FEATURES	30

@@ -98,10 +103,92 @@ struct rte_vhost_memory {
 	struct rte_vhost_mem_region regions[];
 };
 
+struct inflight_desc_packed {
+	uint8_t inflight;

lib/librte_vhost/rte_vhost.h +18 -0

+
+struct inflight_info_packed {
+	uint64_t features;
+	uint16_t version;
+	uint16_t desc_num;
+	uint16_t free_head;
+	uint16_t old_free_head;
+	uint16_t used_idx;
+	uint16_t old_used_idx;
+	uint8_t used_wrap_counter;
+	uint8_t old_used_wrap_counter;
+	uint8_t padding[7];
+	struct inflight_desc_packed desc[0];
+};
+
+struct rte_vhost_resubmit_desc {
+	uint16_t index;
+	uint64_t counter;

lib/librte_vhost/rte_vhost.h +5 -0

+		struct inflight_info_packed *inflight_packed;
+	};
+
+	struct rte_vhost_resubmit_info *resubmit_inflight;
+};

lib/librte_vhost/rte_vhost.h +5 -0

+
+struct vring_packed_desc {
+	uint64_t addr;
+	uint32_t len;
+	uint16_t id;

lib/librte_vhost/rte_vhost.h +15 -3

+#endif
+
 struct rte_vhost_vring {
-	struct vring_desc	*desc;
-	struct vring_avail	*avail;
-	struct vring_used	*used;
+	union {
+		struct vring_desc	*desc;
+		struct vring_packed_desc *desc_packed;
+	};
+	union {
+		struct vring_avail	*avail;
+		struct vring_packed_desc_event *driver_event;
+	};
+	union {
+		struct vring_used	*used;
+		struct vring_packed_desc_event *device_event;
+	};
+
 	uint64_t		log_guest_addr;
 
 	/** Deprecated, use rte_vhost_vring_call() instead. */

lib/librte_vhost/rte_vhost.h +7 -0

+ */
+int __rte_experimental
+rte_vhost_vq_is_packed(int vid);
+
+/**
+ * Get guest inflight vring info, including inflight ring and resubmit list.
+ *

lib/librte_vhost/rte_vhost.h +4 -1

+rte_vhost_get_vhost_ring_inflight(int vid, uint16_t vring_idx,
+	struct rte_vhost_ring_inflight *vring);
+
 /**
  * Get guest vring info, including the vring address, vring size, etc.
  *

@@ -616,7 +730,7 @@ int rte_vhost_get_mem_table(int vid, struct rte_vhost_memory **mem);
  *  0 on success, -1 on failure
  */
 int rte_vhost_get_vhost_vring(int vid, uint16_t vring_idx,
-			      struct rte_vhost_vring *vring);
+	struct rte_vhost_vring *vring);
 
 /**
  * Notify the guest that used descriptors have been added to the vring.  This

lib/librte_vhost/rte_vhost.h +104 -0

+ *
+ * This function save descriptors that has been comsumed in available
+ * ring
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_inflight_desc_split(int vid, uint16_t vring_idx,
+	uint16_t idx);
+
+/**
+ * set packed inflight descriptor and get corresponding inflight entry
+ *
+ * This function save descriptors that has been comsumed
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_inflight_desc_packed(int vid, uint16_t vring_idx,
+	uint16_t head, uint16_t last, uint16_t *inflight_entry);
+
+/**
+ * save the head of list that the last batch of used descriptors.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  descriptor entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_last_inflight_io_split(int vid,
+	uint16_t vring_idx, uint16_t idx);
+
+/**
+ * update the inflight free_head, used_idx and used_wrap_counter.
+ *
+ * This function will update status first before updating descriptors
+ * to used
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_set_last_inflight_io_packed(int vid,
+	uint16_t vring_idx, uint16_t head);
+
+/**
+ * clear the split inflight status.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param last_used_idx
+ *  last used idx of used ring
+ * @param idx
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_clr_inflight_desc_split(int vid, uint16_t vring_idx,
+	uint16_t last_used_idx, uint16_t idx);
+
+/**
+ * clear the packed inflight status.
+ *
+ * @param vid
+ *  vhost device ID
+ * @param vring_idx
+ *  vring index
+ * @param head
+ *  inflight entry index
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_clr_inflight_desc_packed(int vid, uint16_t vring_idx,
+	uint16_t head);
+
 /**
  * Get vhost RX queue avail count.
  *

lib/librte_vhost/rte_vhost.h +23 -1

 /**
  * Get last_avail/used_idx of the vhost virtqueue

@@ -676,6 +897,28 @@ int __rte_experimental
 rte_vhost_get_vring_base(int vid, uint16_t queue_id,
 		uint16_t *last_avail_idx, uint16_t *last_used_idx);
 
+/**
+ * Get last_avail/last_used of the vhost virtqueue
+ *
+ * This function is designed for the reconnection and it's specific for
+ * the packed ring as we can get the two parameters from the inflight
+ * queueregion
+ *
+ * @param vid
+ *  vhost device ID
+ * @param queue_id
+ *  vhost queue index
+ * @param last_avail_idx
+ *  vhost last_avail_idx to get
+ * @param last_used_idx
+ *  vhost last_used_idx to get
+ * @return
+ *  0 on success, -1 on failure
+ */
+int __rte_experimental
+rte_vhost_get_vring_base_from_inflight(int vid,
+	uint16_t queue_id, uint16_t *last_avail_idx, uint16_t *last_used_idx);
+
 /**
  * Set last_avail/used_idx of the vhost virtqueue
  *

@@ -692,7 +935,7 @@ rte_vhost_get_vring_base(int vid, uint16_t queue_id,
  */
 int __rte_experimental
 rte_vhost_set_vring_base(int vid, uint16_t queue_id,
-		uint16_t last_avail_idx, uint16_t last_used_idx);
+	uint16_t last_avail_idx, uint16_t last_used_idx);
 
 /**
  * Register external message handling callbacks

lib/librte_vhost/rte_vhost_version.map +16 -2

index 5f1d4a75c..99f1134ea 100644
@@ -87,4 +87,16 @@ EXPERIMENTAL {
 	rte_vdpa_relay_vring_used;
 	rte_vhost_extern_callback_register;
 	rte_vhost_driver_set_protocol_features;
+	rte_vhost_set_inflight_desc_split;
+	rte_vhost_clr_inflight_desc_split;
+	rte_vhost_set_last_inflight_io_split;
+	rte_vhost_get_vhost_ring_inflight;
+	rte_vhost_vq_is_packed;
+	rte_vhost_set_inflight_desc_packed;
+	rte_vhost_clr_inflight_desc_packed;
+	rte_vhost_set_last_inflight_io_packed;
+	rte_vhost_get_vring_base_counter;
+	rte_vhost_get_vring_base_from_inflight;
+	rte_vhost_get_vring_base_counter_from_inflight;
+	rte_vhost_set_vring_base_counter;
 };

@@ -939,13 +1270,48 @@ int rte_vhost_get_log_base(int vid, uint64_t *log_base,
 int rte_vhost_get_vring_base(int vid, uint16_t queue_id,
 		uint16_t *last_avail_idx, uint16_t *last_used_idx)
 {
+	struct vhost_virtqueue *vq;
 	struct virtio_net *dev = get_device(vid);
 
 	if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
 		return -1;
 
-	*last_avail_idx = dev->virtqueue[queue_id]->last_avail_idx;
-	*last_used_idx = dev->virtqueue[queue_id]->last_used_idx;
+	vq = dev->virtqueue[queue_id];
+	if (!vq)
+		return -1;

lib/librte_vhost/rte_vhost_version.map +27 -2

+
+	return 0;
+}
+
+int rte_vhost_get_vring_base_from_inflight(int vid,
+	uint16_t queue_id, uint16_t *last_avail_idx, uint16_t *last_used_idx)
+{
+	struct inflight_info_packed *inflight_info;
+	struct virtio_net *dev = get_device(vid);
+
+	if (dev == NULL || last_avail_idx == NULL || last_used_idx == NULL)
+		return -1;
+
+	if (!vq_is_packed(dev))
+		return -1;
+
+	inflight_info = dev->virtqueue[queue_id]->inflight_packed;
+	if (!inflight_info)
+		return -1;
+
+	*last_avail_idx = (inflight_info->old_used_wrap_counter << 15) |
+				inflight_info->old_used_idx;
+	*last_used_idx = *last_avail_idx;
 
 	return 0;
 }

@@ -953,13 +1319,25 @@ int rte_vhost_get_vring_base(int vid, uint16_t queue_id,
 int rte_vhost_set_vring_base(int vid, uint16_t queue_id,
 		uint16_t last_avail_idx, uint16_t last_used_idx)
 {
+	struct vhost_virtqueue *vq;
 	struct virtio_net *dev = get_device(vid);
 
 	if (!dev)
 		return -1;
 
-	dev->virtqueue[queue_id]->last_avail_idx = last_avail_idx;
-	dev->virtqueue[queue_id]->last_used_idx = last_used_idx;
+	vq = dev->virtqueue[queue_id];
+	if (!vq)
+		return -1;

lib/librte_vhost/rte_vhost_version.map +0 -0

 	return 0;
 }

lib/librte_vhost/vhost.h +16 -0

index 884befa85..e9d0b983d 100644
@@ -88,6 +88,22 @@ struct vring_used_elem_packed {
 	uint32_t count;
 };
 
+struct inflight_desc_split {
+	uint8_t		inflight;
+	uint8_t		padding[5];
+	uint16_t	next;
+	uint64_t	counter;
+};
+
+struct inflight_info_split {
+	uint64_t		features;
+	uint16_t		version;
+	uint16_t		desc_num;
+	uint16_t		last_inflight_io;
+	uint16_t		used_idx;
+	struct inflight_desc_split desc[0];
+};
+