Skip to contentSkip to conversation

vhost: support async dequeue data path

PATCH New [email protected] 7 participants 73 messages Apr 7, 2022 – Jun 17, 2022

lore
Share & tools
lore
#0 lore raw

cover vhost: support async dequeue data path

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch implements vhost async dequeue data path for split ring.

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 292 +++++++++++-----
 examples/vhost/main.h                  |  35 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  33 ++
 lib/vhost/version.map                  |   3 +
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 459 ++++++++++++++++++++++---
 10 files changed, 711 insertions(+), 148 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 66 +++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 37 deletions(-)
lib/vhost/virtio_net.c +29 -37
index 5f432b0d77..a2d04a1f60 100644
@@ -1030,9 +1030,9 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
@@ -1043,10 +1043,17 @@ sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1165,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
+			sync_fill_seg(dev, vq, m, mbuf_offset,
 					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+					buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2474,7 +2481,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  bool legacy_ol_flags)
 {
 	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint64_t buf_addr, buf_iova, buf_len;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2489,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2519,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2544,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, m, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, true);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2559,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2578,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2598,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#2 lore raw

PATCH 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
index a2d04a1f60..709ff483a3 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst,
+						 (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1161,8 +1168,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+						buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#3 lore raw

PATCH 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 47 ++++++++++++++++++++++++++++++++----------
 2 files changed, 37 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..9209558465 100644
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];
  * inflight async packet information
  */
 struct async_inflight_info {
+	struct virtio_net_hdr nethdr;
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +36 -11

97 lines not on this page · raw message

Signature · 1 line
2.17.1
#4 lore raw

PATCH 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 lib/vhost/rte_vhost_async.h            |  33 +++
 lib/vhost/version.map                  |   3 +
 lib/vhost/virtio_net.c                 | 335 +++++++++++++++++++++++++
 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index 886f8f5e72..40cf315170 100644
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +4 -0
index 42a5f2d990..422a6673cb 100644
@@ -55,6 +55,10 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to accelerate
+  receiving pkts from guest.
 
 Removed Items
 -------------
lib/vhost/rte_vhost_async.h +33 -0
index f1293c6a9d..23fe1a7316 100644
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  The amount of in-flight packets. If error occurred, its value is set to -1.
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +3 -0
index 0a66c5840c..514e3ff6a6 100644
@@ -87,6 +87,9 @@ EXPERIMENTAL {
 
 	# added in 22.03
 	rte_vhost_async_dma_configure;
+
+	# added in 22.07
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +335 -0

338 lines not on this page · raw message

Signature · 1 line
2.17.1
#5 lore raw

PATCH 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 292 ++++++++++++++++++++---------
 examples/vhost/main.h              |  35 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +202 -90

438 lines not on this page · raw message

examples/vhost/main.h +31 -4
index b4a453e77e..40ac2841d1 100644
@@ -52,7 +52,8 @@ struct vhost_dev {
 	uint64_t features;
 	size_t hdr_len;
 	uint16_t nr_vrings;
-	uint16_t pkts_inflight;
+	uint16_t pkts_enq_inflight;
+	uint16_t pkts_deq_inflight;
 	struct rte_vhost_memory *mem;
 	struct device_statistics stats;
 	TAILQ_ENTRY(vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -88,6 +102,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#6 lore raw

v2 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 292 +++++++++++-----
 examples/vhost/main.h                  |  35 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  33 ++
 lib/vhost/version.map                  |   3 +
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 460 ++++++++++++++++++++++---
 10 files changed, 712 insertions(+), 148 deletions(-)
Signature · 1 line
2.17.1
#7 lore raw

PATCH v2 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 66 +++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 37 deletions(-)
lib/vhost/virtio_net.c +29 -37
index 5f432b0d77..a2d04a1f60 100644
@@ -1030,9 +1030,9 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
@@ -1043,10 +1043,17 @@ sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1165,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
+			sync_fill_seg(dev, vq, m, mbuf_offset,
 					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+					buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2474,7 +2481,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  bool legacy_ol_flags)
 {
 	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint64_t buf_addr, buf_iova, buf_len;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2489,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2519,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2544,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, m, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, true);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2559,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2578,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2598,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#8 lore raw

PATCH v2 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
index a2d04a1f60..709ff483a3 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst,
+						 (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1161,8 +1168,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+						buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#9 lore raw

PATCH v2 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..9209558465 100644
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];
  * inflight async packet information
  */
 struct async_inflight_info {
+	struct virtio_net_hdr nethdr;
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#10 lore raw

PATCH v2 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 lib/vhost/rte_vhost_async.h            |  33 +++
 lib/vhost/version.map                  |   3 +
 lib/vhost/virtio_net.c                 | 335 +++++++++++++++++++++++++
 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index 886f8f5e72..40cf315170 100644
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +4 -0
index 42a5f2d990..422a6673cb 100644
@@ -55,6 +55,10 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to accelerate
+  receiving pkts from guest.
 
 Removed Items
 -------------
lib/vhost/rte_vhost_async.h +33 -0
index f1293c6a9d..23fe1a7316 100644
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  The amount of in-flight packets. If error occurred, its value is set to -1.
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +3 -0
index 0a66c5840c..514e3ff6a6 100644
@@ -87,6 +87,9 @@ EXPERIMENTAL {
 
 	# added in 22.03
 	rte_vhost_async_dma_configure;
+
+	# added in 22.07
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +335 -0

338 lines not on this page · raw message

Signature · 1 line
2.17.1
#11 lore raw

PATCH v2 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 292 ++++++++++++++++++++---------
 examples/vhost/main.h              |  35 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +202 -90

438 lines not on this page · raw message

examples/vhost/main.h +31 -4
index b4a453e77e..40ac2841d1 100644
@@ -52,7 +52,8 @@ struct vhost_dev {
 	uint64_t features;
 	size_t hdr_len;
 	uint16_t nr_vrings;
-	uint16_t pkts_inflight;
+	uint16_t pkts_enq_inflight;
+	uint16_t pkts_deq_inflight;
 	struct rte_vhost_memory *mem;
 	struct device_statistics stats;
 	TAILQ_ENTRY(vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -88,6 +102,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#12 lore raw

v3 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 292 ++++++++++-----
 examples/vhost/main.h                  |  35 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  33 ++
 lib/vhost/version.map                  |   3 +
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 470 ++++++++++++++++++++++---
 10 files changed, 720 insertions(+), 150 deletions(-)
Signature · 1 line
2.17.1
#13 lore raw

PATCH v3 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 76 ++++++++++++++++++++----------------------
 1 file changed, 37 insertions(+), 39 deletions(-)
lib/vhost/virtio_net.c +37 -39
index 5f432b0d77..6d53016c75 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
+			sync_fill_seg(dev, vq, m, mbuf_offset,
 					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+					buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
Reviewed-by on v3 1/5
#14 lore raw

Re: [PATCH v3 1/5] vhost: prepare sync for descriptor to mbuf refactoring

Hi Xuan,

Quoted 8 lines · On 4/19/22 05:43, [email protected] wrote:

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

Quoted 5 lines
---
  lib/vhost/virtio_net.c | 76 ++++++++++++++++++++----------------------
  1 file changed, 37 insertions(+), 39 deletions(-)

Nice refactoring, thanks for donig it:

Thanks,
Maxime

#15 lore raw

PATCH v3 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
index 6d53016c75..391fb82f0e 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst,
+						 (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1174,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+						buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
Reviewed-by on v3 2/5
#16 lore raw

Re: [PATCH v3 2/5] vhost: prepare async for descriptor to mbuf refactoring

Quoted 40 lines · On 4/19/22 05:43, [email protected] wrote:

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
  lib/vhost/virtio_net.c | 23 +++++++++++++++--------
  1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +3 -6
index 6d53016c75..391fb82f0e 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
  }
 
  static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
  		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
  {
  	struct vhost_async *async = vq->async;
  	uint64_t mapped_len;
  	uint32_t buf_offset = 0;
+	void *src, *dst;
  	void *host_iova;
 
  	while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
  			return -1;
  		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))

lib/vhost/virtio_net.c async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, · line 1028

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst,
+						 (size_t)mapped_len)))

Minor, but it may fit in a single line.

Quoted 4 lines
lib/vhost/virtio_net.c +0 -0
  			return -1;
 
  		cpy_len -= (uint32_t)mapped_len;

lib/vhost/virtio_net.c mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, · line 1181

Quoted 10 lines
lib/vhost/virtio_net.c +2 -2
@@ -1167,8 +1174,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
  		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
  		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+						buf_iova + buf_offset, cpy_len, true) < 0)
  				goto error;
  		} else {
  			sync_fill_seg(dev, vq, m, mbuf_offset,

Thanks,
Maxime

#17 lore raw

PATCH v3 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..9209558465 100644
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];
  * inflight async packet information
  */
 struct async_inflight_info {
+	struct virtio_net_hdr nethdr;
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#18 lore raw

Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling

We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.

Copy/pasting the code:

+static __rte_always_inline uint16_t +async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id,
+ uint16_t vchan_id, bool legacy_ol_flags)
+{
+ uint16_t start_idx, from, i;
+ uint16_t nr_cpl_pkts = 0;
+ struct async_inflight_info *pkts_info;
+ struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
+

Please, don't pass queue_id as an input parameter for async_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.

Signature · 1 line
David Marchand
#19 lore raw

Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling

Quoted 16 lines · David Marchand

We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.

Copy/pasting the code:

(quoted diff) +10 -0
+static __rte_always_inline uint16_t
+async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id,
+ uint16_t vchan_id, bool legacy_ol_flags)
+{
+ uint16_t start_idx, from, i;
+ uint16_t nr_cpl_pkts = 0;
+ struct async_inflight_info *pkts_info;
+ struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
+
Quoted 6 lines

Please, don't pass queue_id as an input parameter for async_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.

I think David's comment was intended to be a reply to patch 4, but I agree with him.

Could you please fix this and also fix the build issues reported by the CI? I'll continue the review on V4.

Thanks,
Maxime

author
#20 lore raw

RE: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling

Hi Maxime, David,

Quoted 38 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Friday, April 22, 2022 11:46 PM
To: David Marchand <[email protected]>; Ding, Xuan <[email protected]>
Cc: Xia, Chenbo <[email protected]>; dev <[email protected]>; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]
Subject: Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling

Quoted 13 lines · David Marchand

We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.

Copy/pasting the code:

(quoted diff) +7 -0
+static __rte_always_inline uint16_t
+async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t
+queue_id,  struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id,
+uint16_t vchan_id, bool legacy_ol_flags) {  uint16_t start_idx, from,
+i;  uint16_t nr_cpl_pkts = 0;  struct async_inflight_info *pkts_info;
+struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
+
Quoted 6 lines

Please, don't pass queue_id as an input parameter for async_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.

I think David's comment was intended to be a reply to patch 4, but I agree with him.

Quoted 2 lines

Could you please fix this and also fix the build issues reported by the CI? I'll continue the review on V4.

Thanks for your suggestion, please see v4.

Regards,
Xuan

Quoted 3 lines

Thanks,
Maxime

Reviewed-by on v3 3/5
#21 lore raw

Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]
Quoted 11 lines · On 4/19/22 05:43, [email protected] wrote:

From: Xuan Ding <[email protected]>

This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
  lib/vhost/vhost.h      |  1 +
  lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
  2 files changed, 38 insertions(+), 11 deletions(-)

Thanks,
Maxime

#22 lore raw

PATCH v3 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 lib/vhost/rte_vhost_async.h            |  33 +++
 lib/vhost/version.map                  |   3 +
 lib/vhost/virtio_net.c                 | 335 +++++++++++++++++++++++++
 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index 886f8f5e72..40cf315170 100644
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +4 -0
index 42a5f2d990..422a6673cb 100644
@@ -55,6 +55,10 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to accelerate
+  receiving pkts from guest.
 
 Removed Items
 -------------
lib/vhost/rte_vhost_async.h +33 -0
index f1293c6a9d..23fe1a7316 100644
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  The amount of in-flight packets. If error occurred, its value is set to -1.
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +3 -0
index 0a66c5840c..514e3ff6a6 100644
@@ -87,6 +87,9 @@ EXPERIMENTAL {
 
 	# added in 22.03
 	rte_vhost_async_dma_configure;
+
+	# added in 22.07
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +335 -0

338 lines not on this page · raw message

Signature · 1 line
2.17.1
#23 lore raw

PATCH v3 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 292 ++++++++++++++++++++---------
 examples/vhost/main.h              |  35 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +202 -90

438 lines not on this page · raw message

examples/vhost/main.h +31 -4
index b4a453e77e..40ac2841d1 100644
@@ -52,7 +52,8 @@ struct vhost_dev {
 	uint64_t features;
 	size_t hdr_len;
 	uint16_t nr_vrings;
-	uint16_t pkts_inflight;
+	uint16_t pkts_enq_inflight;
+	uint16_t pkts_deq_inflight;
 	struct rte_vhost_memory *mem;
 	struct device_statistics stats;
 	TAILQ_ENTRY(vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -88,6 +102,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#24 lore raw

v4 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 292 +++++++++++-----
 examples/vhost/main.h                  |  35 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  33 ++
 lib/vhost/version.map                  |   3 +
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 467 ++++++++++++++++++++++---
 10 files changed, 716 insertions(+), 151 deletions(-)
Signature · 1 line
2.17.1
#25 lore raw

PATCH v4 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
index 5f432b0d77..d4c94d2a9b 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+			sync_fill_seg(dev, vq, m, mbuf_offset,
+				      buf_addr + buf_offset,
+				      buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
Tested-by on v4 1/5
#26 lore raw

RE: [PATCH v4 1/5] vhost: prepare sync for descriptor to mbuf refactoring

Quoted 19 lines

-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 1/5] vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function
sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

Quoted 6 lines
---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
#27 lore raw

PATCH v4 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
index d4c94d2a9b..a9e2dcd9ce 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+					   buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
Tested-by on v4 2/5
#28 lore raw

RE: [PATCH v4 2/5] vhost: prepare async for descriptor to mbuf refactoring

Quoted 17 lines

-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 2/5] vhost: prepare async for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

Quoted 6 lines
---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
#29 lore raw

PATCH v4 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..00744b234f 100644
@@ -180,6 +180,7 @@ struct async_inflight_info {
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
+	struct virtio_net_hdr nethdr;
 };
 
 struct vhost_async {
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
Tested-by on v4 3/5
#30 lore raw

RE: [PATCH v4 3/5] vhost: merge sync and async descriptor to mbuf filling

Quoted 16 lines

-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 3/5] vhost: merge sync and async descriptor to mbuf filling

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

Quoted 7 lines
---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
#31 lore raw

PATCH v4 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 lib/vhost/rte_vhost_async.h            |  33 +++
 lib/vhost/version.map                  |   3 +
 lib/vhost/virtio_net.c                 | 331 +++++++++++++++++++++++++
 5 files changed, 378 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index 886f8f5e72..40cf315170 100644
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +4 -0
index 42a5f2d990..422a6673cb 100644
@@ -55,6 +55,10 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to accelerate
+  receiving pkts from guest.
 
 Removed Items
 -------------
lib/vhost/rte_vhost_async.h +33 -0
index f1293c6a9d..23fe1a7316 100644
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  The amount of in-flight packets. If error occurred, its value is set to -1.
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +3 -0
index 0a66c5840c..514e3ff6a6 100644
@@ -87,6 +87,9 @@ EXPERIMENTAL {
 
 	# added in 22.03
 	rte_vhost_async_dma_configure;
+
+	# added in 22.07
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +331 -0

334 lines not on this page · raw message

Signature · 1 line
2.17.1
Tested-by on v4 4/5
#32 lore raw

RE: [PATCH v4 4/5] vhost: support async dequeue for split ring

doc/guides/prog_guide/vhost_lib.rst

Quoted 33 lines

-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v4 4/5] vhost: support async dequeue for split ring

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   4 +
 lib/vhost/rte_vhost_async.h            |  33 +++
 lib/vhost/version.map                  |   3 +
 lib/vhost/virtio_net.c                 | 331
+++++++++++++++++++++++++
 5 files changed, 378 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +0 -0
index 886f8f5e72..40cf315170 100644
Reviewed-by on v4 4/5
#33 lore raw

Re: [PATCH v4 4/5] vhost: support async dequeue for split ring

In reply to [email protected]
Quoted 7 lines · On 5/5/22 08:23, [email protected] wrote:

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

Quoted 8 lines
---
  doc/guides/prog_guide/vhost_lib.rst    |   7 +
  doc/guides/rel_notes/release_22_07.rst |   4 +
  lib/vhost/rte_vhost_async.h            |  33 +++
  lib/vhost/version.map                  |   3 +
  lib/vhost/virtio_net.c                 | 331 +++++++++++++++++++++++++
  5 files changed, 378 insertions(+)

Thanks,
Maxime

#34 lore raw

PATCH v4 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 292 ++++++++++++++++++++---------
 examples/vhost/main.h              |  35 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +202 -90

438 lines not on this page · raw message

examples/vhost/main.h +31 -4
index b4a453e77e..40ac2841d1 100644
@@ -52,7 +52,8 @@ struct vhost_dev {
 	uint64_t features;
 	size_t hdr_len;
 	uint16_t nr_vrings;
-	uint16_t pkts_inflight;
+	uint16_t pkts_enq_inflight;
+	uint16_t pkts_deq_inflight;
 	struct rte_vhost_memory *mem;
 	struct device_statistics stats;
 	TAILQ_ENTRY(vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -88,6 +102,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
Tested-by on v4 5/5
#35 lore raw

RE: [PATCH v4 5/5] examples/vhost: support async dequeue data path

Quoted 33 lines

-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v4 5/5] examples/vhost: support async dequeue data path

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 292 ++++++++++++++++++++---------
 examples/vhost/main.h              |  35 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +0 -0
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's
Quoted 1 line

used in combination with dmas

Reviewed-by on v4 5/5
#36 lore raw

Re: [PATCH v4 5/5] examples/vhost: support async dequeue data path

In reply to [email protected]
Quoted 8 lines · On 5/5/22 08:23, [email protected] wrote:

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

Quoted 7 lines
---
  doc/guides/sample_app_ug/vhost.rst |   9 +-
  examples/vhost/main.c              | 292 ++++++++++++++++++++---------
  examples/vhost/main.h              |  35 +++-
  examples/vhost/virtio_net.c        |  16 +-
  4 files changed, 254 insertions(+), 98 deletions(-)

Thanks,
Maxime

#37 lore raw
In reply to [email protected]

Hi Xuan,

Quoted 58 lines · On 5/5/22 08:23, [email protected] wrote:

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

Quoted 12 lines
  doc/guides/prog_guide/vhost_lib.rst    |   7 +
  doc/guides/rel_notes/release_22_07.rst |   4 +
  doc/guides/sample_app_ug/vhost.rst     |   9 +-
  examples/vhost/main.c                  | 292 +++++++++++-----
  examples/vhost/main.h                  |  35 +-
  examples/vhost/virtio_net.c            |  16 +-
  lib/vhost/rte_vhost_async.h            |  33 ++
  lib/vhost/version.map                  |   3 +
  lib/vhost/vhost.h                      |   1 +
  lib/vhost/virtio_net.c                 | 467 ++++++++++++++++++++++---
  10 files changed, 716 insertions(+), 151 deletions(-)

I applied your other series about unsafe API to get DMA inflight packets, so I have some conflicts when applying this series.

Could you please rebase on top of next-virtio/main branch and repost?

Thanks in avance,
Maxime

author
#38 lore raw

Hi Maxime,

Quoted 83 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Friday, May 6, 2022 3:53 AM
To: Ding, Xuan <[email protected]>; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]
Subject: Re: [PATCH v4 0/5] vhost: support async dequeue data path

Hi Xuan,

Quoted 58 lines · On 5/5/22 08:23, [email protected] wrote:

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

Quoted 12 lines
  doc/guides/prog_guide/vhost_lib.rst    |   7 +
  doc/guides/rel_notes/release_22_07.rst |   4 +
  doc/guides/sample_app_ug/vhost.rst     |   9 +-
  examples/vhost/main.c                  | 292 +++++++++++-----
  examples/vhost/main.h                  |  35 +-
  examples/vhost/virtio_net.c            |  16 +-
  lib/vhost/rte_vhost_async.h            |  33 ++
  lib/vhost/version.map                  |   3 +
  lib/vhost/vhost.h                      |   1 +
  lib/vhost/virtio_net.c                 | 467 ++++++++++++++++++++++---
  10 files changed, 716 insertions(+), 151 deletions(-)
Quoted 4 lines

I applied your other series about unsafe API to get DMA inflight packets, so I have some conflicts when applying this series.

Could you please rebase on top of next-virtio/main branch and repost?

Sure, I will rebase on top of latest main branch, please see v5.

Thanks,
Xuan

Quoted 3 lines

Thanks in avance,
Maxime

#39 lore raw

v5 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v4->v5:
* rebase to latest DPDK
* add some checks

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 284 ++++++++++-----
 examples/vhost/main.h                  |  32 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  34 ++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 473 ++++++++++++++++++++++---
 10 files changed, 711 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
#40 lore raw

PATCH v5 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
index 5f432b0d77..d4c94d2a9b 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+			sync_fill_seg(dev, vq, m, mbuf_offset,
+				      buf_addr + buf_offset,
+				      buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#41 lore raw

PATCH v5 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
index d4c94d2a9b..a9e2dcd9ce 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+					   buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#42 lore raw

PATCH v5 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..00744b234f 100644
@@ -180,6 +180,7 @@ struct async_inflight_info {
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
+	struct virtio_net_hdr nethdr;
 };
 
 struct vhost_async {
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#43 lore raw

PATCH v5 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  34 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 384 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index f287b76ebf..09c1c24b48 100644
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+  accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h +34 -0
index 70234debf9..161dabc652 100644
@@ -204,6 +204,40 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  >= 0: The amount of in-flight packets
+ *  -1: meaningless, indicates failed lock acquisition or invalid queue_id/dma_id
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +1 -1
index 5841315386..8c7211bf0d 100644
@@ -90,7 +90,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_vhost_async_get_inflight_thread_unsafe;
-
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +337 -0

340 lines not on this page · raw message

Signature · 1 line
2.17.1
#44 lore raw

Re: [PATCH v5 4/5] vhost: support async dequeue for split ring

Quoted 9 lines · On Fri, 13 May 2022 02:00:21 +0000 [email protected] wrote:

+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *

Please add EXPERIMENTAL header like this, so it shows up in docmentation correctly.

*
* @warning
* @b EXPERIMENTAL:
* All functions in this file may be changed or removed without prior notice.

author
#45 lore raw

RE: [PATCH v5 4/5] vhost: support async dequeue for split ring

Quoted 16 lines

-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Friday, May 13, 2022 10:24 AM
To: Ding, Xuan <[email protected]>
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>
Subject: Re: [PATCH v5 4/5] vhost: support async dequeue for split ring

Quoted 1 line · On Fri, 13 May 2022 02:00:21 +0000 [email protected] wrote:
Quoted 15 lines

Review

Quoted 12 lines
(quoted diff) +12 -0
+/**
+ * This function tries to receive packets from the guest with
+offloading
+ * copies to the async channel. The packets that are transfer
+completed
+ * are returned in "pkts". The other packets that their copies are
+submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies
+are
+ * completed by the async channel.
+ *

Please add EXPERIMENTAL header like this, so it shows up in docmentation correctly.

Get it, thanks for your suggestion.

Regards,
Xuan

Quoted 5 lines

*
* @warning
* @b EXPERIMENTAL:
* All functions in this file may be changed or removed without prior notice.

#46 lore raw

PATCH v5 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 284 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +193 -91

426 lines not on this page · raw message

examples/vhost/main.h +29 -3
index e7f395c3c9..2fcb8376c5 100644
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#47 lore raw

v6 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v5->v6:
* adjust EXPERIMENTAL header

v4->v5:
* rebase to latest DPDK
* add some checks

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 284 ++++++++++-----
 examples/vhost/main.h                  |  32 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  37 ++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 473 ++++++++++++++++++++++---
 10 files changed, 714 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
#48 lore raw

PATCH v6 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
index 5f432b0d77..d4c94d2a9b 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+			sync_fill_seg(dev, vq, m, mbuf_offset,
+				      buf_addr + buf_offset,
+				      buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#49 lore raw

PATCH v6 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
index d4c94d2a9b..a9e2dcd9ce 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+					   buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#50 lore raw

PATCH v6 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..00744b234f 100644
@@ -180,6 +180,7 @@ struct async_inflight_info {
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
+	struct virtio_net_hdr nethdr;
 };
 
 struct vhost_async {
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#51 lore raw

PATCH v6 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  37 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index f287b76ebf..09c1c24b48 100644
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+  accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h +37 -0
index 70234debf9..2789492e38 100644
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  >= 0: The amount of in-flight packets
+ *  -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +1 -1
index 5841315386..8c7211bf0d 100644
@@ -90,7 +90,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_vhost_async_get_inflight_thread_unsafe;
-
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +337 -0

340 lines not on this page · raw message

Signature · 1 line
2.17.1
#52 lore raw

PATCH v6 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 284 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +193 -91

426 lines not on this page · raw message

examples/vhost/main.h +29 -3
index e7f395c3c9..2fcb8376c5 100644
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#53 lore raw

RE: [PATCH v6 5/5] examples/vhost: support async dequeue data path

Quoted 81 lines

-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Friday, May 13, 2022 10:51 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v6 5/5] examples/vhost: support async dequeue data path

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 284 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +0 -0
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's

used in combination with dmas
**--dmas**
This parameter is used to specify the assigned DMA device of a vhost
device.
Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost

doc/guides/sample_app_ug/vhost.rst +4 -2
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in

order,

doc/guides/sample_app_ug/vhost.rst +2 -0
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +6 -0
index c4d46de1c5..d070391727 100644
@@ -63,6 +63,9 @@
 
 #define DMA_RING_SIZE 4096
 
+#define ASYNC_ENQUEUE_VHOST 1
+#define ASYNC_DEQUEUE_VHOST 2
+
 /* number of mbufs in all pools - if specified on command-line. */
 static int total_num_mbufs = NUM_MBUFS_DEFAULT;
 
@@ -116,6 +119,8 @@ static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;
 static char *socket_files;
 static int nb_sockets;
 
+static struct vhost_queue_ops vdev_queue_ops[RTE_MAX_VHOST_DEVICE];
+
 /* empty VMDq configuration structure. Filled in programmatically */
 static struct rte_eth_conf vmdq_conf_default = {
 	.rxmode = {
@@ -205,6 +210,18 @@ struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE *
 RTE_MAX_VHOST_DEVICE];
 #define MBUF_TABLE_DRAIN_TSC	((rte_get_tsc_hz() + US_PER_S - 1) \
 				 / US_PER_S * BURST_TX_DRAIN_US)
 
+static int vid2socketid[RTE_MAX_VHOST_DEVICE];

examples/vhost/main.c struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE * · line 224

Quoted 10 lines
examples/vhost/main.c +10 -0
+
+static uint32_t get_async_flag_by_socketid(int socketid)
+{
+	return dma_bind[socketid].async_flag;
+}
+
+static void init_vid2socketid_array(int vid, int socketid)
+{
+	vid2socketid[vid] = socketid;
+}

Return value and func name should be on separate lines as per coding style. And above func can be inline, same suggestion for short func below, especially ones in data path.

Thanks,
Chenbo

Quoted 544 lines
examples/vhost/main.c +1 -0
+
 static inline bool
 is_dma_configured(int16_t dev_id)

{

examples/vhost/main.c +3 -1
@@ -224,7 +241,7 @@ open_dma(const char *value)
 	char *addrs = input;
 	char *ptrs[2];
 	char *start, *end, *substr;
-	int64_t vid;
+	int64_t socketid, vring_id;
 
 	struct rte_dma_info info;
 	struct rte_dma_conf dev_config = { .nb_vchans = 1 };
@@ -262,7 +279,9 @@ open_dma(const char *value)
 
 	while (i < args_nr) {
 		char *arg_temp = dma_arg[i];
+		char *txd, *rxd;
 		uint8_t sub_nr;
+		int async_flag;
 
 		sub_nr = rte_strsplit(arg_temp, strlen(arg_temp), ptrs, 2,
 '@');
examples/vhost/main.c +15 -5
 		if (sub_nr != 2) {
@@ -270,14 +289,23 @@ open_dma(const char *value)
 			goto out;
 		}
 
-		start = strstr(ptrs[0], "txd");
-		if (start == NULL) {
+		txd = strstr(ptrs[0], "txd");
+		rxd = strstr(ptrs[0], "rxd");
+		if (txd) {
+			start = txd;
+			vring_id = VIRTIO_RXQ;
+			async_flag = ASYNC_ENQUEUE_VHOST;
+		} else if (rxd) {
+			start = rxd;
+			vring_id = VIRTIO_TXQ;
+			async_flag = ASYNC_DEQUEUE_VHOST;
+		} else {
 			ret = -1;
 			goto out;
 		}
 
 		start += 3;
-		vid = strtol(start, &end, 0);
+		socketid = strtol(start, &end, 0);
 		if (end == start) {
 			ret = -1;
 			goto out;
@@ -338,7 +366,8 @@ open_dma(const char *value)
 		dmas_id[dma_count++] = dev_id;
 
 done:
-		(dma_info + vid)->dmas[VIRTIO_RXQ].dev_id = dev_id;
+		(dma_info + socketid)->dmas[vring_id].dev_id = dev_id;
+		(dma_info + socketid)->async_flag |= async_flag;
 		i++;
 	}
 out:
@@ -990,7 +1019,7 @@ complete_async_pkts(struct vhost_dev *vdev)
 {
 	struct rte_mbuf *p_cpl[MAX_PKT_BURST];
 	uint16_t complete_count;
-	int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ].dev_id;
+	int16_t dma_id = dma_bind[vid2socketid[vdev-
 >vid]].dmas[VIRTIO_RXQ].dev_id;
 
 	complete_count = rte_vhost_poll_enqueue_completed(vdev->vid,
examples/vhost/main.c +1 -16
 					VIRTIO_RXQ, p_cpl, MAX_PKT_BURST, dma_id, 0);
@@ -1029,22 +1058,7 @@ drain_vhost(struct vhost_dev *vdev)
 	uint16_t nr_xmit = vhost_txbuff[buff_idx]->len;
 	struct rte_mbuf **m = vhost_txbuff[buff_idx]->m_table;
 
-	if (builtin_net_driver) {
-		ret = vs_enqueue_pkts(vdev, VIRTIO_RXQ, m, nr_xmit);
-	} else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t enqueue_fail = 0;
-		int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ].dev_id;
-
-		complete_async_pkts(vdev);
-		ret = rte_vhost_submit_enqueue_burst(vdev->vid, VIRTIO_RXQ, m,
 nr_xmit, dma_id, 0);
-
-		enqueue_fail = nr_xmit - ret;
-		if (enqueue_fail)
-			free_pkts(&m[ret], nr_xmit - ret);
-	} else {
-		ret = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
-						m, nr_xmit);
-	}
+	ret = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev, VIRTIO_RXQ,
 m, nr_xmit);
 
examples/vhost/main.c +27 -1
 	if (enable_stats) {
 		__atomic_add_fetch(&vdev->stats.rx_total_atomic, nr_xmit,
@@ -1053,7 +1067,7 @@ drain_vhost(struct vhost_dev *vdev)
 				__ATOMIC_SEQ_CST);
 	}
 
-	if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled)
+	if (!dma_bind[vid2socketid[vdev-
 >vid]].dmas[VIRTIO_RXQ].async_enabled)
 		free_pkts(m, nr_xmit);
 }
@@ -1325,6 +1339,32 @@ drain_mbuf_table(struct mbuf_table *tx_q)
 	}
 }
 
+uint16_t
+async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t rx_count)
+{
+	uint16_t enqueue_count;
+	uint16_t enqueue_fail = 0;
+	uint16_t dma_id = dma_bind[vid2socketid[dev-
 >vid]].dmas[VIRTIO_RXQ].dev_id;
+
+	complete_async_pkts(dev);
+	enqueue_count = rte_vhost_submit_enqueue_burst(dev->vid, queue_id,
+					pkts, rx_count, dma_id, 0);
+
+	enqueue_fail = rx_count - enqueue_count;
+	if (enqueue_fail)
+		free_pkts(&pkts[enqueue_count], enqueue_fail);
+
+	return enqueue_count;
+}
+
+uint16_t
+sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t rx_count)
+{
+	return rte_vhost_enqueue_burst(dev->vid, queue_id, pkts, rx_count);
+}
+
 static __rte_always_inline void
 drain_eth_rx(struct vhost_dev *vdev)

{

examples/vhost/main.c +24 -20
@@ -1355,25 +1395,8 @@ drain_eth_rx(struct vhost_dev *vdev)
 		}
 	}
 
-	if (builtin_net_driver) {
-		enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,
-						pkts, rx_count);
-	} else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t enqueue_fail = 0;
-		int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ].dev_id;
-
-		complete_async_pkts(vdev);
-		enqueue_count = rte_vhost_submit_enqueue_burst(vdev->vid,
-					VIRTIO_RXQ, pkts, rx_count, dma_id, 0);
-
-		enqueue_fail = rx_count - enqueue_count;
-		if (enqueue_fail)
-			free_pkts(&pkts[enqueue_count], enqueue_fail);
-
-	} else {
-		enqueue_count = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
-						pkts, rx_count);
-	}
+	enqueue_count = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev,
+					VIRTIO_RXQ, pkts, rx_count);
 
 	if (enable_stats) {
 		__atomic_add_fetch(&vdev->stats.rx_total_atomic, rx_count,
@@ -1382,10 +1405,31 @@ drain_eth_rx(struct vhost_dev *vdev)
 				__ATOMIC_SEQ_CST);
 	}
 
-	if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled)
+	if (!dma_bind[vid2socketid[vdev-
 >vid]].dmas[VIRTIO_RXQ].async_enabled)
 		free_pkts(pkts, rx_count);
 }
 
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			    struct rte_mempool *mbuf_pool,
+			    struct rte_mbuf **pkts, uint16_t count)
+{
+	int nr_inflight;
+	uint16_t dequeue_count;
+	uint16_t dma_id = dma_bind[vid2socketid[dev-
 >vid]].dmas[VIRTIO_TXQ].dev_id;
+
+	dequeue_count = rte_vhost_async_try_dequeue_burst(dev->vid, queue_id,
+			mbuf_pool, pkts, count, &nr_inflight, dma_id, 0);
+
+	return dequeue_count;
+}
+
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			   struct rte_mempool *mbuf_pool,
+			   struct rte_mbuf **pkts, uint16_t count)
+{
+	return rte_vhost_dequeue_burst(dev->vid, queue_id, mbuf_pool, pkts,
 count);
+}
+

static __rte_always_inline void
drain_virtio_tx(struct vhost_dev *vdev)
{

examples/vhost/main.c +18 -7
@@ -1393,13 +1437,8 @@ drain_virtio_tx(struct vhost_dev *vdev)
 	uint16_t count;
 	uint16_t i;
 
-	if (builtin_net_driver) {
-		count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool,
-					pkts, MAX_PKT_BURST);
-	} else {
-		count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ,
-					mbuf_pool, pkts, MAX_PKT_BURST);
-	}
+	count = vdev_queue_ops[vdev->vid].dequeue_pkt_burst(vdev,
+				VIRTIO_TXQ, mbuf_pool, pkts, MAX_PKT_BURST);
 
 	/* setup VMDq for the first packet */
 	if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count) {
@@ -1478,6 +1517,26 @@ switch_worker(void *arg __rte_unused)
 	return 0;
 }
 
+static void
+vhost_clear_queue_thread_unsafe(struct vhost_dev *vdev, uint16_t queue_id)
+{
+	uint16_t n_pkt = 0;
+	int pkts_inflight;
+
+	int16_t dma_id = dma_bind[vid2socketid[vdev-
 >vid]].dmas[queue_id].dev_id;
+	pkts_inflight = rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,
 queue_id);
+
+	struct rte_mbuf *m_cpl[pkts_inflight];
+
+	while (pkts_inflight) {
+		n_pkt = rte_vhost_clear_queue_thread_unsafe(vdev->vid,
 queue_id, m_cpl,
+							pkts_inflight, dma_id, 0);
+		free_pkts(m_cpl, n_pkt);
+		pkts_inflight =

rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,

examples/vhost/main.c +36 -14
+									queue_id);
+	}
+}
+
 /*
  * Remove a device from the specific data core linked list and from the
  * main linked list. Synchronization  occurs through the use of the
@@ -1535,27 +1594,79 @@ destroy_device(int vid)
 		vdev->vid);
 
 	if (dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t n_pkt = 0;
-		int pkts_inflight;
-		int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ].dev_id;
-		pkts_inflight = rte_vhost_async_get_inflight_thread_unsafe(vid,
 VIRTIO_RXQ);
-		struct rte_mbuf *m_cpl[pkts_inflight];
-
-		while (pkts_inflight) {
-			n_pkt = rte_vhost_clear_queue_thread_unsafe(vid,
 VIRTIO_RXQ,
-						m_cpl, pkts_inflight, dma_id, 0);
-			free_pkts(m_cpl, n_pkt);
-			pkts_inflight =
 rte_vhost_async_get_inflight_thread_unsafe(vid,
-										VIRTIO_RXQ);
-		}
-
+		vhost_clear_queue_thread_unsafe(vdev, VIRTIO_RXQ);
 		rte_vhost_async_channel_unregister(vid, VIRTIO_RXQ);
 		dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled = false;
 	}
 
+	if (dma_bind[vid].dmas[VIRTIO_TXQ].async_enabled) {
+		vhost_clear_queue_thread_unsafe(vdev, VIRTIO_TXQ);
+		rte_vhost_async_channel_unregister(vid, VIRTIO_TXQ);
+		dma_bind[vid].dmas[VIRTIO_TXQ].async_enabled = false;
+	}
+
 	rte_free(vdev);
 }
 
+static int
+get_socketid_by_vid(int vid)
+{
+	int i;
+	char ifname[PATH_MAX];
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+
+	for (i = 0; i < nb_sockets; i++) {
+		char *file = socket_files + i * PATH_MAX;
+		if (strcmp(file, ifname) == 0)
+			return i;
+	}
+
+	return -1;
+}
+
+static int
+init_vhost_queue_ops(int vid)
+{
+	if (builtin_net_driver) {
+		vdev_queue_ops[vid].enqueue_pkt_burst = builtin_enqueue_pkts;
+		vdev_queue_ops[vid].dequeue_pkt_burst = builtin_dequeue_pkts;
+	} else {
+		if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled)
+			vdev_queue_ops[vid].enqueue_pkt_burst =

async_enqueue_pkts;

examples/vhost/main.c +2 -0
+		else
+			vdev_queue_ops[vid].enqueue_pkt_burst =

sync_enqueue_pkts;

examples/vhost/main.c +3 -0
+
+		if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled)
+			vdev_queue_ops[vid].dequeue_pkt_burst =

async_dequeue_pkts;

examples/vhost/main.c +2 -0
+		else
+			vdev_queue_ops[vid].dequeue_pkt_burst =

sync_dequeue_pkts;

examples/vhost/main.c +11 -0
+	}
+
+	return 0;
+}
+
+static int
+vhost_async_channel_register(int vid)
+{
+	int rx_ret = 0, tx_ret = 0;
+
+	if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].dev_id !=

INVALID_DMA_ID) {

examples/vhost/main.c +3 -0
+		rx_ret = rte_vhost_async_channel_register(vid, VIRTIO_RXQ);
+		if (rx_ret == 0)
+
	dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled = true;
examples/vhost/main.c +3 -0
+	}
+
+	if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].dev_id !=

INVALID_DMA_ID) {

examples/vhost/main.c +3 -0
+		tx_ret = rte_vhost_async_channel_register(vid, VIRTIO_TXQ);
+		if (tx_ret == 0)
+
	dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled = true;
examples/vhost/main.c +7 -0
+	}
+
+	return rx_ret | tx_ret;
+}
+
+
+
 /*
  * A new device is added to a data core. First the device is added to the

main linked list
* and then allocated to a specific data core.

examples/vhost/main.c +2 -0
@@ -1567,6 +1678,8 @@ new_device(int vid)
 	uint16_t i;
 	uint32_t device_num_min = num_devices;
 	struct vhost_dev *vdev;
+	int ret;
+
 	vdev = rte_zmalloc("vhost device", sizeof(*vdev),
 RTE_CACHE_LINE_SIZE);
 	if (vdev == NULL) {
examples/vhost/main.c +16 -27
 		RTE_LOG(INFO, VHOST_DATA,
@@ -1589,6 +1702,17 @@ new_device(int vid)
 		}
 	}
 
+	int socketid = get_socketid_by_vid(vid);
+	if (socketid == -1)
+		return -1;
+
+	init_vid2socketid_array(vid, socketid);
+
+	ret =  vhost_async_channel_register(vid);
+
+	if (init_vhost_queue_ops(vid) != 0)
+		return -1;
+
 	if (builtin_net_driver)
 		vs_vhost_net_setup(vdev);
 
@@ -1620,16 +1744,7 @@ new_device(int vid)
 		"(%d) device has been added to data core %d\n",
 		vid, vdev->coreid);
 
-	if (dma_bind[vid].dmas[VIRTIO_RXQ].dev_id != INVALID_DMA_ID) {
-		int ret;
-
-		ret = rte_vhost_async_channel_register(vid, VIRTIO_RXQ);
-		if (ret == 0)
-			dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled = true;
-		return ret;
-	}
-
-	return 0;
+	return ret;
 }
 
 static int
@@ -1647,22 +1762,9 @@ vring_state_changed(int vid, uint16_t queue_id, int
 enable)
 	if (queue_id != VIRTIO_RXQ)
 		return 0;
 
-	if (dma_bind[vid].dmas[queue_id].async_enabled) {
-		if (!enable) {
-			uint16_t n_pkt = 0;
-			int pkts_inflight;
-			pkts_inflight =
 rte_vhost_async_get_inflight_thread_unsafe(vid, queue_id);
-			int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ].dev_id;
-			struct rte_mbuf *m_cpl[pkts_inflight];
-
-			while (pkts_inflight) {
-				n_pkt = rte_vhost_clear_queue_thread_unsafe(vid,
 queue_id,
-							m_cpl, pkts_inflight, dma_id, 0);
-				free_pkts(m_cpl, n_pkt);
-				pkts_inflight =
 rte_vhost_async_get_inflight_thread_unsafe(vid,
-
 	queue_id);
-			}
-		}
+	if (dma_bind[vid2socketid[vid]].dmas[queue_id].async_enabled) {
+		if (!enable)
+			vhost_clear_queue_thread_unsafe(vdev, queue_id);
 	}
 
 	return 0;
@@ -1887,7 +1989,7 @@ main(int argc, char *argv[])
 	for (i = 0; i < nb_sockets; i++) {
 		char *file = socket_files + i * PATH_MAX;
 
-		if (dma_count)
+		if (dma_count && get_async_flag_by_socketid(i) != 0)
 			flags = flags | RTE_VHOST_USER_ASYNC_COPY;
 
 		ret = rte_vhost_driver_register(file, flags);
examples/vhost/main.h +29 -3
index e7f395c3c9..2fcb8376c5 100644
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +7 -0
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t
 queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
examples/vhost/virtio_net.c +1 -1
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct
 rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t

count)
{

examples/vhost/virtio_net.c +4 -0
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t
 queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t

count)

examples/vhost/virtio_net.c +3 -1
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
--

2.17.1

author
#54 lore raw

RE: [PATCH v6 5/5] examples/vhost: support async dequeue data path

Quoted 94 lines

-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, May 13, 2022 11:27 AM
To: Ding, Xuan <[email protected]>; [email protected]
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: RE: [PATCH v6 5/5] examples/vhost: support async dequeue data path

Quoted 68 lines

-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Friday, May 13, 2022 10:51 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v6 5/5] examples/vhost: support async dequeue data path

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 284 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +0 -0
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs.

It's used in combination with dmas
**--dmas**
This parameter is used to specify the assigned DMA device of a vhost
device.
Async vhost-user net driver will be used if --dmas is set. For
example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means

doc/guides/sample_app_ug/vhost.rst +4 -0
+use DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue
+operation and use DMA channel 00:04.1/00:04.3 for vhost device 1
+enqueue/dequeue operation. The index of the device corresponds to the
+socket file in

order,

doc/guides/sample_app_ug/vhost.rst +2 -0
+that means vhost device 0 is created through the first socket file,
+vhost device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c index renamed from examples/vhost/main.c +0 -0

c4d46de1c5..d070391727 100644

examples/vhost/main.c +3 -0
@@ -63,6 +63,9 @@
 
 #define DMA_RING_SIZE 4096
 
+#define ASYNC_ENQUEUE_VHOST 1
+#define ASYNC_DEQUEUE_VHOST 2
+
 /* number of mbufs in all pools - if specified on command-line. */
 static int total_num_mbufs = NUM_MBUFS_DEFAULT;
 

examples/vhost/main.c static uint32_t burst_rx_retry_num = · line 121

Quoted 3 lines
examples/vhost/main.c +1 -0
@@ -116,6 +119,8 @@ static uint32_t burst_rx_retry_num =
 BURST_RX_RETRIES;  static char *socket_files;  static int nb_sockets;
 
+static struct vhost_queue_ops

vdev_queue_ops[RTE_MAX_VHOST_DEVICE];

Quoted 9 lines
examples/vhost/main.c +1 -0
+
 /* empty VMDq configuration structure. Filled in programmatically */
 static struct rte_eth_conf vmdq_conf_default = {
 	.rxmode = {
@@ -205,6 +210,18 @@ struct vhost_bufftable
 *vhost_txbuff[RTE_MAX_LCORE * RTE_MAX_VHOST_DEVICE];
 #define MBUF_TABLE_DRAIN_TSC	((rte_get_tsc_hz() + US_PER_S - 1) \
 				 / US_PER_S * BURST_TX_DRAIN_US)
 
Quoted 12 lines

examples/vhost/main.c struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE * RTE_MAX_VHOST_DEVICE]; · line 223

Quoted 8 lines
examples/vhost/main.c +8 -0
+static int vid2socketid[RTE_MAX_VHOST_DEVICE];
+
+static uint32_t get_async_flag_by_socketid(int socketid) {
+	return dma_bind[socketid].async_flag; }
+
+static void init_vid2socketid_array(int vid, int socketid) {
+	vid2socketid[vid] = socketid;
+}

Return value and func name should be on separate lines as per coding style. And above func can be inline, same suggestion for short func below, especially ones in data path.

Thanks Chenbo, will fix it in next version.

Regards,
Xuan

Quoted 568 lines

Thanks,
Chenbo

Quoted 61 lines
examples/vhost/main.c +19 -5
+
 static inline bool
 is_dma_configured(int16_t dev_id)
 {
@@ -224,7 +241,7 @@ open_dma(const char *value)
 	char *addrs = input;
 	char *ptrs[2];
 	char *start, *end, *substr;
-	int64_t vid;
+	int64_t socketid, vring_id;
 
 	struct rte_dma_info info;
 	struct rte_dma_conf dev_config = { .nb_vchans = 1 }; @@ -262,7
+279,9 @@ open_dma(const char *value)
 
 	while (i < args_nr) {
 		char *arg_temp = dma_arg[i];
+		char *txd, *rxd;
 		uint8_t sub_nr;
+		int async_flag;
 
 		sub_nr = rte_strsplit(arg_temp, strlen(arg_temp), ptrs, 2, '@');
 		if (sub_nr != 2) {
@@ -270,14 +289,23 @@ open_dma(const char *value)
 			goto out;
 		}
 
-		start = strstr(ptrs[0], "txd");
-		if (start == NULL) {
+		txd = strstr(ptrs[0], "txd");
+		rxd = strstr(ptrs[0], "rxd");
+		if (txd) {
+			start = txd;
+			vring_id = VIRTIO_RXQ;
+			async_flag = ASYNC_ENQUEUE_VHOST;
+		} else if (rxd) {
+			start = rxd;
+			vring_id = VIRTIO_TXQ;
+			async_flag = ASYNC_DEQUEUE_VHOST;
+		} else {
 			ret = -1;
 			goto out;
 		}
 
 		start += 3;
-		vid = strtol(start, &end, 0);
+		socketid = strtol(start, &end, 0);
 		if (end == start) {
 			ret = -1;
 			goto out;
@@ -338,7 +366,8 @@ open_dma(const char *value)
 		dmas_id[dma_count++] = dev_id;
 
 done:
-		(dma_info + vid)->dmas[VIRTIO_RXQ].dev_id = dev_id;
+		(dma_info + socketid)->dmas[vring_id].dev_id = dev_id;
+		(dma_info + socketid)->async_flag |= async_flag;
 		i++;
 	}
 out:

examples/vhost/main.c complete_async_pkts(struct vhost_dev *vdev) { · line 1025

Quoted 8 lines
examples/vhost/main.c +1 -1
@@ -990,7 +1019,7 @@ complete_async_pkts(struct vhost_dev *vdev)  {
 	struct rte_mbuf *p_cpl[MAX_PKT_BURST];
 	uint16_t complete_count;
-	int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ].dev_id;
+	int16_t dma_id = dma_bind[vid2socketid[vdev-
 >vid]].dmas[VIRTIO_RXQ].dev_id;
 
 	complete_count = rte_vhost_poll_enqueue_completed(vdev->vid,
 					VIRTIO_RXQ, p_cpl, MAX_PKT_BURST,

dma_id, 0); @@ -1029,22

examples/vhost/main.c drain_vhost(struct vhost_dev *vdev)

Quoted 13 lines

+1058,7 @@ drain_vhost(struct vhost_dev *vdev)

examples/vhost/main.c +0 -5
 	uint16_t nr_xmit = vhost_txbuff[buff_idx]->len;
 	struct rte_mbuf **m = vhost_txbuff[buff_idx]->m_table;
 
-	if (builtin_net_driver) {
-		ret = vs_enqueue_pkts(vdev, VIRTIO_RXQ, m, nr_xmit);
-	} else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t enqueue_fail = 0;
-		int16_t dma_id = dma_bind[vdev-

vid].dmas[VIRTIO_RXQ].dev_id;

examples/vhost/main.c +0 -3
-
-		complete_async_pkts(vdev);
-		ret = rte_vhost_submit_enqueue_burst(vdev->vid,

VIRTIO_RXQ, m,

examples/vhost/main.c drain_vhost(struct vhost_dev *vdev)

Quoted 10 lines

nr_xmit, dma_id, 0);

examples/vhost/main.c +1 -8
-
-		enqueue_fail = nr_xmit - ret;
-		if (enqueue_fail)
-			free_pkts(&m[ret], nr_xmit - ret);
-	} else {
-		ret = rte_vhost_enqueue_burst(vdev->vid, VIRTIO_RXQ,
-						m, nr_xmit);
-	}
+	ret = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev,

VIRTIO_RXQ,

examples/vhost/main.c drain_vhost(struct vhost_dev *vdev) · line 1064

Quoted 4 lines

m, nr_xmit);

examples/vhost/main.c +0 -0
 	if (enable_stats) {
 		__atomic_add_fetch(&vdev->stats.rx_total_atomic, nr_xmit,

@@

Quoted 14 lines

-1053,7 +1067,7 @@ drain_vhost(struct vhost_dev *vdev)

examples/vhost/main.c +1 -1
 				__ATOMIC_SEQ_CST);
 	}
 
-	if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled)
+	if (!dma_bind[vid2socketid[vdev-
Quoted 1 line

vid]].dmas[VIRTIO_RXQ].async_enabled)

examples/vhost/main.c +0 -0
 		free_pkts(m, nr_xmit);
 }
@@ -1325,6 +1339,32 @@ drain_mbuf_table(struct mbuf_table *tx_q)
 	}
 }
 

examples/vhost/main.c drain_mbuf_table(struct mbuf_table *tx_q) · line 1351

Quoted 10 lines
examples/vhost/main.c +9 -0
+uint16_t
+async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t rx_count) {
+	uint16_t enqueue_count;
+	uint16_t enqueue_fail = 0;
+	uint16_t dma_id = dma_bind[vid2socketid[dev-
 >vid]].dmas[VIRTIO_RXQ].dev_id;
+
+	complete_async_pkts(dev);
+	enqueue_count = rte_vhost_submit_enqueue_burst(dev->vid,

queue_id,

Quoted 24 lines
examples/vhost/main.c +15 -2
+					pkts, rx_count, dma_id, 0);
+
+	enqueue_fail = rx_count - enqueue_count;
+	if (enqueue_fail)
+		free_pkts(&pkts[enqueue_count], enqueue_fail);
+
+	return enqueue_count;
+}
+
+uint16_t
+sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t rx_count) {
+	return rte_vhost_enqueue_burst(dev->vid, queue_id, pkts, rx_count);
+}
+
 static __rte_always_inline void
 drain_eth_rx(struct vhost_dev *vdev)
 {
@@ -1355,25 +1395,8 @@ drain_eth_rx(struct vhost_dev *vdev)
 		}
 	}
 
-	if (builtin_net_driver) {
-		enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,

examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev) · line 1369

Quoted 10 lines
examples/vhost/main.c +0 -8
-						pkts, rx_count);
-	} else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t enqueue_fail = 0;
-		int16_t dma_id = dma_bind[vdev-
 vid].dmas[VIRTIO_RXQ].dev_id;
-
-		complete_async_pkts(vdev);
-		enqueue_count = rte_vhost_submit_enqueue_burst(vdev-
 vid,
-					VIRTIO_RXQ, pkts, rx_count, dma_id,

0);

examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev)

Quoted 7 lines
examples/vhost/main.c +0 -7
-
-		enqueue_fail = rx_count - enqueue_count;
-		if (enqueue_fail)
-			free_pkts(&pkts[enqueue_count], enqueue_fail);
-
-	} else {
-		enqueue_count = rte_vhost_enqueue_burst(vdev->vid,

VIRTIO_RXQ,

examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev) · line 1402

Quoted 8 lines
examples/vhost/main.c +2 -2
-						pkts, rx_count);
-	}
+	enqueue_count = vdev_queue_ops[vdev-
 vid].enqueue_pkt_burst(vdev,
+					VIRTIO_RXQ, pkts, rx_count);
examples/vhost/main.c +0 -0
 	if (enable_stats) {
 		__atomic_add_fetch(&vdev->stats.rx_total_atomic, rx_count,

@@

Quoted 17 lines

-1382,10 +1405,31 @@ drain_eth_rx(struct vhost_dev *vdev)

examples/vhost/main.c +1 -1
 				__ATOMIC_SEQ_CST);
 	}
 
-	if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ].async_enabled)
+	if (!dma_bind[vid2socketid[vdev-
Quoted 1 line

vid]].dmas[VIRTIO_RXQ].async_enabled)

examples/vhost/main.c +6 -0
 		free_pkts(pkts, rx_count);
 }
 
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			    struct rte_mempool *mbuf_pool,
+			    struct rte_mbuf **pkts, uint16_t count) {
+	int nr_inflight;
+	uint16_t dequeue_count;
+	uint16_t dma_id = dma_bind[vid2socketid[dev-
Quoted 1 line

vid]].dmas[VIRTIO_TXQ].dev_id;

examples/vhost/main.c

Quoted 2 lines
examples/vhost/main.c +2 -0
+
+	dequeue_count = rte_vhost_async_try_dequeue_burst(dev->vid,

queue_id,

examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev)

Quoted 9 lines
examples/vhost/main.c +9 -0
+			mbuf_pool, pkts, count, &nr_inflight, dma_id, 0);
+
+	return dequeue_count;
+}
+
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			   struct rte_mempool *mbuf_pool,
+			   struct rte_mbuf **pkts, uint16_t count) {
+	return rte_vhost_dequeue_burst(dev->vid, queue_id, mbuf_pool,

pkts,

Quoted 8 lines

count);

examples/vhost/main.c +2 -0
+}
+
 static __rte_always_inline void
 drain_virtio_tx(struct vhost_dev *vdev)  { @@ -1393,13 +1437,8 @@

drain_virtio_tx(struct vhost_dev *vdev)

examples/vhost/main.c +0 -0
 	uint16_t count;
 	uint16_t i;

examples/vhost/main.c drain_virtio_tx(struct vhost_dev *vdev)

Quoted 10 lines
examples/vhost/main.c +2 -7
 
-	if (builtin_net_driver) {
-		count = vs_dequeue_pkts(vdev, VIRTIO_TXQ, mbuf_pool,
-					pkts, MAX_PKT_BURST);
-	} else {
-		count = rte_vhost_dequeue_burst(vdev->vid, VIRTIO_TXQ,
-					mbuf_pool, pkts, MAX_PKT_BURST);
-	}
+	count = vdev_queue_ops[vdev->vid].dequeue_pkt_burst(vdev,
+				VIRTIO_TXQ, mbuf_pool, pkts,

MAX_PKT_BURST);

examples/vhost/main.c

Quoted 3 lines
examples/vhost/main.c +0 -0
 	/* setup VMDq for the first packet */
 	if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && count)

{ @@

Quoted 21 lines

-1478,6 +1517,26 @@ switch_worker(void *arg __rte_unused)

examples/vhost/main.c +7 -0
 	return 0;
 }
 
+static void
+vhost_clear_queue_thread_unsafe(struct vhost_dev *vdev, uint16_t
+queue_id) {
+	uint16_t n_pkt = 0;
+	int pkts_inflight;
+
+	int16_t dma_id = dma_bind[vid2socketid[vdev-
Quoted 1 line

vid]].dmas[queue_id].dev_id;

examples/vhost/main.c +2 -0
+	pkts_inflight =
+rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,

queue_id);

examples/vhost/main.c +5 -0
+
+	struct rte_mbuf *m_cpl[pkts_inflight];
+
+	while (pkts_inflight) {
+		n_pkt = rte_vhost_clear_queue_thread_unsafe(vdev->vid,

queue_id, m_cpl,

examples/vhost/main.c

Quoted 1 line
examples/vhost/main.c +1 -0
+							pkts_inflight, dma_id,

0);

Quoted 4 lines
examples/vhost/main.c +2 -0
+		free_pkts(m_cpl, n_pkt);
+		pkts_inflight =

rte_vhost_async_get_inflight_thread_unsafe(vdev->vid, +

	queue_id);
Quoted 7 lines
examples/vhost/main.c +3 -0
+	}
+}
+
 /*
  * Remove a device from the specific data core linked list and from the
  * main linked list. Synchronization  occurs through the use of the

examples/vhost/main.c destroy_device(int vid) · line 1541

Quoted 7 lines
examples/vhost/main.c +0 -4
@@ -1535,27 +1594,79 @@ destroy_device(int vid)
 		vdev->vid);
 
 	if (dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled) {
-		uint16_t n_pkt = 0;
-		int pkts_inflight;
-		int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ].dev_id;
-		pkts_inflight =

rte_vhost_async_get_inflight_thread_unsafe(vid,

examples/vhost/main.c destroy_device(int vid) · line 1548

Quoted 7 lines
examples/vhost/main.c +0 -5
 VIRTIO_RXQ);
-		struct rte_mbuf *m_cpl[pkts_inflight];
-
-		while (pkts_inflight) {
-			n_pkt = rte_vhost_clear_queue_thread_unsafe(vid,
 VIRTIO_RXQ,
-						m_cpl, pkts_inflight, dma_id,

0);

examples/vhost/main.c destroy_device(int vid) · line 1551

Quoted 4 lines
examples/vhost/main.c +0 -3
-			free_pkts(m_cpl, n_pkt);
-			pkts_inflight =
 rte_vhost_async_get_inflight_thread_unsafe(vid,
-
	VIRTIO_RXQ);
Quoted 27 lines
examples/vhost/main.c +18 -2
-		}
-
+		vhost_clear_queue_thread_unsafe(vdev, VIRTIO_RXQ);
 		rte_vhost_async_channel_unregister(vid, VIRTIO_RXQ);
 		dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled = false;
 	}
 
+	if (dma_bind[vid].dmas[VIRTIO_TXQ].async_enabled) {
+		vhost_clear_queue_thread_unsafe(vdev, VIRTIO_TXQ);
+		rte_vhost_async_channel_unregister(vid, VIRTIO_TXQ);
+		dma_bind[vid].dmas[VIRTIO_TXQ].async_enabled = false;
+	}
+
 	rte_free(vdev);
 }
 
+static int
+get_socketid_by_vid(int vid)
+{
+	int i;
+	char ifname[PATH_MAX];
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+
+	for (i = 0; i < nb_sockets; i++) {
+		char *file = socket_files + i * PATH_MAX;
+		if (strcmp(file, ifname) == 0)
+			return i;

examples/vhost/main.c destroy_device(int vid)

Quoted 10 lines
examples/vhost/main.c +10 -0
+	}
+
+	return -1;
+}
+
+static int
+init_vhost_queue_ops(int vid)
+{
+	if (builtin_net_driver) {
+		vdev_queue_ops[vid].enqueue_pkt_burst =

builtin_enqueue_pkts;

examples/vhost/main.c destroy_device(int vid) · line 1635

Quoted 1 line
examples/vhost/main.c +1 -0
+		vdev_queue_ops[vid].dequeue_pkt_burst =

builtin_dequeue_pkts;

examples/vhost/main.c destroy_device(int vid) · line 1637

Quoted 2 lines
examples/vhost/main.c +2 -0
+	} else {
+		if

(dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled)

examples/vhost/main.c

Quoted 7 lines
examples/vhost/main.c +1 -0
+			vdev_queue_ops[vid].enqueue_pkt_burst =

async_enqueue_pkts;

examples/vhost/main.c +2 -0
+		else
+			vdev_queue_ops[vid].enqueue_pkt_burst =

sync_enqueue_pkts;

examples/vhost/main.c +2 -0
+
+		if

(dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled)

Quoted 19 lines
examples/vhost/main.c +1 -0
+			vdev_queue_ops[vid].dequeue_pkt_burst =

async_dequeue_pkts;

examples/vhost/main.c +2 -0
+		else
+			vdev_queue_ops[vid].dequeue_pkt_burst =

sync_dequeue_pkts;

examples/vhost/main.c +10 -0
+	}
+
+	return 0;
+}
+
+static int
+vhost_async_channel_register(int vid) {
+	int rx_ret = 0, tx_ret = 0;
+
+	if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].dev_id !=

INVALID_DMA_ID) {

examples/vhost/main.c +3 -0
+		rx_ret = rte_vhost_async_channel_register(vid, VIRTIO_RXQ);
+		if (rx_ret == 0)
+
Quoted 1 line

dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled =

true;

Quoted 8 lines
examples/vhost/main.c +3 -0
+	}
+
+	if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].dev_id !=

INVALID_DMA_ID) {

examples/vhost/main.c +3 -0
+		tx_ret = rte_vhost_async_channel_register(vid, VIRTIO_TXQ);
+		if (tx_ret == 0)
+

dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled =

true;

Quoted 40 lines
examples/vhost/main.c +7 -0
+	}
+
+	return rx_ret | tx_ret;
+}
+
+
+
 /*
  * A new device is added to a data core. First the device is added to

the main linked list
* and then allocated to a specific data core.

examples/vhost/main.c +2 -0
@@ -1567,6 +1678,8 @@ new_device(int vid)
 	uint16_t i;
 	uint32_t device_num_min = num_devices;
 	struct vhost_dev *vdev;
+	int ret;
+
 	vdev = rte_zmalloc("vhost device", sizeof(*vdev),
 RTE_CACHE_LINE_SIZE);
 	if (vdev == NULL) {
examples/vhost/main.c +11 -0
 		RTE_LOG(INFO, VHOST_DATA,
@@ -1589,6 +1702,17 @@ new_device(int vid)
 		}
 	}
 
+	int socketid = get_socketid_by_vid(vid);
+	if (socketid == -1)
+		return -1;
+
+	init_vid2socketid_array(vid, socketid);
+
+	ret =  vhost_async_channel_register(vid);
+
+	if (init_vhost_queue_ops(vid) != 0)
+		return -1;
+
 	if (builtin_net_driver)
 		vs_vhost_net_setup(vdev);
 

examples/vhost/main.c new_device(int vid) · line 1628

Quoted 9 lines
examples/vhost/main.c +0 -6
@@ -1620,16 +1744,7 @@ new_device(int vid)
 		"(%d) device has been added to data core %d\n",
 		vid, vdev->coreid);
 
-	if (dma_bind[vid].dmas[VIRTIO_RXQ].dev_id != INVALID_DMA_ID) {
-		int ret;
-
-		ret = rte_vhost_async_channel_register(vid, VIRTIO_RXQ);
-		if (ret == 0)
-			dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled =

true;

Quoted 20 lines
examples/vhost/main.c +1 -4
-		return ret;
-	}
-
-	return 0;
+	return ret;
 }
 
 static int
@@ -1647,22 +1762,9 @@ vring_state_changed(int vid, uint16_t queue_id,

int
enable)

examples/vhost/main.c +0 -5
 	if (queue_id != VIRTIO_RXQ)
 		return 0;
 
-	if (dma_bind[vid].dmas[queue_id].async_enabled) {
-		if (!enable) {
-			uint16_t n_pkt = 0;
-			int pkts_inflight;
-			pkts_inflight =

rte_vhost_async_get_inflight_thread_unsafe(vid, queue_id);

examples/vhost/main.c

Quoted 1 line
examples/vhost/main.c +0 -1
-			int16_t dma_id =

dma_bind[vid].dmas[VIRTIO_RXQ].dev_id;

examples/vhost/main.c destroy_device(int vid)

Quoted 4 lines
examples/vhost/main.c +0 -4
-			struct rte_mbuf *m_cpl[pkts_inflight];
-
-			while (pkts_inflight) {
-				n_pkt =

rte_vhost_clear_queue_thread_unsafe(vid,

examples/vhost/main.c

Quoted 2 lines

queue_id,

examples/vhost/main.c +0 -1
-							m_cpl, pkts_inflight,

dma_id, 0);

Quoted 117 lines
examples/vhost/main.c +0 -2
-				free_pkts(m_cpl, n_pkt);
-				pkts_inflight =

rte_vhost_async_get_inflight_thread_unsafe(vid,
-

	queue_id);
examples/vhost/main.c +4 -3
-			}
-		}
+	if (dma_bind[vid2socketid[vid]].dmas[queue_id].async_enabled) {
+		if (!enable)
+			vhost_clear_queue_thread_unsafe(vdev, queue_id);
 	}
 
 	return 0;
@@ -1887,7 +1989,7 @@ main(int argc, char *argv[])
 	for (i = 0; i < nb_sockets; i++) {
 		char *file = socket_files + i * PATH_MAX;
 
-		if (dma_count)
+		if (dma_count && get_async_flag_by_socketid(i) != 0)
 			flags = flags | RTE_VHOST_USER_ASYNC_COPY;
 
 		ret = rte_vhost_driver_register(file, flags); diff --git

a/examples/vhost/main.h b/examples/vhost/main.h index e7f395c3c9..2fcb8376c5 100644

examples/vhost/main.h +13 -0
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];  }
 __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst; };
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */ @@ -97,7 +111,19 @@

void vs_vhost_net_remove(struct vhost_dev *dev); uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,

examples/vhost/main.h +15 -3
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count); uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count); uint16_t
+sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count); uint16_t
+sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count); uint16_t
+async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count); uint16_t
+async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +5 -0
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t
 queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count) {
+	return vs_enqueue_pkts(dev, queue_id, pkts, count); }
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx, @@ -363,7 +370,7 @@

dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,

examples/vhost/virtio_net.c +1 -1
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t

count)
{

examples/vhost/virtio_net.c +4 -0
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t
 queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t

count)

examples/vhost/virtio_net.c +2 -1
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count); }
--

2.17.1

#55 lore raw

v7 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v6->v7:
* correct code formatting
* change some functions to inline

v5->v6:
* adjust EXPERIMENTAL header

v4->v5:
* rebase to latest DPDK
* add some checks

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 286 ++++++++++-----
 examples/vhost/main.h                  |  32 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  37 ++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 473 ++++++++++++++++++++++---
 10 files changed, 716 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
#56 lore raw

PATCH v7 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
index 5f432b0d77..d4c94d2a9b 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+			sync_fill_seg(dev, vq, m, mbuf_offset,
+				      buf_addr + buf_offset,
+				      buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#57 lore raw

PATCH v7 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
index d4c94d2a9b..a9e2dcd9ce 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+					   buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#58 lore raw

PATCH v7 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..00744b234f 100644
@@ -180,6 +180,7 @@ struct async_inflight_info {
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
+	struct virtio_net_hdr nethdr;
 };
 
 struct vhost_async {
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#59 lore raw

PATCH v7 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  37 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
index f287b76ebf..09c1c24b48 100644
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
+
+  Receives (dequeues) ``count`` packets from guest to host in async data path,
+  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+  accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h +37 -0
index 70234debf9..2789492e38 100644
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  >= 0: The amount of in-flight packets
+ *  -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +1 -1
index 5841315386..8c7211bf0d 100644
@@ -90,7 +90,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_vhost_async_get_inflight_thread_unsafe;
-
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +337 -0

340 lines not on this page · raw message

Signature · 1 line
2.17.1
#60 lore raw

RE: [PATCH v7 4/5] vhost: support async dequeue for split ring

Hi Xuan,

Quoted 38 lines

-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Monday, May 16, 2022 10:43 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v7 4/5] vhost: support async dequeue for split ring

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  37 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +0 -0
index f287b76ebf..09c1c24b48 100644
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API

functions:
Clear inflight packets which are submitted to DMA engine in vhost async
data
path. Completed packets are returned to applications through ``pkts``.

doc/guides/prog_guide/vhost_lib.rst The following is an overview of some key Vhost API functions: · line 287

Quoted 3 lines
doc/guides/prog_guide/vhost_lib.rst +3 -0
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``

In dmadev library, dma_id is int16_t, rather than uint16_t.

Quoted 32 lines
doc/guides/prog_guide/vhost_lib.rst +4 -0
+
+  Receives (dequeues) ``count`` packets from guest to host in async
+ data path,  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+ accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h index renamed from lib/vhost/rte_vhost_async.h +0 -0

70234debf9..2789492e38 100644

lib/vhost/rte_vhost_async.h +0 -0
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int

vid, uint16_t queue_id, __rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);

lib/vhost/rte_vhost_async.h uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id, · line 212

Quoted 8 lines
lib/vhost/rte_vhost_async.h +8 -0
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior
+notice
+ *
+ * This function tries to receive packets from the guest with
+offloading
+ * copies to the async channel. The packets that are transfer completed

In rte_vhost_async.h, DMA vChannel is referred as async channel or async copy engine. But I think it's better to replace all with DMA vChannel to be consistent with DPDK.

Quoted 9 lines

+ * are returned in "pkts". The other packets that their copies are

lib/vhost/rte_vhost_async.h +8 -0
+submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies
+are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data

lib/vhost/rte_vhost_async.h uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id, · line 229

Quoted 10 lines
lib/vhost/rte_vhost_async.h +10 -0
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  >= 0: The amount of in-flight packets

Better to add more descriptions about the meaning of "nr_inflight".

Thanks,
Jiayu

author
#61 lore raw

RE: [PATCH v7 4/5] vhost: support async dequeue for split ring

Hi Jiayu,

Quoted 51 lines

-----Original Message-----
From: Hu, Jiayu <[email protected]>
Sent: Monday, May 16, 2022 1:53 PM
To: Ding, Xuan <[email protected]>; [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>
Subject: RE: [PATCH v7 4/5] vhost: support async dequeue for split ring

Hi Xuan,

Quoted 39 lines

-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Monday, May 16, 2022 10:43 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v7 4/5] vhost: support async dequeue for split ring

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   7 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  37 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +0 -0
index f287b76ebf..09c1c24b48 100644
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost

API
functions:
Clear inflight packets which are submitted to DMA engine in vhost
async data
path. Completed packets are returned to applications through ``pkts``.

Quoted 5 lines

doc/guides/prog_guide/vhost_lib.rst

Quoted 3 lines
doc/guides/prog_guide/vhost_lib.rst +3 -0
+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+  struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t
+count, int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``

In dmadev library, dma_id is int16_t, rather than uint16_t.

Thanks for the reminder. I will fix it in v8.

Quoted 35 lines

lib/vhost/rte_vhost_async.h uint16_t

Quoted 31 lines
doc/guides/prog_guide/vhost_lib.rst +4 -0
+
+  Receives (dequeues) ``count`` packets from guest to host in async
+ data path,  and stored them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+ accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h +0 -0

index
70234debf9..2789492e38 100644

lib/vhost/rte_vhost_async.h +0 -0
@@ -204,6 +204,43 @@ uint16_t

rte_vhost_clear_queue_thread_unsafe(int

Quoted 3 lines

vid, uint16_t queue_id, __rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);

Quoted 13 lines

lib/vhost/rte_vhost_async.h

Quoted 9 lines
lib/vhost/rte_vhost_async.h +9 -0
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior
+notice
+ *
+ * This function tries to receive packets from the guest with
+offloading
+ * copies to the async channel. The packets that are transfer
+completed

In rte_vhost_async.h, DMA vChannel is referred as async channel or async copy engine. But I think it's better to replace all with DMA vChannel to be consistent with DPDK.

Thanks for your suggestion, I will refine the API description in next version.

Quoted 14 lines
Quoted 14 lines

+ * are returned in "pkts". The other packets that their copies are

lib/vhost/rte_vhost_async.h +12 -0
+submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies
+are
+ * completed by the async channel.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
Quoted 8 lines
Quoted 6 lines

+ * @param pkts
+ * Blank array to keep successfully dequeued packets
+ * @param count
+ * Size of the packet array
+ * @param nr_inflight
+ * >= 0: The amount of in-flight packets

Better to add more descriptions about the meaning of "nr_inflight".

Same as above.

Regards,
Xuan

Quoted 3 lines

Thanks,
Jiayu

#62 lore raw

PATCH v7 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 286 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 245 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +195 -91

428 lines not on this page · raw message

examples/vhost/main.h +29 -3
index e7f395c3c9..2fcb8376c5 100644
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#63 lore raw

v8 cover vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v7->v8:
* change dma_id to int16_t
* refine API documentation

v6->v7:
* correct code formatting
* change some functions to inline

v5->v6:
* adjust EXPERIMENTAL header

v4->v5:
* rebase to latest DPDK
* add some checks

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

 doc/guides/prog_guide/vhost_lib.rst    |   6 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 doc/guides/sample_app_ug/vhost.rst     |   9 +-
 examples/vhost/main.c                  | 286 ++++++++++-----
 examples/vhost/main.h                  |  32 +-
 examples/vhost/virtio_net.c            |  16 +-
 lib/vhost/rte_vhost_async.h            |  37 ++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/vhost.h                      |   1 +
 lib/vhost/virtio_net.c                 | 473 ++++++++++++++++++++++---
 10 files changed, 715 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
#64 lore raw

PATCH v8 1/5 vhost: prepare sync for descriptor to mbuf refactoring

From: Xuan Ding <[email protected]>

This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 78 ++++++++++++++++++++----------------------
 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
index 5f432b0d77..d4c94d2a9b 100644
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 }
 
 static __rte_always_inline void
-sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+sync_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_addr, uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
 
 	if (likely(cpy_len > MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) {
-		rte_memcpy((void *)((uintptr_t)(buf_addr)),
+		if (to_desc) {
+			rte_memcpy((void *)((uintptr_t)(buf_addr)),
 				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
 				cpy_len);
+		} else {
+			rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset),
+				(void *)((uintptr_t)(buf_addr)),
+				cpy_len);
+		}
 		vhost_log_cache_write_iova(dev, vq, buf_iova, cpy_len);
 		PRINT_PACKET(dev, (uintptr_t)(buf_addr), cpy_len, 0);
 	} else {
-		batch_copy[vq->batch_copy_nb_elems].dst =
-			(void *)((uintptr_t)(buf_addr));
-		batch_copy[vq->batch_copy_nb_elems].src =
-			rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		if (to_desc) {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				(void *)((uintptr_t)(buf_addr));
+			batch_copy[vq->batch_copy_nb_elems].src =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+		} else {
+			batch_copy[vq->batch_copy_nb_elems].dst =
+				rte_pktmbuf_mtod_offset(m, void *, mbuf_offset);
+			batch_copy[vq->batch_copy_nb_elems].src =
+				(void *)((uintptr_t)(buf_addr));
+		}
 		batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova;
 		batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
 		vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 						buf_iova + buf_offset, cpy_len) < 0)
 				goto error;
 		} else {
-			sync_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-					buf_addr + buf_offset,
-					buf_iova + buf_offset, cpy_len);
+			sync_fill_seg(dev, vq, m, mbuf_offset,
+				      buf_addr + buf_offset,
+				      buf_iova + buf_offset, cpy_len, true);
 		}
 
 		mbuf_avail  -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		  struct rte_mbuf *m, struct rte_mempool *mbuf_pool,
 		  bool legacy_ol_flags)
 {
-	uint32_t buf_avail, buf_offset;
-	uint64_t buf_addr, buf_len;
+	uint32_t buf_avail, buf_offset, buf_len;
+	uint64_t buf_addr, buf_iova;
 	uint32_t mbuf_avail, mbuf_offset;
 	uint32_t cpy_len;
 	struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	struct virtio_net_hdr *hdr = NULL;
 	/* A counter to avoid desc dead loop chain */
 	uint16_t vec_idx = 0;
-	struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
-	int error = 0;
 
 	buf_addr = buf_vec[vec_idx].buf_addr;
+	buf_iova = buf_vec[vec_idx].buf_iova;
 	buf_len = buf_vec[vec_idx].buf_len;
 
-	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) {
-		error = -1;
-		goto out;
-	}
+	if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1))
+		return -1;
 
 	if (virtio_net_with_host_offload(dev)) {
 		if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		buf_offset = dev->vhost_hlen - buf_len;
 		vec_idx++;
 		buf_addr = buf_vec[vec_idx].buf_addr;
+		buf_iova = buf_vec[vec_idx].buf_iova;
 		buf_len = buf_vec[vec_idx].buf_len;
 		buf_avail  = buf_len - buf_offset;
 	} else if (buf_len == dev->vhost_hlen) {
 		if (unlikely(++vec_idx >= nr_vec))
-			goto out;
+			goto error;
 		buf_addr = buf_vec[vec_idx].buf_addr;
 		buf_len = buf_vec[vec_idx].buf_len;
 
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	while (1) {
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
-		if (likely(cpy_len > MAX_BATCH_LEN ||
-					vq->batch_copy_nb_elems >= vq->size ||
-					(hdr && cur == m))) {
-			rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset),
-					(void *)((uintptr_t)(buf_addr +
-							buf_offset)), cpy_len);
-		} else {
-			batch_copy[vq->batch_copy_nb_elems].dst =
-				rte_pktmbuf_mtod_offset(cur, void *,
-						mbuf_offset);
-			batch_copy[vq->batch_copy_nb_elems].src =
-				(void *)((uintptr_t)(buf_addr + buf_offset));
-			batch_copy[vq->batch_copy_nb_elems].len = cpy_len;
-			vq->batch_copy_nb_elems++;
-		}
+		sync_fill_seg(dev, vq, cur, mbuf_offset,
+			      buf_addr + buf_offset,
+			      buf_iova + buf_offset, cpy_len, false);
 
 		mbuf_avail  -= cpy_len;
 		mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 				break;
 
 			buf_addr = buf_vec[vec_idx].buf_addr;
+			buf_iova = buf_vec[vec_idx].buf_iova;
 			buf_len = buf_vec[vec_idx].buf_len;
 
 			buf_offset = 0;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			if (unlikely(cur == NULL)) {
 				VHOST_LOG_DATA(ERR, "(%s) failed to allocate memory for mbuf.\n",
 						dev->ifname);
-				error = -1;
-				goto out;
+				goto error;
 			}
 
 			prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (hdr)
 		vhost_dequeue_offload(dev, hdr, m, legacy_ol_flags);
 
-out:
-
-	return error;
+	return 0;
+error:
+	return -1;
 }
 
 static void
Signature · 1 line
2.17.1
#65 lore raw

PATCH v8 2/5 vhost: prepare async for descriptor to mbuf refactoring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.

---
 lib/vhost/virtio_net.c | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
index d4c94d2a9b..a9e2dcd9ce 100644
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async)
 }
 
 static __rte_always_inline int
-async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
+async_fill_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		struct rte_mbuf *m, uint32_t mbuf_offset,
-		uint64_t buf_iova, uint32_t cpy_len)
+		uint64_t buf_iova, uint32_t cpy_len, bool to_desc)
 {
 	struct vhost_async *async = vq->async;
 	uint64_t mapped_len;
 	uint32_t buf_offset = 0;
+	void *src, *dst;
 	void *host_iova;
 
 	while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,
 			return -1;
 		}
 
-		if (unlikely(async_iter_add_iovec(dev, async,
-						(void *)(uintptr_t)rte_pktmbuf_iova_offset(m,
-							mbuf_offset),
-						host_iova, (size_t)mapped_len)))
+		if (to_desc) {
+			src = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+			dst = host_iova;
+		} else {
+			src = host_iova;
+			dst = (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, mbuf_offset);
+		}
+
+		if (unlikely(async_iter_add_iovec(dev, async, src, dst, (size_t)mapped_len)))
 			return -1;
 
 		cpy_len -= (uint32_t)mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq,
 		cpy_len = RTE_MIN(buf_avail, mbuf_avail);
 
 		if (is_async) {
-			if (async_mbuf_to_desc_seg(dev, vq, m, mbuf_offset,
-						buf_iova + buf_offset, cpy_len) < 0)
+			if (async_fill_seg(dev, vq, m, mbuf_offset,
+					   buf_iova + buf_offset, cpy_len, true) < 0)
 				goto error;
 		} else {
 			sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
#66 lore raw

PATCH v8 3/5 vhost: merge sync and async descriptor to mbuf filling

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.

---
 lib/vhost/vhost.h      |  1 +
 lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
index a9edc271aa..00744b234f 100644
@@ -180,6 +180,7 @@ struct async_inflight_info {
 	struct rte_mbuf *mbuf;
 	uint16_t descs; /* num of descs inflight */
 	uint16_t nr_buffers; /* num of buffers inflight for packed ring */
+	struct virtio_net_hdr nethdr;
 };
 
 struct vhost_async {
lib/vhost/virtio_net.c +37 -11

104 lines not on this page · raw message

Signature · 1 line
2.17.1
#67 lore raw

PATCH v8 4/5 vhost: support async dequeue for split ring

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.

---
 doc/guides/prog_guide/vhost_lib.rst    |   6 +
 doc/guides/rel_notes/release_22_07.rst |   5 +
 lib/vhost/rte_vhost_async.h            |  37 +++
 lib/vhost/version.map                  |   2 +-
 lib/vhost/virtio_net.c                 | 337 +++++++++++++++++++++++++
 5 files changed, 386 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +6 -0
index f287b76ebf..98f4509d1a 100644
@@ -282,6 +282,12 @@ The following is an overview of some key Vhost API functions:
   Clear inflight packets which are submitted to DMA engine in vhost async data
   path. Completed packets are returned to applications through ``pkts``.
 
+* ``rte_vhost_async_try_dequeue_burst(vid, queue_id, mbuf_pool, pkts, count,
+  nr_inflight, dma_id, vchan_id)``
+
+  Receive ``count`` packets from guest to host in async data path,
+  and store them at ``pkts``.
+
 Vhost-user Implementations
 --------------------------
 
doc/guides/rel_notes/release_22_07.rst +5 -0
index 88b1e478d4..564d88623e 100644
@@ -70,6 +70,11 @@ New Features
   Added an API which can get the number of inflight packets in
   vhost async data path without using lock.
 
+* **Added vhost async dequeue API to receive pkts from guest.**
+
+  Added vhost async dequeue API which can leverage DMA devices to
+  accelerate receiving pkts from guest.
+
 Removed Items
 -------------
 
lib/vhost/rte_vhost_async.h +37 -0
index 70234debf9..a1e7f674ed 100644
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,
 __rte_experimental
 int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
+ *
+ * This function tries to receive packets from the guest with offloading
+ * copies to the DMA vChannels. Successfully dequeued packets are returned
+ * in "pkts". The other packets that their copies are submitted to
+ * the DMA vChannels but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the DMA vChannels.
+ *
+ * @param vid
+ *  ID of vhost device to dequeue data
+ * @param queue_id
+ *  ID of virtqueue to dequeue data
+ * @param mbuf_pool
+ *  Mbuf_pool where host mbuf is allocated
+ * @param pkts
+ *  Blank array to keep successfully dequeued packets
+ * @param count
+ *  Size of the packet array
+ * @param nr_inflight
+ *  >= 0: The amount of in-flight packets
+ *  -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id
+ * @param dma_id
+ *  The identifier of DMA device
+ * @param vchan_id
+ *  The identifier of virtual DMA channel
+ * @return
+ *  Number of successfully dequeued packets
+ */
+__rte_experimental
+uint16_t
+rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+	int *nr_inflight, int16_t dma_id, uint16_t vchan_id);
+
 #ifdef __cplusplus
 }
 #endif
lib/vhost/version.map +1 -1
index 5841315386..8c7211bf0d 100644
@@ -90,7 +90,7 @@ EXPERIMENTAL {
 
 	# added in 22.07
 	rte_vhost_async_get_inflight_thread_unsafe;
-
+	rte_vhost_async_try_dequeue_burst;
 };
 
 INTERNAL {
lib/vhost/virtio_net.c +337 -0

340 lines not on this page · raw message

Signature · 1 line
2.17.1
#68 lore raw

Re: [PATCH v8 4/5] vhost: support async dequeue for split ring

Quoted 5 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
lib/vhost/virtio_net.c +5 -0
+static __rte_always_inline uint16_t
+virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
+               struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+               int16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags)
+{

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+       static bool allocerr_warned;
+       bool dropped = false;
+       uint16_t free_entries;
+       uint16_t pkt_idx, slot_idx = 0;
+       uint16_t nr_done_pkts = 0;
+       uint16_t pkt_err = 0;
+       uint16_t n_xfer;
+       struct vhost_async *async = vq->async;
+       struct async_inflight_info *pkts_info = async->pkts_info;
+       struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST];

Why do we need this array?
Plus, see blow.

Quoted 11 lines
lib/vhost/virtio_net.c +11 -0
+       uint16_t pkts_size = count;
+
+       /**
+        * The ordering between avail index and
+        * desc reads needs to be enforced.
+        */
+       free_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) -
+                       vq->last_avail_idx;
+       if (free_entries == 0)
+               goto out;
+

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3252

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+       rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+
+       async_iter_reset(async);
+
+       count = RTE_MIN(count, MAX_PKT_BURST);
+       count = RTE_MIN(count, free_entries);
+       VHOST_LOG_DATA(DEBUG, "(%s) about to dequeue %u buffers\n",
+                       dev->ifname, count);
+
+       if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts_prealloc, count))

'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than MAX_PKT_BURST.

Calling rte_pktmbuf_alloc_bulk on a fixed-size array pkts_prealloc, allocated on the stack, it may cause a stack overflow.

This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.
I point here another bug.
There are probably more...

<grmbl>
I don't like how async code has been added in the vhost library by Intel.

Maxime did a cleanup on the enqueue patch
https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&archive=both.
I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>

Signature · 1 line
David Marchand
#69 lore raw

Re: [PATCH v8 4/5] vhost: support async dequeue for split ring

Quoted 21 lines · David Marchand
Quoted 5 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
lib/vhost/virtio_net.c +5 -0
+static __rte_always_inline uint16_t
+virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue *vq,
+               struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count,
+               int16_t dma_id, uint16_t vchan_id, bool legacy_ol_flags)
+{

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+       static bool allocerr_warned;
+       bool dropped = false;
+       uint16_t free_entries;
+       uint16_t pkt_idx, slot_idx = 0;
+       uint16_t nr_done_pkts = 0;
+       uint16_t pkt_err = 0;
+       uint16_t n_xfer;
+       struct vhost_async *async = vq->async;
+       struct async_inflight_info *pkts_info = async->pkts_info;
+       struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST];

Why do we need this array?
Plus, see blow.

Quoted 16 lines
Quoted 6 lines
lib/vhost/virtio_net.c +6 -0
+       uint16_t pkts_size = count;
+
+       /**
+        * The ordering between avail index and
+        * desc reads needs to be enforced.
+        */

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3247

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+       free_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) -
+                       vq->last_avail_idx;
+       if (free_entries == 0)
+               goto out;
+
+       rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]);
+
+       async_iter_reset(async);
+
+       count = RTE_MIN(count, MAX_PKT_BURST);

^^^
Ok, my point about the overflow does not stand.
Just the pkts_prealloc array is probably useless.

Quoted 11 lines

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3252

Quoted 5 lines
lib/vhost/virtio_net.c +5 -0
+       count = RTE_MIN(count, free_entries);
+       VHOST_LOG_DATA(DEBUG, "(%s) about to dequeue %u buffers\n",
+                       dev->ifname, count);
+
+       if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts_prealloc, count))

'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than MAX_PKT_BURST.

Calling rte_pktmbuf_alloc_bulk on a fixed-size array pkts_prealloc, allocated on the stack, it may cause a stack overflow.

The rest still stands for me.
vvv

Quoted 17 lines

This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.
I point here another bug.
There are probably more...

<grmbl>
I don't like how async code has been added in the vhost library by Intel.

Maxime did a cleanup on the enqueue patch
https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&archive=both.
I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>

Signature · 1 line
David Marchand
author
#70 lore raw

RE: [PATCH v8 4/5] vhost: support async dequeue for split ring

Hi David,

Quoted 44 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, June 16, 2022 10:40 PM
To: Ding, Xuan <[email protected]>
Cc: Maxime Coquelin <[email protected]>; Xia, Chenbo <[email protected]>; dev <[email protected]>; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>; Mcnamara, John <[email protected]>
Subject: Re: [PATCH v8 4/5] vhost: support async dequeue for split ring

Quoted 4 lines · David Marchand

Review

Quoted 2 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
(quoted diff) +2 -0
+static __rte_always_inline uint16_t
+virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue

*vq,

Quoted 1 line
Quoted 1 line

+ struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts,

uint16_t count,

Quoted 13 lines

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231

Quoted 12 lines
lib/vhost/virtio_net.c +12 -0
+               int16_t dma_id, uint16_t vchan_id, bool
+legacy_ol_flags) {
+       static bool allocerr_warned;
+       bool dropped = false;
+       uint16_t free_entries;
+       uint16_t pkt_idx, slot_idx = 0;
+       uint16_t nr_done_pkts = 0;
+       uint16_t pkt_err = 0;
+       uint16_t n_xfer;
+       struct vhost_async *async = vq->async;
+       struct async_inflight_info *pkts_info = async->pkts_info;
+       struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST];
Quoted 10 lines

Why do we need this array?
Plus, see blow.

Quoted 7 lines
lib/vhost/virtio_net.c +7 -0
+       uint16_t pkts_size = count;
+
+       /**
+        * The ordering between avail index and
+        * desc reads needs to be enforced.
+        */
+       free_entries = __atomic_load_n(&vq->avail->idx,

__ATOMIC_ACQUIRE) -

Quoted 14 lines
Quoted 10 lines

lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3247

Quoted 10 lines
lib/vhost/virtio_net.c +10 -0
+                       vq->last_avail_idx;
+       if (free_entries == 0)
+               goto out;
+
+       rte_prefetch0(&vq->avail->ring[vq->last_avail_idx &
+ (vq->size - 1)]);
+
+       async_iter_reset(async);
+
+       count = RTE_MIN(count, MAX_PKT_BURST);

^^^
Ok, my point about the overflow does not stand. Just the pkts_prealloc array is probably useless.

Allocating a bulk of mbufs by rte_pktmbuf_alloc_bulk() is for performance consideration. The pkts_prealloc array is for keeping a temporary variable to orderly update async_inflight_info, which is required in async path.

Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
lib/vhost/virtio_net.c +5 -0
+       count = RTE_MIN(count, free_entries);
+       VHOST_LOG_DATA(DEBUG, "(%s) about to dequeue %u buffers\n",
+                       dev->ifname, count);
+
+       if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts_prealloc, count))
Quoted 14 lines
Quoted 5 lines

'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than MAX_PKT_BURST.

Calling rte_pktmbuf_alloc_bulk on a fixed-size array pkts_prealloc, allocated on the stack, it may cause a stack overflow.

The rest still stands for me.
vvv

Quoted 5 lines

This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.

I need to explain the fix here.

The async dequeue patches and stats patches were both sent out
and merged in this release, while stats patch requires changes in both sync/async enq and deq. Sorry for not notice this change in the stats patch that was merged first, and that is the reason for this bug.

Quoted 14 lines
Quoted 8 lines

I point here another bug.
There are probably more...

<grmbl>
I don't like how async code has been added in the vhost library by Intel.

Maxime did a cleanup on the enqueue patch

https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&ar
chive=both.

Quoted 4 lines

I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>

The cleanup for the code in dequeue patch was suggested in RFC v3.

https://patchwork.dpdk.org/project/dpdk/patch/[email protected]/

With merging copy_desc_to_mbuf and async_desc_to_mbuf in a single function, and reuse the fill_seg function, the code can be simplified without performance degradation.

Could you help to point out where in the current code that needs to be further cleaned? Are you referring to the common parts of async deq API and sync deq API can be abstracted into a function, such as ARAP?

https://patchwork.dpdk.org/project/dpdk/patch/[email protected]/

There are code duplications between async deq API and sync deq API, these parts are both needed. But the code clean of dequeue is by no means mindless copy/pasting.

Hope to get your insights.

Thanks,
Xuan

Quoted 4 lines

--
David Marchand

#71 lore raw

PATCH v8 5/5 examples/vhost: support async dequeue data path

In reply to [email protected]

From: Xuan Ding <[email protected]>

This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.

---
 doc/guides/sample_app_ug/vhost.rst |   9 +-
 examples/vhost/main.c              | 286 ++++++++++++++++++++---------
 examples/vhost/main.h              |  32 +++-
 examples/vhost/virtio_net.c        |  16 +-
 4 files changed, 245 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
index a6ce4bc8ac..09db965e70 100644
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas
 **--dmas**
 This parameter is used to specify the assigned DMA device of a vhost device.
 Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost
-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1
-enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use
+DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation
+and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue
+operation. The index of the device corresponds to the socket file in order,
+that means vhost device 0 is created through the first socket file, vhost
+device 1 is created through the second socket file, and so on.
 
 Common Issues
 -------------
examples/vhost/main.c +195 -91

428 lines not on this page · raw message

examples/vhost/main.h +29 -3
index e7f395c3c9..2fcb8376c5 100644
@@ -61,6 +61,19 @@ struct vhost_dev {
 	struct vhost_queue queues[MAX_QUEUE_PAIRS * 2];
 } __rte_cache_aligned;
 
+typedef uint16_t (*vhost_enqueue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mbuf **pkts,
+			uint32_t count);
+
+typedef uint16_t (*vhost_dequeue_burst_t)(struct vhost_dev *dev,
+			uint16_t queue_id, struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+
+struct vhost_queue_ops {
+	vhost_enqueue_burst_t enqueue_pkt_burst;
+	vhost_dequeue_burst_t dequeue_pkt_burst;
+};
+
 TAILQ_HEAD(vhost_dev_tailq_list, vhost_dev);
 
 
@@ -87,6 +100,7 @@ struct dma_info {
 
 struct dma_for_vhost {
 	struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT * 2];
+	uint32_t async_flag;
 };
 
 /* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);
 uint16_t vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 			 struct rte_mbuf **pkts, uint32_t count);
 
-uint16_t vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
-			 struct rte_mempool *mbuf_pool,
-			 struct rte_mbuf **pkts, uint16_t count);
+uint16_t builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mbuf **pkts, uint32_t count);
+uint16_t builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t sync_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t sync_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
+uint16_t async_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			 struct rte_mbuf **pkts, uint32_t count);
+uint16_t async_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+			struct rte_mempool *mbuf_pool,
+			struct rte_mbuf **pkts, uint16_t count);
 #endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
index 9064fc3a82..2432a96566 100644
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	return count;
 }
 
+uint16_t
+builtin_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+		struct rte_mbuf **pkts, uint32_t count)
+{
+	return vs_enqueue_pkts(dev, queue_id, pkts, count);
+}
+
 static __rte_always_inline int
 dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	    struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,
 	return 0;
 }
 
-uint16_t
+static uint16_t
 vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
 {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
 
 	return i;
 }
+
+uint16_t
+builtin_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,
+	struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count)
+{
+	return vs_dequeue_pkts(dev, queue_id, mbuf_pool, pkts, count);
+}
Signature · 1 line
2.17.1
#72 lore raw
In reply to [email protected]
Quoted 73 lines · On 5/16/22 13:10, [email protected] wrote:

From: Xuan Ding <[email protected]>

The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].

This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.

A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)

Receive packets from the guest and offloads copies to DMA
virtual channel.

[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html

v7->v8:
* change dma_id to int16_t
* refine API documentation

v6->v7:
* correct code formatting
* change some functions to inline

v5->v6:
* adjust EXPERIMENTAL header

v4->v5:
* rebase to latest DPDK
* add some checks

v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id

v2->v3:
* fix mbuf not updated correctly for large packets

v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling

RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs

RFC v2 -> RFC v3:
* rebase to latest DPDK version

RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard

Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path

Quoted 12 lines
  doc/guides/prog_guide/vhost_lib.rst    |   6 +
  doc/guides/rel_notes/release_22_07.rst |   5 +
  doc/guides/sample_app_ug/vhost.rst     |   9 +-
  examples/vhost/main.c                  | 286 ++++++++++-----
  examples/vhost/main.h                  |  32 +-
  examples/vhost/virtio_net.c            |  16 +-
  lib/vhost/rte_vhost_async.h            |  37 ++
  lib/vhost/version.map                  |   2 +-
  lib/vhost/vhost.h                      |   1 +
  lib/vhost/virtio_net.c                 | 473 ++++++++++++++++++++++---
  10 files changed, 715 insertions(+), 152 deletions(-)

Applied to dpdk-next-virtio/main.

Thanks,
Maxime