vhost: support async dequeue data path
cover vhost: support async dequeue data path
From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch implements vhost async dequeue data path for split ring.
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 +++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 459 ++++++++++++++++++++++--- 10 files changed, 711 insertions(+), 148 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplies the code without performance degradation.
--- lib/vhost/virtio_net.c | 66 +++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 37 deletions(-)
lib/vhost/virtio_net.c +29 -37
@@ -1030,9 +1030,9 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
@@ -1043,10 +1043,17 @@ sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1165,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, +sync_fill_seg (dev, vq, m, mbuf_offset, buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2474,7 +2481,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, bool legacy_ol_flags) {uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; +uint64_t buf_addr, buf_iova, buf_len;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2489,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2519,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2544,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2559,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2578,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2598,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplies the code without performance degradation.
--- lib/vhost/virtio_net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, + (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1161,8 +1168,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 47 ++++++++++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];* inflight async packet information */ struct async_inflight_info { +struct virtio_net_hdr nethdr;struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +36 -11
97 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 335 +++++++++++++++++++++++++ 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -55,6 +55,10 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to accelerate + receiving pkts from guest. Removed Items -------------
lib/vhost/rte_vhost_async.h +33 -0
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* The amount of in-flight packets. If error occurred, its value is set to -1. +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +3 -0
@@ -87,6 +87,9 @@ EXPERIMENTAL { # added in 22.03 rte_vhost_async_dma_configure; + + # added in 22.07 + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +335 -0
338 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +202 -90
438 lines not on this page · raw message
examples/vhost/main.h +31 -4
@@ -52,7 +52,8 @@ struct vhost_dev {uint64_t features;size_t hdr_len;uint16_t nr_vrings; -uint16_t pkts_inflight; +uint16_t pkts_enq_inflight; +uint16_t pkts_deq_inflight;struct rte_vhost_memory *mem;struct device_statistics stats;TAILQ_ENTRY (vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -88,6 +102,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
v2 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 +++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 460 ++++++++++++++++++++++--- 10 files changed, 712 insertions(+), 148 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 66 +++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 37 deletions(-)
lib/vhost/virtio_net.c +29 -37
@@ -1030,9 +1030,9 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;
@@ -1043,10 +1043,17 @@ sync_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1165,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, +sync_fill_seg (dev, vq, m, mbuf_offset, buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2474,7 +2481,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, bool legacy_ol_flags) {uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; +uint64_t buf_addr, buf_iova, buf_len;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2489,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2519,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2544,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2559,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2578,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2598,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH v2 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, + (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1161,8 +1168,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH v2 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];* inflight async packet information */ struct async_inflight_info { +struct virtio_net_hdr nethdr;struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 335 +++++++++++++++++++++++++ 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -55,6 +55,10 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to accelerate + receiving pkts from guest. Removed Items -------------
lib/vhost/rte_vhost_async.h +33 -0
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* The amount of in-flight packets. If error occurred, its value is set to -1. +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +3 -0
@@ -87,6 +87,9 @@ EXPERIMENTAL { # added in 22.03 rte_vhost_async_dma_configure; + + # added in 22.07 + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +335 -0
338 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v2 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +202 -90
438 lines not on this page · raw message
examples/vhost/main.h +31 -4
@@ -52,7 +52,8 @@ struct vhost_dev {uint64_t features;size_t hdr_len;uint16_t nr_vrings; -uint16_t pkts_inflight; +uint16_t pkts_enq_inflight; +uint16_t pkts_deq_inflight;struct rte_vhost_memory *mem;struct device_statistics stats;TAILQ_ENTRY (vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -88,6 +102,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
v3 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 470 ++++++++++++++++++++++--- 10 files changed, 720 insertions(+), 150 deletions(-)
Signature · 1 line
2.17.1
PATCH v3 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 76 ++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 39 deletions(-)
lib/vhost/virtio_net.c +37 -39
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, +sync_fill_seg (dev, vq, m, mbuf_offset, buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
Re: [PATCH v3 1/5] vhost: prepare sync for descriptor to mbuf refactoring
Hi Xuan,
Quoted 8 lines · On 4/19/22 05:43, [email protected] wrote:
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from
copy_desc_to_mbuf()into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same functionsync_fill_seg()for preparing batch elements, which simplifies the code without performance degradation.
Quoted 5 lines
Signed-off-by Xuan Ding <[email protected]>--- lib/vhost/virtio_net.c | 76 ++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 39 deletions(-)
Nice refactoring, thanks for donig it:
Thanks,
Maxime
PATCH v3 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +15 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, + (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1174,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
Re: [PATCH v3 2/5] vhost: prepare async for descriptor to mbuf refactoring
Quoted 40 lines · On 4/19/22 05:43, [email protected] wrote:
From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function
async_fill_seg()for preparing batch elements, which simplifies the code without performance degradation.Signed-off-by Xuan Ding <[email protected]>--- lib/vhost/virtio_net.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-)lib/vhost/virtio_net.c +3
-6@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {@@ -1015,10 +1016,16 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len)))
lib/vhost/virtio_net.c async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, · line 1028
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, + (size_t )mapped_len)))
Minor, but it may fit in a single line.
Quoted 4 lines
lib/vhost/virtio_net.c +0
-0return -1 ; cpy_len -= (uint32_t )mapped_len;
lib/vhost/virtio_net.c mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, · line 1181
Quoted 10 lines
lib/vhost/virtio_net.c +2
-2@@ -1167,8 +1174,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Thanks,
Maxime
PATCH v3 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patches refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -177,6 +177,7 @@ extern struct async_dma_info dma_copy_track[RTE_DMADEV_DEFAULT_MAX];* inflight async packet information */ struct async_inflight_info { +struct virtio_net_hdr nethdr;struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling
We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.
Copy/pasting the code:
+static __rte_always_inline uint16_t +async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t queue_id,
+ struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id,
+ uint16_t vchan_id, bool legacy_ol_flags)
+{
+ uint16_t start_idx, from, i;
+ uint16_t nr_cpl_pkts = 0;
+ struct async_inflight_info *pkts_info;
+ struct vhost_virtqueue *vq = dev->virtqueue[queue_id];
+
Please, don't pass queue_id as an input parameter for async_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.
Signature · 1 line
David Marchand
Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling
Quoted 16 lines · David Marchand
We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.
Copy/pasting the code:
(quoted diff) +10
-0+static __rte_always_inline uint16_t +async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t queue_id, + struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id, + uint16_t vchan_id, bool legacy_ol_flags) +{ + uint16_t start_idx, from, i; + uint16_t nr_cpl_pkts = 0; + struct async_inflight_info *pkts_info; + struct vhost_virtqueue *vq = dev->virtqueue[queue_id]; +
Quoted 6 lines
Please, don't pass
queue_idas an input parameter forasync_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.
I think David's comment was intended to be a reply to patch 4, but I agree with him.
Could you please fix this and also fix the build issues reported by the CI? I'll continue the review on V4.
Thanks,
Maxime
RE: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling
Hi Maxime, David,
Quoted 38 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Friday, April 22, 2022 11:46 PM
To: David Marchand <[email protected]>; Ding, Xuan <[email protected]>
Cc: Xia, Chenbo <[email protected]>; dev <[email protected]>; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]
Subject: Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf fillingQuoted 13 lines · David Marchand
We (at RH) have some issues with our email infrastructure, so I can't reply inline of the patch.
Copy/pasting the code:
(quoted diff) +7
-0+static __rte_always_inline uint16_t +async_poll_dequeue_completed_split(struct virtio_net *dev, uint16_t +queue_id, struct rte_mbuf **pkts, uint16_t count, uint16_t dma_id, +uint16_t vchan_id, bool legacy_ol_flags) { uint16_t start_idx, from, +i; uint16_t nr_cpl_pkts = 0; struct async_inflight_info *pkts_info; +struct vhost_virtqueue *vq = dev->virtqueue[queue_id]; +Quoted 6 lines
Please, don't pass
queue_idas an input parameter forasync_poll_dequeue_completed_split().
The caller of this helper already dereferenced the vq. You can pass vq.I think David's comment was intended to be a reply to patch 4, but I agree with him.
Quoted 2 lines
Could you please fix this and also fix the build issues reported by the CI? I'll continue the review on V4.
Thanks for your suggestion, please see v4.
Regards,
Xuan
Quoted 3 lines
Thanks,
Maxime
Re: [PATCH v3 3/5] vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]Quoted 11 lines · On 4/19/22 05:43, [email protected] wrote:
From: Xuan Ding <[email protected]>
This patches refactors
copy_desc_to_mbuf()used by the sync path to support both sync and async descriptor to mbuf filling.Signed-off-by Xuan Ding <[email protected]>--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
Thanks,
Maxime
PATCH v3 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 335 +++++++++++++++++++++++++ 5 files changed, 382 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -55,6 +55,10 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to accelerate + receiving pkts from guest. Removed Items -------------
lib/vhost/rte_vhost_async.h +33 -0
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* The amount of in-flight packets. If error occurred, its value is set to -1. +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +3 -0
@@ -87,6 +87,9 @@ EXPERIMENTAL { # added in 22.03 rte_vhost_async_dma_configure; + + # added in 22.07 + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +335 -0
338 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v3 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +202 -90
438 lines not on this page · raw message
examples/vhost/main.h +31 -4
@@ -52,7 +52,8 @@ struct vhost_dev {uint64_t features;size_t hdr_len;uint16_t nr_vrings; -uint16_t pkts_inflight; +uint16_t pkts_enq_inflight; +uint16_t pkts_deq_inflight;struct rte_vhost_memory *mem;struct device_statistics stats;TAILQ_ENTRY (vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -88,6 +102,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
v4 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 +++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 467 ++++++++++++++++++++++--- 10 files changed, 716 insertions(+), 151 deletions(-)
Signature · 1 line
2.17.1
PATCH v4 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
RE: [PATCH v4 1/5] vhost: prepare sync for descriptor to mbuf refactoring
Quoted 19 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 1/5] vhost: prepare sync for descriptor to mbuf refactoringFrom: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from
copy_desc_to_mbuf()into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same functionsync_fill_seg()for preparing batch elements, which simplifies the code without performance degradation.
Quoted 6 lines
Signed-off-by Xuan Ding <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
PATCH v4 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
RE: [PATCH v4 2/5] vhost: prepare async for descriptor to mbuf refactoring
Quoted 17 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 2/5] vhost: prepare async for descriptor to mbuf refactoringFrom: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function
async_fill_seg()for preparing batch elements, which simplifies the code without performance degradation.
Quoted 6 lines
Signed-off-by Xuan Ding <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
PATCH v4 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -180,6 +180,7 @@ struct async_inflight_info {struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */ +struct virtio_net_hdr nethdr; };struct vhost_async {
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
RE: [PATCH v4 3/5] vhost: merge sync and async descriptor to mbuf filling
Quoted 16 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>
Subject: [PATCH v4 3/5] vhost: merge sync and async descriptor to mbuf fillingFrom: Xuan Ding <[email protected]>
This patch refactors
copy_desc_to_mbuf()used by the sync path to support both sync and async descriptor to mbuf filling.
Quoted 7 lines
Signed-off-by Xuan Ding <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
PATCH v4 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 331 +++++++++++++++++++++++++ 5 files changed, 378 insertions(+)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -276,6 +276,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +4 -0
@@ -55,6 +55,10 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to accelerate + receiving pkts from guest. Removed Items -------------
lib/vhost/rte_vhost_async.h +33 -0
@@ -187,6 +187,39 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* The amount of in-flight packets. If error occurred, its value is set to -1. +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +3 -0
@@ -87,6 +87,9 @@ EXPERIMENTAL { # added in 22.03 rte_vhost_async_dma_configure; + + # added in 22.07 + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +331 -0
334 lines not on this page · raw message
Signature · 1 line
2.17.1
RE: [PATCH v4 4/5] vhost: support async dequeue for split ring
doc/guides/prog_guide/vhost_lib.rst
Quoted 33 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v4 4/5] vhost: support async dequeue for split ringFrom: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API
rte_vhost_async_try_dequeue_burst()is introduced.Signed-off-by Xuan Ding <[email protected]>Signed-off-by Yuan Wang <[email protected]>--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 331 +++++++++++++++++++++++++ 5 files changed, 378 insertions(+)doc/guides/prog_guide/vhost_lib.rst +0
-0
Re: [PATCH v4 4/5] vhost: support async dequeue for split ring
In reply to [email protected]Quoted 7 lines · On 5/5/22 08:23, [email protected] wrote:
From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API
rte_vhost_async_try_dequeue_burst()is introduced.Signed-off-by Xuan Ding <[email protected]>Signed-off-by Yuan Wang <[email protected]>
Quoted 8 lines
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + lib/vhost/rte_vhost_async.h | 33 +++ lib/vhost/version.map | 3 + lib/vhost/virtio_net.c | 331 +++++++++++++++++++++++++ 5 files changed, 378 insertions(+)
Thanks,
Maxime
PATCH v4 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +202 -90
438 lines not on this page · raw message
examples/vhost/main.h +31 -4
@@ -52,7 +52,8 @@ struct vhost_dev {uint64_t features;size_t hdr_len;uint16_t nr_vrings; -uint16_t pkts_inflight; +uint16_t pkts_enq_inflight; +uint16_t pkts_deq_inflight;struct rte_vhost_memory *mem;struct device_statistics stats;TAILQ_ENTRY (vhost_dev) global_vdev_entry;
@@ -62,6 +63,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -88,6 +102,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -98,7 +113,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
RE: [PATCH v4 5/5] examples/vhost: support async dequeue data path
Quoted 33 lines
-----Original Message-----
From: [email protected] <[email protected]>
Sent: 2022年5月5日 14:24
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v4 5/5] examples/vhost: support async dequeue data pathFrom: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
Signed-off-by Wenwu Ma <[email protected]>Signed-off-by Yuan Wang <[email protected]>Signed-off-by Xuan Ding <[email protected]>--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)doc/guides/sample_app_ug/vhost.rst +0
-0@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's
Quoted 1 line
used in combination with dmas
Re: [PATCH v4 5/5] examples/vhost: support async dequeue data path
In reply to [email protected]Quoted 8 lines · On 5/5/22 08:23, [email protected] wrote:
From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
Signed-off-by Wenwu Ma <[email protected]>Signed-off-by Yuan Wang <[email protected]>Signed-off-by Xuan Ding <[email protected]>
Quoted 7 lines
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 ++++++++++++++++++++--------- examples/vhost/main.h | 35 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 254 insertions(+), 98 deletions(-)
Thanks,
Maxime
Maxime Coquelin
AppliedHi Xuan,
Quoted 58 lines · On 5/5/22 08:23, [email protected] wrote:
From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid,uint16_tqueue_id, structrte_mempool*mbuf_pool, structrte_mbuf**pkts,uint16_tcount, int*nr_inflight,
uint16_tdma_id,uint16_tvchan_id)Receive packets from the guest and offloads copies to DMA
virtual channel.[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead ofqueue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug indesc_to_mbuffillingRFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docsRFC v2 -> RFC v3:
* rebase to latest DPDK versionRFC v1 -> RFC v2:
* fix one bug in example
* rename vchan tovchan_id
* check ifdma_idandvchan_idvalid
* rework all the logs to new standardXuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
Quoted 12 lines
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 +++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 467 ++++++++++++++++++++++--- 10 files changed, 716 insertions(+), 151 deletions(-)
I applied your other series about unsafe API to get DMA inflight packets, so I have some conflicts when applying this series.
Could you please rebase on top of next-virtio/main branch and repost?
Thanks in avance,
Maxime
Hi Maxime,
Quoted 83 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Friday, May 6, 2022 3:53 AM
To: Ding, Xuan <[email protected]>; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]
Subject: Re: [PATCH v4 0/5] vhost: support async dequeue data pathHi Xuan,
Quoted 58 lines · On 5/5/22 08:23, [email protected] wrote:
From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid,uint16_tqueue_id, structrte_mempool*mbuf_pool, structrte_mbuf**pkts,uint16_tcount, int*nr_inflight,
uint16_tdma_id,uint16_tvchan_id)Receive packets from the guest and offloads copies to DMA
virtual channel.[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead ofqueue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug indesc_to_mbuffillingRFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docsRFC v2 -> RFC v3:
* rebase to latest DPDK versionRFC v1 -> RFC v2:
* fix one bug in example
* rename vchan tovchan_id
* check ifdma_idandvchan_idvalid
* rework all the logs to new standardXuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data pathQuoted 12 lines
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 4 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 292 +++++++++++----- examples/vhost/main.h | 35 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 33 ++ lib/vhost/version.map | 3 + lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 467 ++++++++++++++++++++++--- 10 files changed, 716 insertions(+), 151 deletions(-)
Quoted 4 lines
I applied your other series about unsafe API to get DMA inflight packets, so I have some conflicts when applying this series.
Could you please rebase on top of next-virtio/main branch and repost?
Sure, I will rebase on top of latest main branch, please see v5.
Thanks,
Xuan
Quoted 3 lines
Thanks in avance,
Maxime
v5 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v4->v5:
* rebase to latest DPDK
* add some checks
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++----- examples/vhost/main.h | 32 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 34 ++ lib/vhost/version.map | 2 +- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 473 ++++++++++++++++++++++--- 10 files changed, 711 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
PATCH v5 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH v5 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH v5 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -180,6 +180,7 @@ struct async_inflight_info {struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */ +struct virtio_net_hdr nethdr; };struct vhost_async {
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v5 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 34 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 384 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------
lib/vhost/rte_vhost_async.h +34 -0
@@ -204,6 +204,40 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* >= 0: The amount of in-flight packets +* -1: meaningless, indicates failed lock acquisition or invalid queue_id/dma_id +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +1 -1
@@ -90,7 +90,7 @@ EXPERIMENTAL { # added in 22.07 rte_vhost_async_get_inflight_thread_unsafe; - + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +337 -0
340 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v5 4/5] vhost: support async dequeue for split ring
Quoted 9 lines · On Fri, 13 May 2022 02:00:21 +0000 [email protected] wrote:
+/**
+ * This function tries to receive packets from the guest with offloading
+ * copies to the async channel. The packets that are transfer completed
+ * are returned in "pkts". The other packets that their copies are submitted to
+ * the async channel but not completed are called "in-flight packets".
+ * This function will not return in-flight packets until their copies are
+ * completed by the async channel.
+ *
Please add EXPERIMENTAL header like this, so it shows up in docmentation correctly.
*
* @warning
* @b EXPERIMENTAL:
* All functions in this file may be changed or removed without prior notice.
RE: [PATCH v5 4/5] vhost: support async dequeue for split ring
Quoted 16 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Friday, May 13, 2022 10:24 AM
To: Ding, Xuan <[email protected]>
Cc: [email protected]; Xia, Chenbo <[email protected]>; [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>
Subject: Re: [PATCH v5 4/5] vhost: support async dequeue for split ringQuoted 1 line · On Fri, 13 May 2022 02:00:21 +0000 [email protected] wrote:
Quoted 15 lines
Review
Quoted 12 lines
(quoted diff) +12
-0+/** + * This function tries to receive packets from the guest with +offloading + * copies to the async channel. The packets that are transfer +completed + * are returned in "pkts". The other packets that their copies are +submitted to + * the async channel but not completed are called "in-flight packets". + * This function will not return in-flight packets until their copies +are + * completed by the async channel. + *Please add EXPERIMENTAL header like this, so it shows up in docmentation correctly.
Get it, thanks for your suggestion.
Regards,
Xuan
Quoted 5 lines
*
* @warning
* @b EXPERIMENTAL:
* All functions in this file may be changed or removed without prior notice.
PATCH v5 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +193 -91
426 lines not on this page · raw message
examples/vhost/main.h +29 -3
@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
v6 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v5->v6:
* adjust EXPERIMENTAL header
v4->v5:
* rebase to latest DPDK
* add some checks
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++----- examples/vhost/main.h | 32 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 37 ++ lib/vhost/version.map | 2 +- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 473 ++++++++++++++++++++++--- 10 files changed, 714 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
PATCH v6 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH v6 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH v6 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -180,6 +180,7 @@ struct async_inflight_info {struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */ +struct virtio_net_hdr nethdr; };struct vhost_async {
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 37 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------
lib/vhost/rte_vhost_async.h +37 -0
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* @warning +* @b EXPERIMENTAL: this API may change, or be removed, without prior notice +* +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* >= 0: The amount of in-flight packets +* -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +1 -1
@@ -90,7 +90,7 @@ EXPERIMENTAL { # added in 22.07 rte_vhost_async_get_inflight_thread_unsafe; - + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +337 -0
340 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v6 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 243 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +193 -91
426 lines not on this page · raw message
examples/vhost/main.h +29 -3
@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
RE: [PATCH v6 5/5] examples/vhost: support async dequeue data path
Quoted 81 lines
-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Friday, May 13, 2022 10:51 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v6 5/5] examples/vhost: support async dequeue data pathFrom: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
Signed-off-by Wenwu Ma <[email protected]>Signed-off-by Yuan Wang <[email protected]>Signed-off-by Xuan Ding <[email protected]>Tested-by Yvonne Yang <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 243 insertions(+), 98 deletions(-)doc/guides/sample_app_ug/vhost.rst +0
-0@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It'sused in combination with dmas
**--dmas**
This parameter is used to specify the assigned DMA device of a vhost
device.
Async vhost-user net driver will be used if --dmas is set. For example
---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhostdoc/guides/sample_app_ug/vhost.rst +4
-2-device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file inorder,
doc/guides/sample_app_ug/vhost.rst +2
-0+that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------examples/vhost/main.c +6
-0@@ -63,6 +63,9 @@#define DMA_RING_SIZE 4096 +#define ASYNC_ENQUEUE_VHOST 1 +#define ASYNC_DEQUEUE_VHOST 2 +/* number of mbufs in all pools - if specified on command-line. */ static int total_num_mbufs =NUM_MBUFS_DEFAULT ;@@ -116,6 +119,8 @@ static uint32_t burst_rx_retry_num = BURST_RX_RETRIES;static char *socket_files;static int nb_sockets; +static struct vhost_queue_ops vdev_queue_ops[RTE_MAX_VHOST_DEVICE ]; +/* empty VMDq configuration structure. Filled in programmatically */ static struct rte_eth_conf vmdq_conf_default = { .rxmode = {@@ -205,6 +210,18 @@ struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE * RTE_MAX_VHOST_DEVICE];#define MBUF_TABLE_DRAIN_TSC ((rte_get_tsc_hz() + US_PER_S - 1) \ / US_PER_S * BURST_TX_DRAIN_US) +static int vid2socketid[RTE_MAX_VHOST_DEVICE ];
examples/vhost/main.c struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE * · line 224
Quoted 10 lines
examples/vhost/main.c +10
-0+ +static uint32_t get_async_flag_by_socketid (int socketid) +{ +return dma_bind[socketid].async_flag; +} + +static void init_vid2socketid_array (int vid,int socketid) +{ + vid2socketid[vid] = socketid; +}
Return value and func name should be on separate lines as per coding style. And above func can be inline, same suggestion for short func below, especially ones in data path.
Thanks,
Chenbo
Quoted 544 lines
examples/vhost/main.c +1
-0+static inline bool is_dma_configured (int16_t dev_id){
examples/vhost/main.c +3
-1@@ -224,7 +241,7 @@ open_dma(const char *value)char *addrs = input;char *ptrs[2 ];char *start, *end, *substr; -int64_t vid; +int64_t socketid, vring_id;struct rte_dma_info info;struct rte_dma_conf dev_config = { .nb_vchans =1 };@@ -262,7 +279,9 @@ open_dma(const char *value)while (i < args_nr) {char *arg_temp = dma_arg[i]; +char *txd, *rxd;uint8_t sub_nr; +int async_flag; sub_nr =rte_strsplit (arg_temp,strlen (arg_temp), ptrs,2 ,'@' );examples/vhost/main.c +15
-5if (sub_nr !=2 ) {@@ -270,14 +289,23 @@ open_dma(const char *value) goto out; } - start =strstr (ptrs[0 ],"txd" ); -if (start ==NULL ) { + txd =strstr (ptrs[0 ],"txd" ); + rxd =strstr (ptrs[0 ],"rxd" ); +if (txd) { + start = txd; + vring_id =VIRTIO_RXQ ; + async_flag =ASYNC_ENQUEUE_VHOST ; + }else if (rxd) { + start = rxd; + vring_id =VIRTIO_TXQ ; + async_flag =ASYNC_DEQUEUE_VHOST ; + }else { ret =-1 ; goto out; } start +=3 ; - vid =strtol (start, &end,0 ); + socketid =strtol (start, &end,0 );if (end == start) { ret =-1 ; goto out;@@ -338,7 +366,8 @@ open_dma(const char *value) dmas_id[dma_count++] = dev_id; done: - (dma_info + vid)->dmas[VIRTIO_RXQ ].dev_id = dev_id; + (dma_info + socketid)->dmas[vring_id].dev_id = dev_id; + (dma_info + socketid)->async_flag |= async_flag; i++; } out:@@ -990,7 +1019,7 @@ complete_async_pkts(struct vhost_dev *vdev) {struct rte_mbuf *p_cpl[MAX_PKT_BURST ];uint16_t complete_count; -int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].dev_id; +int16_t dma_id = dma_bind[vid2socketid[vdev- >vid]].dmas[VIRTIO_RXQ].dev_id; complete_count = rte_vhost_poll_enqueue_completed(vdev->vid,examples/vhost/main.c +1
-16VIRTIO_RXQ, p_cpl, MAX_PKT_BURST, dma_id, 0);@@ -1029,22 +1058,7 @@ drain_vhost(struct vhost_dev *vdev)uint16_t nr_xmit = vhost_txbuff[buff_idx]->len;struct rte_mbuf **m = vhost_txbuff[buff_idx]->m_table; -if (builtin_net_driver) { - ret =vs_enqueue_pkts (vdev,VIRTIO_RXQ , m, nr_xmit); - }else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t enqueue_fail =0 ; -int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].dev_id; - -complete_async_pkts (vdev); - ret =rte_vhost_submit_enqueue_burst (vdev->vid,VIRTIO_RXQ , m, nr_xmit, dma_id, 0); - - enqueue_fail = nr_xmit - ret; -if (enqueue_fail) -free_pkts (&m[ret], nr_xmit - ret); - }else { - ret =rte_vhost_enqueue_burst (vdev->vid,VIRTIO_RXQ , - m, nr_xmit); - } + ret = vdev_queue_ops[vdev->vid].enqueue_pkt_burst (vdev,VIRTIO_RXQ , m, nr_xmit);examples/vhost/main.c +27
-1if (enable_stats) { __atomic_add_fetch(&vdev->stats.rx_total_atomic, nr_xmit,@@ -1053,7 +1067,7 @@ drain_vhost(struct vhost_dev *vdev) __ATOMIC_SEQ_CST); } -if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) +if (!dma_bind[vid2socketid[vdev- >vid]].dmas[VIRTIO_RXQ].async_enabled)free_pkts (m, nr_xmit); }@@ -1325,6 +1339,32 @@ drain_mbuf_table(struct mbuf_table *tx_q) } } +uint16_t +async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t rx_count) +{ +uint16_t enqueue_count; +uint16_t enqueue_fail =0 ; +uint16_t dma_id = dma_bind[vid2socketid[dev- >vid]].dmas[VIRTIO_RXQ].dev_id; + +complete_async_pkts (dev); + enqueue_count =rte_vhost_submit_enqueue_burst (dev->vid, queue_id, + pkts, rx_count, dma_id,0 ); + + enqueue_fail = rx_count - enqueue_count; +if (enqueue_fail) +free_pkts (&pkts[enqueue_count], enqueue_fail); + +return enqueue_count; +} + +uint16_t +sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t rx_count) +{ +return rte_vhost_enqueue_burst (dev->vid, queue_id, pkts, rx_count); +} + static __rte_always_inline voiddrain_eth_rx (struct vhost_dev *vdev){
examples/vhost/main.c +24
-20@@ -1355,25 +1395,8 @@ drain_eth_rx(struct vhost_dev *vdev) } } -if (builtin_net_driver) { - enqueue_count =vs_enqueue_pkts (vdev,VIRTIO_RXQ , - pkts, rx_count); - }else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t enqueue_fail =0 ; -int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].dev_id; - -complete_async_pkts (vdev); - enqueue_count =rte_vhost_submit_enqueue_burst (vdev->vid, -VIRTIO_RXQ , pkts, rx_count, dma_id,0 ); - - enqueue_fail = rx_count - enqueue_count; -if (enqueue_fail) -free_pkts (&pkts[enqueue_count], enqueue_fail); - - }else { - enqueue_count =rte_vhost_enqueue_burst (vdev->vid,VIRTIO_RXQ , - pkts, rx_count); - } + enqueue_count = vdev_queue_ops[vdev->vid].enqueue_pkt_burst (vdev, +VIRTIO_RXQ , pkts, rx_count);if (enable_stats) { __atomic_add_fetch(&vdev->stats.rx_total_atomic, rx_count,@@ -1382,10 +1405,31 @@ drain_eth_rx(struct vhost_dev *vdev) __ATOMIC_SEQ_CST); } -if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) +if (!dma_bind[vid2socketid[vdev- >vid]].dmas[VIRTIO_RXQ].async_enabled)free_pkts (pkts, rx_count); } +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count) +{ +int nr_inflight; +uint16_t dequeue_count; +uint16_t dma_id = dma_bind[vid2socketid[dev- >vid]].dmas[VIRTIO_TXQ].dev_id; + + dequeue_count =rte_vhost_async_try_dequeue_burst (dev->vid, queue_id, + mbuf_pool, pkts, count, &nr_inflight, dma_id,0 ); + +return dequeue_count; +} + +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count) +{ +return rte_vhost_dequeue_burst (dev->vid, queue_id, mbuf_pool, pkts, count); +} +static
__rte_always_inlinevoid
drain_virtio_tx(struct vhost_dev *vdev)
{examples/vhost/main.c +18
-7@@ -1393,13 +1437,8 @@ drain_virtio_tx(struct vhost_dev *vdev)uint16_t count;uint16_t i; -if (builtin_net_driver) { - count =vs_dequeue_pkts (vdev,VIRTIO_TXQ , mbuf_pool, - pkts,MAX_PKT_BURST ); - }else { - count =rte_vhost_dequeue_burst (vdev->vid,VIRTIO_TXQ , - mbuf_pool, pkts,MAX_PKT_BURST ); - } + count = vdev_queue_ops[vdev->vid].dequeue_pkt_burst (vdev, +VIRTIO_TXQ , mbuf_pool, pkts,MAX_PKT_BURST );/* setup VMDq for the first packet */ if (unlikely (vdev->ready ==DEVICE_MAC_LEARNING ) && count) {@@ -1478,6 +1517,26 @@ switch_worker(void *arg __rte_unused)return 0 ; } +static void +vhost_clear_queue_thread_unsafe (struct vhost_dev *vdev,uint16_t queue_id) +{ +uint16_t n_pkt =0 ; +int pkts_inflight; + +int16_t dma_id = dma_bind[vid2socketid[vdev- >vid]].dmas[queue_id].dev_id; + pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe (vdev->vid, queue_id); + +struct rte_mbuf *m_cpl[pkts_inflight]; + +while (pkts_inflight) { + n_pkt =rte_vhost_clear_queue_thread_unsafe (vdev->vid, queue_id, m_cpl, + pkts_inflight, dma_id,0 ); +free_pkts (m_cpl, n_pkt); + pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,
examples/vhost/main.c +36
-14+ queue_id); + } +} +/* * Remove a device from the specific data core linked list and from the * main linked list. Synchronization occurs through the use of the @@ -1535,27 +1594,79 @@ destroy_device(int vid) vdev->vid);if (dma_bind[vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t n_pkt =0 ; -int pkts_inflight; -int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ ].dev_id; - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe (vid, VIRTIO_RXQ); -struct rte_mbuf *m_cpl[pkts_inflight]; - -while (pkts_inflight) { - n_pkt =rte_vhost_clear_queue_thread_unsafe (vid,VIRTIO_RXQ , - m_cpl, pkts_inflight, dma_id,0 ); -free_pkts (m_cpl, n_pkt); - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe (vid, -VIRTIO_RXQ ); - } - + vhost_clear_queue_thread_unsafe(vdev, VIRTIO_RXQ);rte_vhost_async_channel_unregister (vid,VIRTIO_RXQ ); dma_bind[vid].dmas[VIRTIO_RXQ ].async_enabled = false; } +if (dma_bind[vid].dmas[VIRTIO_TXQ ].async_enabled) { +vhost_clear_queue_thread_unsafe (vdev,VIRTIO_TXQ ); +rte_vhost_async_channel_unregister (vid,VIRTIO_TXQ ); + dma_bind[vid].dmas[VIRTIO_TXQ ].async_enabled = false; + } +rte_free (vdev); } +static int +get_socketid_by_vid (int vid) +{ +int i; +char ifname[PATH_MAX ]; +rte_vhost_get_ifname (vid, ifname,sizeof (ifname)); + +for (i =0 ; i < nb_sockets; i++) { +char *file = socket_files + i *PATH_MAX ; +if (strcmp (file, ifname) ==0 ) +return i; + } + +return -1 ; +} + +static int +init_vhost_queue_ops (int vid) +{ +if (builtin_net_driver) { + vdev_queue_ops[vid].enqueue_pkt_burst = builtin_enqueue_pkts; + vdev_queue_ops[vid].dequeue_pkt_burst = builtin_dequeue_pkts; + }else { +if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ ].async_enabled) + vdev_queue_ops[vid].enqueue_pkt_burst =
async_enqueue_pkts;examples/vhost/main.c +2
-0+else + vdev_queue_ops[vid].enqueue_pkt_burst =
sync_enqueue_pkts;examples/vhost/main.c +3
-0+ +if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ ].async_enabled) + vdev_queue_ops[vid].dequeue_pkt_burst =
async_dequeue_pkts;examples/vhost/main.c +2
-0+else + vdev_queue_ops[vid].dequeue_pkt_burst =
sync_dequeue_pkts;examples/vhost/main.c +11
-0+ } + +return 0 ; +} + +static int +vhost_async_channel_register (int vid) +{ +int rx_ret =0 , tx_ret =0 ; + + if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].dev_id !=
INVALID_DMA_ID) {examples/vhost/main.c +3
-0+ rx_ret =rte_vhost_async_channel_register (vid,VIRTIO_RXQ ); +if (rx_ret ==0 ) +dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled = true;examples/vhost/main.c +3
-0+ } + + if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].dev_id !=
INVALID_DMA_ID) {examples/vhost/main.c +3
-0+ tx_ret =rte_vhost_async_channel_register (vid,VIRTIO_TXQ ); +if (tx_ret ==0 ) +dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled = true;examples/vhost/main.c +7
-0+ } + +return rx_ret | tx_ret; +} + + +/* * A new device is added to a data core. First the device is added to the main linked list
* and then allocated to a specific data core.examples/vhost/main.c +2
-0@@ -1567,6 +1678,8 @@ new_device(int vid)uint16_t i;uint32_t device_num_min = num_devices;struct vhost_dev *vdev; +int ret; + vdev =rte_zmalloc ("vhost device" ,sizeof (*vdev),RTE_CACHE_LINE_SIZE );if (vdev ==NULL ) {examples/vhost/main.c +16
-27RTE_LOG(INFO, VHOST_DATA,@@ -1589,6 +1702,17 @@ new_device(int vid) } } +int socketid =get_socketid_by_vid (vid); +if (socketid ==-1 ) +return -1 ; + +init_vid2socketid_array (vid, socketid); + + ret =vhost_async_channel_register (vid); + +if (init_vhost_queue_ops (vid) !=0 ) +return -1 ; +if (builtin_net_driver)vs_vhost_net_setup (vdev);@@ -1620,16 +1744,7 @@ new_device(int vid)"(%d) device has been added to data core %d\n" , vid, vdev->coreid); -if (dma_bind[vid].dmas[VIRTIO_RXQ ].dev_id !=INVALID_DMA_ID ) { -int ret; - - ret =rte_vhost_async_channel_register (vid,VIRTIO_RXQ ); -if (ret ==0 ) - dma_bind[vid].dmas[VIRTIO_RXQ ].async_enabled = true; -return ret; - } - -return 0 ; +return ret; } static int@@ -1647,22 +1762,9 @@ vring_state_changed(int vid, uint16_t queue_id, int enable)if (queue_id !=VIRTIO_RXQ )return 0 ; -if (dma_bind[vid].dmas[queue_id].async_enabled) { -if (!enable) { -uint16_t n_pkt =0 ; -int pkts_inflight; - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe (vid, queue_id); -int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ ].dev_id; -struct rte_mbuf *m_cpl[pkts_inflight]; - -while (pkts_inflight) { - n_pkt =rte_vhost_clear_queue_thread_unsafe (vid, queue_id, - m_cpl, pkts_inflight, dma_id,0 ); -free_pkts (m_cpl, n_pkt); - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe (vid, - queue_id); - } - } +if (dma_bind[vid2socketid[vid]].dmas[queue_id].async_enabled) { +if (!enable) +vhost_clear_queue_thread_unsafe (vdev, queue_id); }return 0 ;@@ -1887,7 +1989,7 @@ main(int argc, char *argv[])for (i =0 ; i < nb_sockets; i++) {char *file = socket_files + i *PATH_MAX ; -if (dma_count) +if (dma_count &&get_async_flag_by_socketid (i) !=0 ) flags = flags |RTE_VHOST_USER_ASYNC_COPY ; ret =rte_vhost_driver_register (file, flags);examples/vhost/main.h +29
-3@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */ @@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */ examples/vhost/virtio_net.c +7
-0@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,examples/vhost/virtio_net.c +1
-1struct rte_mbuf *m, uint16_t desc_idx,@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, structrte_vhost_vring *vr, return 0; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_tcount)
{examples/vhost/virtio_net.c +4
-0@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id, return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_tcount)
examples/vhost/virtio_net.c +3
-1+{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +} --2.17.1
RE: [PATCH v6 5/5] examples/vhost: support async dequeue data path
Quoted 94 lines
-----Original Message-----
From: Xia, Chenbo <[email protected]>
Sent: Friday, May 13, 2022 11:27 AM
To: Ding, Xuan <[email protected]>; [email protected]
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: RE: [PATCH v6 5/5] examples/vhost: support async dequeue data pathQuoted 68 lines
-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Friday, May 13, 2022 10:51 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Ma, WenwuX <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v6 5/5] examples/vhost: support async dequeue data pathFrom: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
Signed-off-by Wenwu Ma <[email protected]>Signed-off-by Yuan Wang <[email protected]>Signed-off-by Xuan Ding <[email protected]>Tested-by Yvonne Yang <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 284 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 243 insertions(+), 98 deletions(-)doc/guides/sample_app_ug/vhost.rst +0
-0@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs.It's used in combination with dmas
**--dmas**
This parameter is used to specify the assigned DMA device of a vhost
device.
Async vhost-user net driver will be used if --dmas is set. For
example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation.
+--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] meansdoc/guides/sample_app_ug/vhost.rst +4
-0+use DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue +operation and use DMA channel 00:04.1/00:04.3 for vhost device 1 +enqueue/dequeue operation. The index of the device corresponds to the +socket file inorder,
doc/guides/sample_app_ug/vhost.rst +2
-0+that means vhost device 0 is created through the first socket file, +vhost device 1 is created through the second socket file, and so on. Common Issues -------------examples/vhost/main.c index renamed from examples/vhost/main.c +0
-0
c4d46de1c5..d070391727100644examples/vhost/main.c +3
-0@@ -63,6 +63,9 @@#define DMA_RING_SIZE 4096 +#define ASYNC_ENQUEUE_VHOST 1 +#define ASYNC_DEQUEUE_VHOST 2 +/* number of mbufs in all pools - if specified on command-line. */ static int total_num_mbufs =NUM_MBUFS_DEFAULT ;examples/vhost/main.c static uint32_t burst_rx_retry_num = · line 121
Quoted 3 lines
examples/vhost/main.c +1
-0@@ -116,6 +119,8 @@ static uint32_t burst_rx_retry_num =BURST_RX_RETRIES ;static char *socket_files;static int nb_sockets; +static struct vhost_queue_opsvdev_queue_ops[RTE_MAX_VHOST_DEVICE];
Quoted 9 lines
examples/vhost/main.c +1
-0+/* empty VMDq configuration structure. Filled in programmatically */ static struct rte_eth_conf vmdq_conf_default = { .rxmode = {@@ -205,6 +210,18 @@ struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE *RTE_MAX_VHOST_DEVICE ];#define MBUF_TABLE_DRAIN_TSC ((rte_get_tsc_hz() + US_PER_S - 1) \ / US_PER_S * BURST_TX_DRAIN_US)
Quoted 12 lines
examples/vhost/main.c struct vhost_bufftable *vhost_txbuff[RTE_MAX_LCORE * RTE_MAX_VHOST_DEVICE]; · line 223
Quoted 8 lines
examples/vhost/main.c +8
-0+static int vid2socketid[RTE_MAX_VHOST_DEVICE ]; + +static uint32_t get_async_flag_by_socketid (int socketid) { +return dma_bind[socketid].async_flag; } + +static void init_vid2socketid_array (int vid,int socketid) { + vid2socketid[vid] = socketid; +}Return value and func name should be on separate lines as per coding style. And above func can be inline, same suggestion for short func below, especially ones in data path.
Thanks Chenbo, will fix it in next version.
Regards,
Xuan
Quoted 568 lines
Thanks,
ChenboQuoted 61 lines
examples/vhost/main.c +19
-5+static inline bool is_dma_configured (int16_t dev_id) {@@ -224,7 +241,7 @@ open_dma(const char *value)char *addrs = input;char *ptrs[2 ];char *start, *end, *substr; -int64_t vid; +int64_t socketid, vring_id;struct rte_dma_info info; struct rte_dma_conf dev_config = { .nb_vchans = 1 }; @@ -262,7 +279,9 @@ open_dma(const char *value) while (i < args_nr) {char *arg_temp = dma_arg[i]; +char *txd, *rxd;uint8_t sub_nr; +int async_flag; sub_nr =rte_strsplit (arg_temp,strlen (arg_temp), ptrs,2 ,'@' );if (sub_nr !=2 ) {@@ -270,14 +289,23 @@ open_dma(const char *value) goto out; } - start =strstr (ptrs[0 ],"txd" ); -if (start ==NULL ) { + txd =strstr (ptrs[0 ],"txd" ); + rxd =strstr (ptrs[0 ],"rxd" ); +if (txd) { + start = txd; + vring_id =VIRTIO_RXQ ; + async_flag =ASYNC_ENQUEUE_VHOST ; + }else if (rxd) { + start = rxd; + vring_id =VIRTIO_TXQ ; + async_flag =ASYNC_DEQUEUE_VHOST ; + }else { ret =-1 ; goto out; } start +=3 ; - vid =strtol (start, &end,0 ); + socketid =strtol (start, &end,0 );if (end == start) { ret =-1 ; goto out;@@ -338,7 +366,8 @@ open_dma(const char *value) dmas_id[dma_count++] = dev_id; done: - (dma_info + vid)->dmas[VIRTIO_RXQ ].dev_id = dev_id; + (dma_info + socketid)->dmas[vring_id].dev_id = dev_id; + (dma_info + socketid)->async_flag |= async_flag; i++; } out:examples/vhost/main.c complete_async_pkts(struct vhost_dev *vdev) { · line 1025
Quoted 8 lines
examples/vhost/main.c +1
-1@@ -990,7 +1019,7 @@ complete_async_pkts(struct vhost_dev *vdev) {struct rte_mbuf *p_cpl[MAX_PKT_BURST ];uint16_t complete_count; -int16_t dma_id = dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].dev_id; +int16_t dma_id = dma_bind[vid2socketid[vdev- >vid]].dmas[VIRTIO_RXQ].dev_id; complete_count = rte_vhost_poll_enqueue_completed(vdev->vid, VIRTIO_RXQ, p_cpl, MAX_PKT_BURST,
dma_id, 0); @@ -1029,22examples/vhost/main.c drain_vhost(struct vhost_dev *vdev)
Quoted 13 lines
+1058,7 @@
drain_vhost(struct vhost_dev *vdev)examples/vhost/main.c +0
-5uint16_t nr_xmit = vhost_txbuff[buff_idx]->len;struct rte_mbuf **m = vhost_txbuff[buff_idx]->m_table; -if (builtin_net_driver) { - ret =vs_enqueue_pkts (vdev,VIRTIO_RXQ , m, nr_xmit); - }else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t enqueue_fail =0 ; - int16_t dma_id = dma_bind[vdev-vid].dmas[VIRTIO_RXQ].dev_id;
examples/vhost/main.c +0
-3- -complete_async_pkts (vdev); - ret = rte_vhost_submit_enqueue_burst(vdev->vid,
VIRTIO_RXQ, m,examples/vhost/main.c drain_vhost(struct vhost_dev *vdev)
Quoted 10 lines
nr_xmit,dma_id, 0);examples/vhost/main.c +1
-8- - enqueue_fail = nr_xmit - ret; -if (enqueue_fail) -free_pkts (&m[ret], nr_xmit - ret); - }else { - ret =rte_vhost_enqueue_burst (vdev->vid,VIRTIO_RXQ , - m, nr_xmit); - } + ret = vdev_queue_ops[vdev->vid].enqueue_pkt_burst(vdev,
VIRTIO_RXQ,examples/vhost/main.c drain_vhost(struct vhost_dev *vdev) · line 1064
Quoted 4 lines
m,
nr_xmit);examples/vhost/main.c +0
-0if (enable_stats) { __atomic_add_fetch(&vdev->stats.rx_total_atomic, nr_xmit,@@
Quoted 14 lines
-1053,7 +1067,7 @@
drain_vhost(struct vhost_dev *vdev)examples/vhost/main.c +1
-1__ATOMIC_SEQ_CST); } -if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) + if (!dma_bind[vid2socketid[vdev-Quoted 1 line
vid]].dmas[VIRTIO_RXQ].async_enabled)
examples/vhost/main.c +0
-0free_pkts (m, nr_xmit); }@@ -1325,6 +1339,32 @@ drain_mbuf_table(struct mbuf_table *tx_q) } }examples/vhost/main.c drain_mbuf_table(struct mbuf_table *tx_q) · line 1351
Quoted 10 lines
examples/vhost/main.c +9
-0+uint16_t +async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t rx_count) { +uint16_t enqueue_count; +uint16_t enqueue_fail =0 ; +uint16_t dma_id = dma_bind[vid2socketid[dev- >vid]].dmas[VIRTIO_RXQ].dev_id; + +complete_async_pkts (dev); + enqueue_count = rte_vhost_submit_enqueue_burst(dev->vid,
queue_id,Quoted 24 lines
examples/vhost/main.c +15
-2+ pkts, rx_count, dma_id, 0); + + enqueue_fail = rx_count - enqueue_count; +if (enqueue_fail) +free_pkts (&pkts[enqueue_count], enqueue_fail); + +return enqueue_count; +} + +uint16_t +sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t rx_count) { +return rte_vhost_enqueue_burst (dev->vid, queue_id, pkts, rx_count); +} + static __rte_always_inline voiddrain_eth_rx (struct vhost_dev *vdev) {@@ -1355,25 +1395,8 @@ drain_eth_rx(struct vhost_dev *vdev) } } -if (builtin_net_driver) { - enqueue_count = vs_enqueue_pkts(vdev, VIRTIO_RXQ,examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev) · line 1369
Quoted 10 lines
examples/vhost/main.c +0
-8- pkts, rx_count); - }else if (dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t enqueue_fail =0 ; -int16_t dma_id = dma_bind[vdev- vid].dmas[VIRTIO_RXQ].dev_id; - -complete_async_pkts (vdev); - enqueue_count = rte_vhost_submit_enqueue_burst(vdev- vid, - VIRTIO_RXQ, pkts, rx_count, dma_id,0);
examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev)
Quoted 7 lines
examples/vhost/main.c +0
-7- - enqueue_fail = rx_count - enqueue_count; -if (enqueue_fail) -free_pkts (&pkts[enqueue_count], enqueue_fail); - - }else { - enqueue_count = rte_vhost_enqueue_burst(vdev->vid,
VIRTIO_RXQ,examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev) · line 1402
Quoted 8 lines
examples/vhost/main.c +2
-2- pkts, rx_count); - } + enqueue_count = vdev_queue_ops[vdev- vid].enqueue_pkt_burst (vdev, +VIRTIO_RXQ , pkts, rx_count);examples/vhost/main.c +0
-0if (enable_stats) { __atomic_add_fetch(&vdev->stats.rx_total_atomic, rx_count,@@
Quoted 17 lines
-1382,10 +1405,31 @@
drain_eth_rx(struct vhost_dev *vdev)examples/vhost/main.c +1
-1__ATOMIC_SEQ_CST); } -if (!dma_bind[vdev->vid].dmas[VIRTIO_RXQ ].async_enabled) + if (!dma_bind[vid2socketid[vdev-Quoted 1 line
vid]].dmas[VIRTIO_RXQ].async_enabled)
examples/vhost/main.c +6
-0free_pkts (pkts, rx_count); } +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count) { +int nr_inflight; +uint16_t dequeue_count; + uint16_t dma_id = dma_bind[vid2socketid[dev-Quoted 1 line
vid]].dmas[VIRTIO_TXQ].dev_id;
examples/vhost/main.c
Quoted 2 lines
examples/vhost/main.c +2
-0+ + dequeue_count = rte_vhost_async_try_dequeue_burst(dev->vid,
queue_id,examples/vhost/main.c drain_eth_rx(struct vhost_dev *vdev)
Quoted 9 lines
examples/vhost/main.c +9
-0+ mbuf_pool, pkts, count, &nr_inflight, dma_id, 0); + +return dequeue_count; +} + +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count) { + return rte_vhost_dequeue_burst(dev->vid, queue_id, mbuf_pool,pkts,
Quoted 8 lines
count);
examples/vhost/main.c +2
-0+} + static __rte_always_inline void drain_virtio_tx(struct vhost_dev *vdev) { @@ -1393,13 +1437,8 @@
drain_virtio_tx(struct vhost_dev *vdev)examples/vhost/main.c +0
-0uint16_t count;uint16_t i;examples/vhost/main.c drain_virtio_tx(struct vhost_dev *vdev)
Quoted 10 lines
examples/vhost/main.c +2
-7-if (builtin_net_driver) { - count =vs_dequeue_pkts (vdev,VIRTIO_TXQ , mbuf_pool, - pkts,MAX_PKT_BURST ); - }else { - count =rte_vhost_dequeue_burst (vdev->vid,VIRTIO_TXQ , - mbuf_pool, pkts,MAX_PKT_BURST ); - } + count = vdev_queue_ops[vdev->vid].dequeue_pkt_burst(vdev, + VIRTIO_TXQ, mbuf_pool, pkts,
MAX_PKT_BURST);examples/vhost/main.c
Quoted 3 lines
examples/vhost/main.c +0
-0/* setup VMDq for the first packet */ if (unlikely (vdev->ready ==DEVICE_MAC_LEARNING ) && count){ @@
Quoted 21 lines
-1478,6 +1517,26 @@
switch_worker(void *arg __rte_unused)examples/vhost/main.c +7
-0return 0 ; } +static void +vhost_clear_queue_thread_unsafe (struct vhost_dev *vdev,uint16_t +queue_id) { +uint16_t n_pkt =0 ; +int pkts_inflight; + + int16_t dma_id = dma_bind[vid2socketid[vdev-Quoted 1 line
vid]].dmas[queue_id].dev_id;
examples/vhost/main.c +2
-0+ pkts_inflight = +rte_vhost_async_get_inflight_thread_unsafe(vdev->vid,
queue_id);examples/vhost/main.c +5
-0+ +struct rte_mbuf *m_cpl[pkts_inflight]; + +while (pkts_inflight) { + n_pkt = rte_vhost_clear_queue_thread_unsafe(vdev->vid,
queue_id,m_cpl,examples/vhost/main.c
Quoted 1 line
examples/vhost/main.c +1
-0+ pkts_inflight, dma_id,0);
Quoted 4 lines
examples/vhost/main.c +2
-0+free_pkts (m_cpl, n_pkt); + pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe(vdev->vid, +
queue_id);Quoted 7 lines
examples/vhost/main.c +3
-0+ } +} +/* * Remove a device from the specific data core linked list and from the * main linked list. Synchronization occurs through the use of the examples/vhost/main.c destroy_device(int vid) · line 1541
Quoted 7 lines
examples/vhost/main.c +0
-4@@ -1535,27 +1594,79 @@ destroy_device(int vid) vdev->vid);if (dma_bind[vid].dmas[VIRTIO_RXQ ].async_enabled) { -uint16_t n_pkt =0 ; -int pkts_inflight; -int16_t dma_id = dma_bind[vid].dmas[VIRTIO_RXQ ].dev_id; - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe(vid,
examples/vhost/main.c destroy_device(int vid) · line 1548
Quoted 7 lines
examples/vhost/main.c +0
-5VIRTIO_RXQ); -struct rte_mbuf *m_cpl[pkts_inflight]; - -while (pkts_inflight) { - n_pkt = rte_vhost_clear_queue_thread_unsafe(vid, VIRTIO_RXQ, - m_cpl, pkts_inflight, dma_id,0);
examples/vhost/main.c destroy_device(int vid) · line 1551
Quoted 4 lines
examples/vhost/main.c +0
-3-free_pkts (m_cpl, n_pkt); - pkts_inflight = rte_vhost_async_get_inflight_thread_unsafe(vid, -VIRTIO_RXQ);Quoted 27 lines
examples/vhost/main.c +18
-2- } - +vhost_clear_queue_thread_unsafe (vdev,VIRTIO_RXQ );rte_vhost_async_channel_unregister (vid,VIRTIO_RXQ ); dma_bind[vid].dmas[VIRTIO_RXQ ].async_enabled = false; } +if (dma_bind[vid].dmas[VIRTIO_TXQ ].async_enabled) { +vhost_clear_queue_thread_unsafe (vdev,VIRTIO_TXQ ); +rte_vhost_async_channel_unregister (vid,VIRTIO_TXQ ); + dma_bind[vid].dmas[VIRTIO_TXQ ].async_enabled = false; + } +rte_free (vdev); } +static int +get_socketid_by_vid (int vid) +{ +int i; +char ifname[PATH_MAX ]; +rte_vhost_get_ifname (vid, ifname,sizeof (ifname)); + +for (i =0 ; i < nb_sockets; i++) { +char *file = socket_files + i *PATH_MAX ; +if (strcmp (file, ifname) ==0 ) +return i;examples/vhost/main.c destroy_device(int vid)
Quoted 10 lines
examples/vhost/main.c +10
-0+ } + +return -1 ; +} + +static int +init_vhost_queue_ops (int vid) +{ +if (builtin_net_driver) { + vdev_queue_ops[vid].enqueue_pkt_burst =
builtin_enqueue_pkts;examples/vhost/main.c destroy_device(int vid) · line 1635
Quoted 1 line
examples/vhost/main.c +1
-0+ vdev_queue_ops[vid].dequeue_pkt_burst =
builtin_dequeue_pkts;examples/vhost/main.c destroy_device(int vid) · line 1637
Quoted 2 lines
examples/vhost/main.c +2
-0+ }else { + if(dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled)
examples/vhost/main.c
Quoted 7 lines
examples/vhost/main.c +1
-0+ vdev_queue_ops[vid].enqueue_pkt_burst =
async_enqueue_pkts;examples/vhost/main.c +2
-0+else + vdev_queue_ops[vid].enqueue_pkt_burst =
sync_enqueue_pkts;examples/vhost/main.c +2
-0+ + if(dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled)
Quoted 19 lines
examples/vhost/main.c +1
-0+ vdev_queue_ops[vid].dequeue_pkt_burst =
async_dequeue_pkts;examples/vhost/main.c +2
-0+else + vdev_queue_ops[vid].dequeue_pkt_burst =
sync_dequeue_pkts;examples/vhost/main.c +10
-0+ } + +return 0 ; +} + +static int +vhost_async_channel_register (int vid) { +int rx_ret =0 , tx_ret =0 ; + + if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].dev_id !=
INVALID_DMA_ID) {examples/vhost/main.c +3
-0+ rx_ret =rte_vhost_async_channel_register (vid,VIRTIO_RXQ ); +if (rx_ret ==0 ) +Quoted 1 line
dma_bind[vid2socketid[vid]].dmas[VIRTIO_RXQ].async_enabled =
true;
Quoted 8 lines
examples/vhost/main.c +3
-0+ } + + if (dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].dev_id !=
INVALID_DMA_ID) {examples/vhost/main.c +3
-0+ tx_ret =rte_vhost_async_channel_register (vid,VIRTIO_TXQ ); +if (tx_ret ==0 ) +dma_bind[vid2socketid[vid]].dmas[VIRTIO_TXQ].async_enabled =
true;
Quoted 40 lines
examples/vhost/main.c +7
-0+ } + +return rx_ret | tx_ret; +} + + +/* * A new device is added to a data core. First the device is added to the main linked list
* and then allocated to a specific data core.examples/vhost/main.c +2
-0@@ -1567,6 +1678,8 @@ new_device(int vid)uint16_t i;uint32_t device_num_min = num_devices;struct vhost_dev *vdev; +int ret; + vdev =rte_zmalloc ("vhost device" ,sizeof (*vdev),RTE_CACHE_LINE_SIZE );if (vdev ==NULL ) {examples/vhost/main.c +11
-0RTE_LOG(INFO, VHOST_DATA,@@ -1589,6 +1702,17 @@ new_device(int vid) } } +int socketid =get_socketid_by_vid (vid); +if (socketid ==-1 ) +return -1 ; + +init_vid2socketid_array (vid, socketid); + + ret =vhost_async_channel_register (vid); + +if (init_vhost_queue_ops (vid) !=0 ) +return -1 ; +if (builtin_net_driver)vs_vhost_net_setup (vdev);examples/vhost/main.c new_device(int vid) · line 1628
Quoted 9 lines
examples/vhost/main.c +0
-6@@ -1620,16 +1744,7 @@ new_device(int vid)"(%d) device has been added to data core %d\n" , vid, vdev->coreid); -if (dma_bind[vid].dmas[VIRTIO_RXQ ].dev_id !=INVALID_DMA_ID ) { -int ret; - - ret =rte_vhost_async_channel_register (vid,VIRTIO_RXQ ); -if (ret ==0 ) - dma_bind[vid].dmas[VIRTIO_RXQ].async_enabled =true;
Quoted 20 lines
examples/vhost/main.c +1
-4-return ret; - } - -return 0 ; +return ret; } static int@@ -1647,22 +1762,9 @@ vring_state_changed(int vid, uint16_t queue_id,int
enable)examples/vhost/main.c +0
-5if (queue_id !=VIRTIO_RXQ )return 0 ; -if (dma_bind[vid].dmas[queue_id].async_enabled) { -if (!enable) { -uint16_t n_pkt =0 ; -int pkts_inflight; - pkts_inflight =
rte_vhost_async_get_inflight_thread_unsafe(vid, queue_id);examples/vhost/main.c
Quoted 1 line
examples/vhost/main.c +0
-1- int16_t dma_id =dma_bind[vid].dmas[VIRTIO_RXQ].dev_id;
examples/vhost/main.c destroy_device(int vid)
Quoted 4 lines
examples/vhost/main.c +0
-4-struct rte_mbuf *m_cpl[pkts_inflight]; - -while (pkts_inflight) { - n_pkt =rte_vhost_clear_queue_thread_unsafe(vid,
examples/vhost/main.c
Quoted 2 lines
queue_id,examples/vhost/main.c +0
-1- m_cpl, pkts_inflight,
dma_id, 0);Quoted 117 lines
examples/vhost/main.c +0
-2-free_pkts (m_cpl, n_pkt); - pkts_inflight =rte_vhost_async_get_inflight_thread_unsafe(vid,
-queue_id);examples/vhost/main.c +4
-3- } - } +if (dma_bind[vid2socketid[vid]].dmas[queue_id].async_enabled) { +if (!enable) +vhost_clear_queue_thread_unsafe (vdev, queue_id); }return 0 ;@@ -1887,7 +1989,7 @@ main(int argc, char *argv[])for (i =0 ; i < nb_sockets; i++) {char *file = socket_files + i *PATH_MAX ; -if (dma_count) +if (dma_count &&get_async_flag_by_socketid (i) !=0 ) flags = flags |RTE_VHOST_USER_ASYNC_COPY ; ret = rte_vhost_driver_register(file, flags); diff --git
a/examples/vhost/main.hb/examples/vhost/main.hindexe7f395c3c9..2fcb8376c5100644examples/vhost/main.h +13
-0@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; }; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; }; /* we implement non-extra virtio net features */ @@ -97,7 +111,19 @@void
vs_vhost_net_remove(struct vhost_dev *dev);uint16_tvs_enqueue_pkts(structvhost_dev*dev,uint16_tqueue_id,examples/vhost/main.h +15
-3struct rte_mbuf **pkts, uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count);uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);uint16_t +sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count);uint16_t +sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);uint16_t +async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count);uint16_t +async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */ examples/vhost/virtio_net.c +5
-0@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id, return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) { +return vs_enqueue_pkts (dev, queue_id, pkts, count); } + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx, @@ -363,7 +370,7 @@dequeue_pkt(struct
vhost_dev*dev, structrte_vhost_vring*vr,examples/vhost/virtio_net.c +1
-1return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_tcount)
{examples/vhost/virtio_net.c +4
-0@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id, return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_tcount)
examples/vhost/virtio_net.c +2
-1+{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); } --2.17.1
v7 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v6->v7:
* correct code formatting
* change some functions to inline
v5->v6:
* adjust EXPERIMENTAL header
v4->v5:
* rebase to latest DPDK
* add some checks
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 286 ++++++++++----- examples/vhost/main.h | 32 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 37 ++ lib/vhost/version.map | 2 +- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 473 ++++++++++++++++++++++--- 10 files changed, 716 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
PATCH v7 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH v7 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH v7 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -180,6 +180,7 @@ struct async_inflight_info {struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */ +struct virtio_net_hdr nethdr; };struct vhost_async {
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v7 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 37 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 387 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +7 -0
@@ -282,6 +282,13 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)`` + + Receives (dequeues) ``count`` packets from guest to host in async data path, + and stored them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------
lib/vhost/rte_vhost_async.h +37 -0
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* @warning +* @b EXPERIMENTAL: this API may change, or be removed, without prior notice +* +* This function tries to receive packets from the guest with offloading +* copies to the async channel. The packets that are transfer completed +* are returned in "pkts". The other packets that their copies are submitted to +* the async channel but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the async channel. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* >= 0: The amount of in-flight packets +* -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,uint16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +1 -1
@@ -90,7 +90,7 @@ EXPERIMENTAL { # added in 22.07 rte_vhost_async_get_inflight_thread_unsafe; - + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +337 -0
340 lines not on this page · raw message
Signature · 1 line
2.17.1
RE: [PATCH v7 4/5] vhost: support async dequeue for split ring
Hi Xuan,
Quoted 38 lines
-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Monday, May 16, 2022 10:43 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v7 4/5] vhost: support async dequeue for split ringFrom: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API
rte_vhost_async_try_dequeue_burst()is introduced.Signed-off-by Xuan Ding <[email protected]>Signed-off-by Yuan Wang <[email protected]>Tested-by Yvonne Yang <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 37 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 387 insertions(+), 1 deletion(-)doc/guides/prog_guide/vhost_lib.rst +0
-0@@ -282,6 +282,13 @@ The following is an overview of some key Vhost APIfunctions:
Clear inflight packets which are submitted to DMA engine in vhost async
data
path. Completed packets are returned to applications through ``pkts``.
doc/guides/prog_guide/vhost_lib.rst The following is an overview of some key Vhost API functions: · line 287
Quoted 3 lines
doc/guides/prog_guide/vhost_lib.rst +3
-0+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, + int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``
In dmadev library, dma_id is int16_t, rather than uint16_t.
Quoted 32 lines
doc/guides/prog_guide/vhost_lib.rst +4
-0+ + Receives (dequeues) ``count`` packets from guest to host in async + data path, and stored them at ``pkts``. + Vhost-user Implementations --------------------------doc/guides/rel_notes/release_22_07.rst +5
-0@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------lib/vhost/rte_vhost_async.h index renamed from lib/vhost/rte_vhost_async.h +0
-0
70234debf9..2789492e38100644lib/vhost/rte_vhost_async.h +0
-0@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(intvid,
uint16_tqueue_id,__rte_experimentalintrte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
lib/vhost/rte_vhost_async.h uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id, · line 212
Quoted 8 lines
lib/vhost/rte_vhost_async.h +8
-0+/** +* @warning +* @b EXPERIMENTAL: this API may change, or be removed, without prior +notice +* +* This function tries to receive packets from the guest with +offloading +* copies to the async channel. The packets that are transfer completed
In rte_vhost_async.h, DMA vChannel is referred as async channel or async copy engine. But I think it's better to replace all with DMA vChannel to be consistent with DPDK.
Quoted 9 lines
+ * are returned in "pkts". The other packets that their copies are
lib/vhost/rte_vhost_async.h +8
-0+submitted to + * the async channel but not completed are called "in-flight packets". + * This function will not return in-flight packets until their copies +are + * completed by the async channel. + * + * @param vid + * ID of vhost device to dequeue data
lib/vhost/rte_vhost_async.h uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id, · line 229
Quoted 10 lines
lib/vhost/rte_vhost_async.h +10
-0+* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* >= 0: The amount of in-flight packets
Better to add more descriptions about the meaning of "nr_inflight".
Thanks,
Jiayu
RE: [PATCH v7 4/5] vhost: support async dequeue for split ring
Hi Jiayu,
Quoted 51 lines
-----Original Message-----
From: Hu, Jiayu <[email protected]>
Sent: Monday, May 16, 2022 1:53 PM
To: Ding, Xuan <[email protected]>; [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>
Subject: RE: [PATCH v7 4/5] vhost: support async dequeue for split ringHi Xuan,
Quoted 39 lines
-----Original Message-----
From: Ding, Xuan <[email protected]>
Sent: Monday, May 16, 2022 10:43 AM
To: [email protected]; Xia, Chenbo <[email protected]>
Cc: [email protected]; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Ding, Xuan <[email protected]>; Wang, YuanX <[email protected]>
Subject: [PATCH v7 4/5] vhost: support async dequeue for split ringFrom: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API
rte_vhost_async_try_dequeue_burst()is introduced.Signed-off-by Xuan Ding <[email protected]>Signed-off-by Yuan Wang <[email protected]>Tested-by Yvonne Yang <[email protected]>Reviewed-by Maxime Coquelin <[email protected]>--- doc/guides/prog_guide/vhost_lib.rst | 7 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 37 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 387 insertions(+), 1 deletion(-)doc/guides/prog_guide/vhost_lib.rst +0
-0@@ -282,6 +282,13 @@ The following is an overview of some key VhostAPI
functions:
Clear inflight packets which are submitted to DMA engine in vhost
async data
path. Completed packets are returned to applications through ``pkts``.
Quoted 5 lines
doc/guides/prog_guide/vhost_lib.rst
Quoted 3 lines
doc/guides/prog_guide/vhost_lib.rst +3
-0+* ``rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, + struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t +count, int *nr_inflight, uint16_t dma_id, uint16_t vchan_id)``In dmadev library,
dma_idisint16_t, rather thanuint16_t.
Thanks for the reminder. I will fix it in v8.
Quoted 35 lines
lib/vhost/rte_vhost_async.h uint16_t
Quoted 31 lines
doc/guides/prog_guide/vhost_lib.rst +4
-0+ + Receives (dequeues) ``count`` packets from guest to host in async + data path, and stored them at ``pkts``. + Vhost-user Implementations --------------------------doc/guides/rel_notes/release_22_07.rst +5
-0@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------lib/vhost/rte_vhost_async.h +0
-0index
70234debf9..2789492e38100644lib/vhost/rte_vhost_async.h +0
-0@@ -204,6 +204,43 @@ uint16_trte_vhost_clear_queue_thread_unsafe(int
Quoted 3 lines
vid,
uint16_tqueue_id,__rte_experimentalintrte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id);
Quoted 13 lines
lib/vhost/rte_vhost_async.h
Quoted 9 lines
lib/vhost/rte_vhost_async.h +9
-0+/** +* @warning +* @b EXPERIMENTAL: this API may change, or be removed, without prior +notice +* +* This function tries to receive packets from the guest with +offloading +* copies to the async channel. The packets that are transfer +completed In
rte_vhost_async.h, DMA vChannel is referred as async channel or async copy engine. But I think it's better to replace all with DMA vChannel to be consistent with DPDK.
Thanks for your suggestion, I will refine the API description in next version.
Quoted 14 lines
Quoted 14 lines
+ * are returned in "pkts". The other packets that their copies are
lib/vhost/rte_vhost_async.h +12
-0+submitted to + * the async channel but not completed are called "in-flight packets". + * This function will not return in-flight packets until their copies +are + * completed by the async channel. + * + * @param vid + * ID of vhost device to dequeue data + * @param queue_id + * ID of virtqueue to dequeue data + * @param mbuf_pool + * Mbuf_pool where host mbuf is allocated
Quoted 8 lines
Quoted 6 lines
+ * @param pkts
+ * Blank array to keep successfully dequeued packets
+ * @param count
+ * Size of the packet array
+ * @paramnr_inflight
+ * >= 0: The amount of in-flight packetsBetter to add more descriptions about the meaning of "
nr_inflight".
Same as above.
Regards,
Xuan
Quoted 3 lines
Thanks,
Jiayu
PATCH v7 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 286 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 245 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +195 -91
428 lines not on this page · raw message
examples/vhost/main.h +29 -3
@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
v8 cover vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid, uint16_t queue_id, struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count, int *nr_inflight,
uint16_t dma_id, uint16_t vchan_id)
Receive packets from the guest and offloads copies to DMA
virtual channel.
[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v7->v8:
* change dma_id to int16_t
* refine API documentation
v6->v7:
* correct code formatting
* change some functions to inline
v5->v6:
* adjust EXPERIMENTAL header
v4->v5:
* rebase to latest DPDK
* add some checks
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead of queue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug in desc_to_mbuf filling
RFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docs
RFC v2 -> RFC v3:
* rebase to latest DPDK version
RFC v1 -> RFC v2:
* fix one bug in example
* rename vchan to vchan_id
* check if dma_id and vchan_id valid
* rework all the logs to new standard
Xuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
doc/guides/prog_guide/vhost_lib.rst | 6 + doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 286 ++++++++++----- examples/vhost/main.h | 32 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 37 ++ lib/vhost/version.map | 2 +- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 473 ++++++++++++++++++++++--- 10 files changed, 715 insertions(+), 152 deletions(-)
Signature · 1 line
2.17.1
PATCH v8 1/5 vhost: prepare sync for descriptor to mbuf refactoring
From: Xuan Ding <[email protected]>
This patch extracts the descriptors to buffers filling from copy_desc_to_mbuf() into a dedicated function. Besides, enqueue and dequeue path are refactored to use the same function sync_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 78 ++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 40 deletions(-)
lib/vhost/virtio_net.c +38 -40
@@ -1030,23 +1030,36 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq, } static __rte_always_inline void -sync_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +sync_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_addr,uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct batch_copy_elem *batch_copy = vq->batch_copy_elems;if (likely (cpy_len >MAX_BATCH_LEN || vq->batch_copy_nb_elems >= vq->size)) { -rte_memcpy ((void *)((uintptr_t )(buf_addr)), +if (to_desc) { +rte_memcpy ((void *)((uintptr_t )(buf_addr)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); + }else { + rte_memcpy(rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), + (void *)((uintptr_t)(buf_addr)), + cpy_len); + }vhost_log_cache_write_iova (dev, vq, buf_iova, cpy_len);PRINT_PACKET (dev, (uintptr_t )(buf_addr), cpy_len,0 ); }else { - batch_copy[vq->batch_copy_nb_elems].dst = - (void *)((uintptr_t )(buf_addr)); - batch_copy[vq->batch_copy_nb_elems].src = - rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); +if (to_desc) { + batch_copy[vq->batch_copy_nb_elems].dst = + (void *)((uintptr_t )(buf_addr)); + batch_copy[vq->batch_copy_nb_elems].src = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + }else { + batch_copy[vq->batch_copy_nb_elems].dst = + rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); + batch_copy[vq->batch_copy_nb_elems].src = + (void *)((uintptr_t )(buf_addr)); + } batch_copy[vq->batch_copy_nb_elems].log_addr = buf_iova; batch_copy[vq->batch_copy_nb_elems].len = cpy_len; vq->batch_copy_nb_elems++;
@@ -1158,9 +1171,9 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_iova + buf_offset, cpy_len) < 0) goto error; }else { -sync_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_addr + buf_offset, - buf_iova + buf_offset, cpy_len); +sync_fill_seg (dev, vq, m, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, true); } mbuf_avail -= cpy_len;
@@ -2473,8 +2486,8 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, struct rte_mbuf *m, struct rte_mempool *mbuf_pool, bool legacy_ol_flags) { - uint32_t buf_avail, buf_offset; -uint64_t buf_addr, buf_len; + uint32_t buf_avail, buf_offset, buf_len; +uint64_t buf_addr, buf_iova;uint32_t mbuf_avail, mbuf_offset;uint32_t cpy_len;struct rte_mbuf *cur = m, *prev = m;
@@ -2482,16 +2495,13 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,struct virtio_net_hdr *hdr =NULL ;/* A counter to avoid desc dead loop chain */ uint16_t vec_idx =0 ; -struct batch_copy_elem *batch_copy = vq->batch_copy_elems; -int error =0 ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; -if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) { - error =-1 ; - goto out; - } +if (unlikely (buf_len < dev->vhost_hlen && nr_vec <=1 )) +return -1 ;if (virtio_net_with_host_offload (dev)) {if (unlikely (buf_len <sizeof (struct virtio_net_hdr ))) {
@@ -2515,11 +2525,12 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, buf_offset = dev->vhost_hlen - buf_len; vec_idx++; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_avail = buf_len - buf_offset; }else if (buf_len == dev->vhost_hlen) {if (unlikely (++vec_idx >= nr_vec)) - goto out; + goto error; buf_addr = buf_vec[vec_idx].buf_addr; buf_len = buf_vec[vec_idx].buf_len;
@@ -2539,22 +2550,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,while (1 ) { cpy_len =RTE_MIN (buf_avail, mbuf_avail); -if (likely (cpy_len >MAX_BATCH_LEN || - vq->batch_copy_nb_elems >= vq->size || - (hdr && cur == m))) { -rte_memcpy (rte_pktmbuf_mtod_offset (cur ,void *, - mbuf_offset), - (void *)((uintptr_t)(buf_addr + - buf_offset)), cpy_len); - }else { - batch_copy[vq->batch_copy_nb_elems].dst = - rte_pktmbuf_mtod_offset(cur, void *, - mbuf_offset); - batch_copy[vq->batch_copy_nb_elems].src = - (void *)((uintptr_t )(buf_addr + buf_offset)); - batch_copy[vq->batch_copy_nb_elems].len = cpy_len; - vq->batch_copy_nb_elems++; - } +sync_fill_seg (dev, vq, cur, mbuf_offset, + buf_addr + buf_offset, + buf_iova + buf_offset, cpy_len, false); mbuf_avail -= cpy_len; mbuf_offset += cpy_len;
@@ -2567,6 +2565,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,break ; buf_addr = buf_vec[vec_idx].buf_addr; + buf_iova = buf_vec[vec_idx].buf_iova; buf_len = buf_vec[vec_idx].buf_len; buf_offset =0 ;
@@ -2585,8 +2584,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (unlikely (cur ==NULL )) {VHOST_LOG_DATA (ERR ,"(%s) failed to allocate memory for mbuf.\n" , dev->ifname); - error =-1 ; - goto out; + goto error; } prev->next = cur;
@@ -2606,9 +2604,9 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq,if (hdr)vhost_dequeue_offload (dev, hdr, m, legacy_ol_flags); -out: - -return error; +return 0 ; +error: +return -1 ; } static void
Signature · 1 line
2.17.1
PATCH v8 2/5 vhost: prepare async for descriptor to mbuf refactoring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors vhost async enqueue path and dequeue path to use the same function async_fill_seg() for preparing batch elements, which simplifies the code without performance degradation.
--- lib/vhost/virtio_net.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-)
lib/vhost/virtio_net.c +14 -8
@@ -997,13 +997,14 @@ async_iter_reset(struct vhost_async *async) } static __rte_always_inline int -async_mbuf_to_desc_seg (struct virtio_net *dev,struct vhost_virtqueue *vq, +async_fill_seg (struct virtio_net *dev,struct vhost_virtqueue *vq,struct rte_mbuf *m,uint32_t mbuf_offset, -uint64_t buf_iova,uint32_t cpy_len) +uint64_t buf_iova,uint32_t cpy_len,bool to_desc) {struct vhost_async *async = vq->async;uint64_t mapped_len;uint32_t buf_offset =0 ; +void *src, *dst;void *host_iova;while (cpy_len) {
@@ -1015,10 +1016,15 @@ async_mbuf_to_desc_seg(struct virtio_net *dev, struct vhost_virtqueue *vq,return -1 ; } -if (unlikely (async_iter_add_iovec (dev, async, - (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, - mbuf_offset), - host_iova, (size_t )mapped_len))) +if (to_desc) { + src = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + dst = host_iova; + }else { + src = host_iova; + dst = (void *)(uintptr_t )rte_pktmbuf_iova_offset (m, mbuf_offset); + } + +if (unlikely (async_iter_add_iovec (dev, async, src, dst, (size_t )mapped_len)))return -1 ; cpy_len -= (uint32_t )mapped_len;
@@ -1167,8 +1173,8 @@ mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, cpy_len =RTE_MIN (buf_avail, mbuf_avail);if (is_async) { -if (async_mbuf_to_desc_seg (dev, vq, m, mbuf_offset, - buf_iova + buf_offset, cpy_len) <0 ) +if (async_fill_seg (dev, vq, m, mbuf_offset, + buf_iova + buf_offset, cpy_len, true) <0 ) goto error; }else { sync_fill_seg(dev, vq, m, mbuf_offset,
Signature · 1 line
2.17.1
PATCH v8 3/5 vhost: merge sync and async descriptor to mbuf filling
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch refactors copy_desc_to_mbuf() used by the sync path to support both sync and async descriptor to mbuf filling.
--- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 48 ++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 11 deletions(-)
lib/vhost/vhost.h +1 -0
@@ -180,6 +180,7 @@ struct async_inflight_info {struct rte_mbuf *mbuf;uint16_t descs;/* num of descs inflight */ uint16_t nr_buffers;/* num of buffers inflight for packed ring */ +struct virtio_net_hdr nethdr; };struct vhost_async {
lib/vhost/virtio_net.c +37 -11
104 lines not on this page · raw message
Signature · 1 line
2.17.1
PATCH v8 4/5 vhost: support async dequeue for split ring
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch implements asynchronous dequeue data path for vhost split ring, a new API rte_vhost_async_try_dequeue_burst() is introduced.
--- doc/guides/prog_guide/vhost_lib.rst | 6 + doc/guides/rel_notes/release_22_07.rst | 5 + lib/vhost/rte_vhost_async.h | 37 +++ lib/vhost/version.map | 2 +- lib/vhost/virtio_net.c | 337 +++++++++++++++++++++++++ 5 files changed, 386 insertions(+), 1 deletion(-)
doc/guides/prog_guide/vhost_lib.rst +6 -0
@@ -282,6 +282,12 @@ The following is an overview of some key Vhost API functions: Clear inflight packets which are submitted to DMA engine in vhost async data path. Completed packets are returned to applications through ``pkts``. +* ``rte_vhost_async_try_dequeue_burst(vid, queue_id, mbuf_pool, pkts, count, + nr_inflight, dma_id, vchan_id)`` + + Receive ``count`` packets from guest to host in async data path, + and store them at ``pkts``. + Vhost-user Implementations --------------------------
doc/guides/rel_notes/release_22_07.rst +5 -0
@@ -70,6 +70,11 @@ New Features Added an API which can get the number of inflight packets in vhost async data path without using lock. +* **Added vhost async dequeue API to receive pkts from guest.** + + Added vhost async dequeue API which can leverage DMA devices to + accelerate receiving pkts from guest. + Removed Items -------------
lib/vhost/rte_vhost_async.h +37 -0
@@ -204,6 +204,43 @@ uint16_t rte_vhost_clear_queue_thread_unsafe(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_dma_configure(int16_t dma_id, uint16_t vchan_id); +/** +* @warning +* @b EXPERIMENTAL: this API may change, or be removed, without prior notice +* +* This function tries to receive packets from the guest with offloading +* copies to the DMA vChannels. Successfully dequeued packets are returned +* in "pkts". The other packets that their copies are submitted to +* the DMA vChannels but not completed are called "in-flight packets". +* This function will not return in-flight packets until their copies are +* completed by the DMA vChannels. +* +* @param vid +* ID of vhost device to dequeue data +* @param queue_id +* ID of virtqueue to dequeue data +* @param mbuf_pool +* Mbuf_pool where host mbuf is allocated +* @param pkts +* Blank array to keep successfully dequeued packets +* @param count +* Size of the packet array +* @param nr_inflight +* >= 0: The amount of in-flight packets +* -1: Meaningless, indicates failed lock acquisition or invalid queue_id/dma_id +* @param dma_id +* The identifier of DMA device +* @param vchan_id +* The identifier of virtual DMA channel +* @return +* Number of successfully dequeued packets +*/ +__rte_experimental +uint16_t +rte_vhost_async_try_dequeue_burst (int vid,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int *nr_inflight,int16_t dma_id,uint16_t vchan_id); + #ifdef __cplusplus }#endif
lib/vhost/version.map +1 -1
@@ -90,7 +90,7 @@ EXPERIMENTAL { # added in 22.07 rte_vhost_async_get_inflight_thread_unsafe; - + rte_vhost_async_try_dequeue_burst; }; INTERNAL {
lib/vhost/virtio_net.c +337 -0
340 lines not on this page · raw message
Signature · 1 line
2.17.1
Re: [PATCH v8 4/5] vhost: support async dequeue for split ring
Quoted 5 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
lib/vhost/virtio_net.c +5
-0+static __rte_always_inline uint16_t +virtio_dev_tx_async_split (struct virtio_net *dev,struct vhost_virtqueue *vq, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int16_t dma_id,uint16_t vchan_id,bool legacy_ol_flags) +{
lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+static bool allocerr_warned; +bool dropped = false; +uint16_t free_entries; +uint16_t pkt_idx, slot_idx =0 ; +uint16_t nr_done_pkts =0 ; +uint16_t pkt_err =0 ; +uint16_t n_xfer; +struct vhost_async *async = vq->async; +struct async_inflight_info *pkts_info = async->pkts_info; +struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST ];
Why do we need this array?
Plus, see blow.
Quoted 11 lines
lib/vhost/virtio_net.c +11
-0+uint16_t pkts_size = count; + +/** +* The ordering between avail index and +* desc reads needs to be enforced. +*/ + free_entries =__atomic_load_n (&vq->avail->idx, __ATOMIC_ACQUIRE) - + vq->last_avail_idx; +if (free_entries ==0 ) + goto out; +
lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3252
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+rte_prefetch0 (&vq->avail->ring[vq->last_avail_idx & (vq->size -1 )]); + +async_iter_reset (async); + + count =RTE_MIN (count,MAX_PKT_BURST ); + count =RTE_MIN (count, free_entries); +VHOST_LOG_DATA (DEBUG ,"(%s) about to dequeue %u buffers\n" , + dev->ifname, count); + +if (rte_pktmbuf_alloc_bulk (mbuf_pool, pkts_prealloc, count))
'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than MAX_PKT_BURST.
Calling rte_pktmbuf_alloc_bulk on a fixed-size array pkts_prealloc, allocated on the stack, it may cause a stack overflow.
This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.
I point here another bug.
There are probably more...
<grmbl>
I don't like how async code has been added in the vhost library by Intel.
Maxime did a cleanup on the enqueue patch
https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&archive=both.
I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>
Signature · 1 line
David Marchand
Re: [PATCH v8 4/5] vhost: support async dequeue for split ring
Quoted 21 lines · David Marchand
Quoted 5 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
lib/vhost/virtio_net.c +5
-0+static __rte_always_inline uint16_t +virtio_dev_tx_async_split (struct virtio_net *dev,struct vhost_virtqueue *vq, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count, +int16_t dma_id,uint16_t vchan_id,bool legacy_ol_flags) +{lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+static bool allocerr_warned; +bool dropped = false; +uint16_t free_entries; +uint16_t pkt_idx, slot_idx =0 ; +uint16_t nr_done_pkts =0 ; +uint16_t pkt_err =0 ; +uint16_t n_xfer; +struct vhost_async *async = vq->async; +struct async_inflight_info *pkts_info = async->pkts_info; +struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST ];Why do we need this array?
Plus, see blow.
Quoted 16 lines
Quoted 6 lines
lib/vhost/virtio_net.c +6
-0+uint16_t pkts_size = count; + +/** +* The ordering between avail index and +* desc reads needs to be enforced. +*/ lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3247
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+ free_entries =__atomic_load_n (&vq->avail->idx, __ATOMIC_ACQUIRE) - + vq->last_avail_idx; +if (free_entries ==0 ) + goto out; + +rte_prefetch0 (&vq->avail->ring[vq->last_avail_idx & (vq->size -1 )]); + +async_iter_reset (async); + + count =RTE_MIN (count,MAX_PKT_BURST );
^^^
Ok, my point about the overflow does not stand.
Just the pkts_prealloc array is probably useless.
Quoted 11 lines
lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3252
Quoted 5 lines
lib/vhost/virtio_net.c +5
-0+ count =RTE_MIN (count, free_entries); +VHOST_LOG_DATA (DEBUG ,"(%s) about to dequeue %u buffers\n" , + dev->ifname, count); + +if (rte_pktmbuf_alloc_bulk (mbuf_pool, pkts_prealloc, count))'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than
MAX_PKT_BURST.Calling
rte_pktmbuf_alloc_bulkon a fixed-size arraypkts_prealloc, allocated on the stack, it may cause a stack overflow.
The rest still stands for me.
vvv
Quoted 17 lines
This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.
I point here another bug.
There are probably more...<grmbl>
I don't like how async code has been added in the vhost library by Intel.Maxime did a cleanup on the enqueue patch
https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&archive=both.
I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>
Signature · 1 line
David Marchand
RE: [PATCH v8 4/5] vhost: support async dequeue for split ring
Hi David,
Quoted 44 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, June 16, 2022 10:40 PM
To: Ding, Xuan <[email protected]>
Cc: Maxime Coquelin <[email protected]>; Xia, Chenbo <[email protected]>; dev <[email protected]>; Hu, Jiayu <[email protected]>; Jiang, Cheng1 <[email protected]>; Pai G, Sunil <[email protected]>; [email protected]; Wang, YuanX <[email protected]>; Mcnamara, John <[email protected]>
Subject: Re: [PATCH v8 4/5] vhost: support async dequeue for split ringQuoted 4 lines · David Marchand
Review
Quoted 2 lines · On Mon, May 16, 2022 at 1:16 PM <[email protected]> wrote:
(quoted diff) +2
-0+static __rte_always_inline uint16_t +virtio_dev_tx_async_split(struct virtio_net *dev, struct vhost_virtqueue*vq,
Quoted 1 line
Quoted 1 line
+ struct
rte_mempool*mbuf_pool, structrte_mbuf**pkts,
uint16_tcount,Quoted 13 lines
lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3231
Quoted 12 lines
lib/vhost/virtio_net.c +12
-0+ int16_t dma_id, uint16_t vchan_id, bool +legacy_ol_flags) { + static bool allocerr_warned; +bool dropped = false; +uint16_t free_entries; +uint16_t pkt_idx, slot_idx =0 ; +uint16_t nr_done_pkts =0 ; +uint16_t pkt_err =0 ; +uint16_t n_xfer; +struct vhost_async *async = vq->async; +struct async_inflight_info *pkts_info = async->pkts_info; +struct rte_mbuf *pkts_prealloc[MAX_PKT_BURST ];Quoted 10 lines
Why do we need this array?
Plus, see blow.Quoted 7 lines
lib/vhost/virtio_net.c +7
-0+uint16_t pkts_size = count; + +/** +* The ordering between avail index and +* desc reads needs to be enforced. +*/ + free_entries = __atomic_load_n(&vq->avail->idx,
__ATOMIC_ACQUIRE) -
Quoted 14 lines
Quoted 10 lines
lib/vhost/virtio_net.c rte_vhost_dequeue_burst(int vid, uint16_t queue_id, · line 3247
Quoted 10 lines
lib/vhost/virtio_net.c +10
-0+ vq->last_avail_idx; +if (free_entries ==0 ) + goto out; + +rte_prefetch0 (&vq->avail->ring[vq->last_avail_idx & + (vq->size -1 )]); + +async_iter_reset (async); + + count =RTE_MIN (count,MAX_PKT_BURST );^^^
Ok, my point about the overflow does not stand. Just thepkts_preallocarray is probably useless.
Allocating a bulk of mbufs by rte_pktmbuf_alloc_bulk() is for performance consideration. The pkts_prealloc array is for keeping a temporary variable to orderly update async_inflight_info, which is required in async path.
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
lib/vhost/virtio_net.c +5
-0+ count =RTE_MIN (count, free_entries); +VHOST_LOG_DATA (DEBUG ,"(%s) about to dequeue %u buffers\n" , + dev->ifname, count); + +if (rte_pktmbuf_alloc_bulk (mbuf_pool, pkts_prealloc, count))
Quoted 14 lines
Quoted 5 lines
'count' is provided by the user of the vhost async dequeue public API. There is no check that it is not bigger than
MAX_PKT_BURST.Calling
rte_pktmbuf_alloc_bulkon a fixed-size arraypkts_prealloc, allocated on the stack, it may cause a stack overflow.The rest still stands for me.
vvvQuoted 5 lines
This code is mostly copy/pasted from the "sync" code. I see a fix on the stats has been sent.
I need to explain the fix here.
The async dequeue patches and stats patches were both sent out
and merged in this release, while stats patch requires changes in both sync/async enq and deq. Sorry for not notice this change in the stats patch that was merged first, and that is the reason for this bug.
Quoted 14 lines
Quoted 8 lines
I point here another bug.
There are probably more...<grmbl>
I don't like how async code has been added in the vhost library by Intel.Maxime did a cleanup on the enqueue patch
https://patchwork.dpdk.org/project/dpdk/list/?series=20020&state=%2A&ar
chive=both.Quoted 4 lines
I see that the recent dequeue path additions have the same method of copying/pasting code and adding some branches in a non systematic way. Please clean this code and stop copy/pasting without a valid reason. </grmbl>
The cleanup for the code in dequeue patch was suggested in RFC v3.
https://patchwork.dpdk.org/project/dpdk/patch/[email protected]/
With merging copy_desc_to_mbuf and async_desc_to_mbuf in a single function, and reuse the fill_seg function, the code can be simplified without performance degradation.
Could you help to point out where in the current code that needs to be further cleaned? Are you referring to the common parts of async deq API and sync deq API can be abstracted into a function, such as ARAP?
https://patchwork.dpdk.org/project/dpdk/patch/[email protected]/
There are code duplications between async deq API and sync deq API, these parts are both needed. But the code clean of dequeue is by no means mindless copy/pasting.
Hope to get your insights.
Thanks,
Xuan
Quoted 4 lines
--
David Marchand
PATCH v8 5/5 examples/vhost: support async dequeue data path
In reply to [email protected]From: Xuan Ding <[email protected]>
This patch adds the use case for async dequeue API. Vswitch can leverage DMA device to accelerate vhost async dequeue path.
--- doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 286 ++++++++++++++++++++--------- examples/vhost/main.h | 32 +++- examples/vhost/virtio_net.c | 16 +- 4 files changed, 245 insertions(+), 98 deletions(-)
doc/guides/sample_app_ug/vhost.rst +6 -3
@@ -169,9 +169,12 @@ demonstrates how to use the async vhost APIs. It's used in combination with dmas **--dmas** This parameter is used to specify the assigned DMA device of a vhost device. Async vhost-user net driver will be used if --dmas is set. For example ---dmas [txd0@00:04.0,txd1@00:04.1] means use DMA channel 00:04.0 for vhost -device 0 enqueue operation and use DMA channel 00:04.1 for vhost device 1 -enqueue operation. +--dmas [txd0@00:04.0,txd1@00:04.1,rxd0@00:04.2,rxd1@00:04.3] means use +DMA channel 00:04.0/00:04.2 for vhost device 0 enqueue/dequeue operation +and use DMA channel 00:04.1/00:04.3 for vhost device 1 enqueue/dequeue +operation. The index of the device corresponds to the socket file in order, +that means vhost device 0 is created through the first socket file, vhost +device 1 is created through the second socket file, and so on. Common Issues -------------
examples/vhost/main.c +195 -91
428 lines not on this page · raw message
examples/vhost/main.h +29 -3
@@ -61,6 +61,19 @@ struct vhost_dev {struct vhost_queue queues[MAX_QUEUE_PAIRS *2 ]; } __rte_cache_aligned; +typedef uint16_t (*vhost_enqueue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mbuf **pkts, +uint32_t count); + +typedef uint16_t (*vhost_dequeue_burst_t )(struct vhost_dev *dev, +uint16_t queue_id,struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); + +struct vhost_queue_ops { +vhost_enqueue_burst_t enqueue_pkt_burst; +vhost_dequeue_burst_t dequeue_pkt_burst; +}; +TAILQ_HEAD (vhost_dev_tailq_list, vhost_dev);
@@ -87,6 +100,7 @@ struct dma_info {struct dma_for_vhost {struct dma_info dmas[RTE_MAX_QUEUES_PER_PORT *2 ]; +uint32_t async_flag; };/* we implement non-extra virtio net features */
@@ -97,7 +111,19 @@ void vs_vhost_net_remove(struct vhost_dev *dev);uint16_t vs_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mbuf **pkts,uint32_t count); -uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, -struct rte_mempool *mbuf_pool, -struct rte_mbuf **pkts,uint16_t count); +uint16_t builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t sync_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t sync_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count); +uint16_t async_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count); +uint16_t async_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool, +struct rte_mbuf **pkts,uint16_t count);#endif /* _MAIN_H_ */
examples/vhost/virtio_net.c +15 -1
@@ -238,6 +238,13 @@ vs_enqueue_pkts(struct vhost_dev *dev, uint16_t queue_id,return count; } +uint16_t +builtin_enqueue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +return vs_enqueue_pkts (dev, queue_id, pkts, count); +} + static __rte_always_inline int dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr, struct rte_mbuf *m, uint16_t desc_idx,
@@ -363,7 +370,7 @@ dequeue_pkt(struct vhost_dev *dev, struct rte_vhost_vring *vr,return 0 ; } -uint16_t +static uint16_t vs_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id,struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) {
@@ -440,3 +447,10 @@ vs_dequeue_pkts(struct vhost_dev *dev, uint16_t queue_id,return i; } + +uint16_t +builtin_dequeue_pkts (struct vhost_dev *dev,uint16_t queue_id, +struct rte_mempool *mbuf_pool,struct rte_mbuf **pkts,uint16_t count) +{ +return vs_dequeue_pkts (dev, queue_id, mbuf_pool, pkts, count); +}
Signature · 1 line
2.17.1
Maxime Coquelin
AppliedQuoted 73 lines · On 5/16/22 13:10, [email protected] wrote:
From: Xuan Ding <[email protected]>
The presence of asynchronous path allows applications to offload memory copies to DMA engine, so as to save CPU cycles and improve the copy performance. This patch set implements vhost async dequeue data path for split ring. The code is based on latest enqueue changes [1].
This patch set is a new design and implementation of [2]. Since dmadev was introduced in DPDK 21.11, to simplify application logics, this patch integrates dmadev in vhost. With dmadev integrated, vhost supports M:N mapping between vrings and DMA virtual channels. Specifically, one vring can use multiple different DMA channels and one DMA channel can be shared by multiple vrings at the same time.
A new asynchronous dequeue function is introduced:
1) rte_vhost_async_try_dequeue_burst(int vid,uint16_tqueue_id, structrte_mempool*mbuf_pool, structrte_mbuf**pkts,uint16_tcount, int*nr_inflight,
uint16_tdma_id,uint16_tvchan_id)Receive packets from the guest and offloads copies to DMA
virtual channel.[1] https://mails.dpdk.org/archives/dev/2022-February/234555.html
[2] https://mails.dpdk.org/archives/dev/2021-September/218591.html
v7->v8:
* changedma_idtoint16_t
* refine API documentation
v6->v7:
* correct code formatting
* change some functions to inline
v5->v6:
* adjust EXPERIMENTAL header
v4->v5:
* rebase to latest DPDK
* add some checks
v3->v4:
* fix CI build warnings
* adjust some indentation
* pass vq instead ofqueue_id
v2->v3:
* fix mbuf not updated correctly for large packets
v1->v2:
* fix a typo
* fix a bug indesc_to_mbuffillingRFC v3 -> v1:
* add sync and async path descriptor to mbuf refactoring
* add API description in docsRFC v2 -> RFC v3:
* rebase to latest DPDK versionRFC v1 -> RFC v2:
* fix one bug in example
* rename vchan tovchan_id
* check ifdma_idandvchan_idvalid
* rework all the logs to new standardXuan Ding (5):
vhost: prepare sync for descriptor to mbuf refactoring
vhost: prepare async for descriptor to mbuf refactoring
vhost: merge sync and async descriptor to mbuf filling
vhost: support async dequeue for split ring examples/vhost: support async dequeue data path
Quoted 12 lines
doc/guides/prog_guide/vhost_lib.rst | 6 + doc/guides/rel_notes/release_22_07.rst | 5 + doc/guides/sample_app_ug/vhost.rst | 9 +- examples/vhost/main.c | 286 ++++++++++----- examples/vhost/main.h | 32 +- examples/vhost/virtio_net.c | 16 +- lib/vhost/rte_vhost_async.h | 37 ++ lib/vhost/version.map | 2 +- lib/vhost/vhost.h | 1 + lib/vhost/virtio_net.c | 473 ++++++++++++++++++++++--- 10 files changed, 715 insertions(+), 152 deletions(-)
Applied to dpdk-next-virtio/main.
Thanks,
Maxime