introduce asynchronous data path for vhost
v2 cover introduce asynchronous data path for vhost
From: Patrick Fu <[email protected]>
Performing large memory copies usually takes up a major part of CPU cycles and becomes the hot spot in vhost-user enqueue operation. To offload expensive memory operations from the CPU, this patch set proposes to leverage DMA engines, e.g., I/OAT, a DMA engine in the Intel's processor, to accelerate large copies.
Large copies are offloaded from the CPU to the DMA in an asynchronous manner. The CPU just submits copy jobs to the DMA but without waiting for its copy completion. Thus, there is no CPU intervention during data transfer; we can save precious CPU cycles and improve the overall throughput for vhost-user based applications, like OVS. During packet transmission, it offloads large copies to the DMA and performs small copies by the CPU, due to startup overheads associated with the DMA.
This patch set construct a general framework that applications can leverage to attach DMA channels with vhost-user transmit queues. Four new RTE APIs are introduced to vhost library for applications to register and use the asynchronous data path. In addition, two new DMA operation callbacks are defined, by which vhost-user asynchronous data path can interact with DMA hardware. Currently only enqueue operation for split queue is implemented, but the framework is flexible to extend support for packed queue.
v2:
update meson file for new header file
update rte_vhost_version.map to include new APIs rename async APIs/structures to be prefixed with "rte_vhost" rename some variables/structures for readibility correct minor typo in comments/license statements refine memory allocation logic for vq internal buffer add error message printing in some failure cases check inflight async packets in unregistration API call mark new APIs as experimental
Patrick Fu (2):
vhost: introduce async enqueue registration API
vhost: introduce async enqueue for split ring
lib/librte_vhost/Makefile | 2 +- lib/librte_vhost/meson.build | 2 +- lib/librte_vhost/rte_vhost.h | 1 + lib/librte_vhost/rte_vhost_async.h | 176 +++++++++++ lib/librte_vhost/rte_vhost_version.map | 4 + lib/librte_vhost/socket.c | 20 ++ lib/librte_vhost/vhost.c | 127 +++++++- lib/librte_vhost/vhost.h | 30 +- lib/librte_vhost/vhost_user.c | 27 +- lib/librte_vhost/virtio_net.c | 539 ++++++++++++++++++++++++++++++++- 10 files changed, 919 insertions(+), 9 deletions(-) create mode 100644 lib/librte_vhost/rte_vhost_async.h
Signature · 1 line
1.8.3.1
PATCH v2 1/2 vhost: introduce async enqueue registration API
From: Patrick Fu <[email protected]>
This patch introduces registration/un-registration APIs for vhost async data enqueue operation. Together with the registration APIs implementations, data structures and async callback functions required for async enqueue data path are also defined.
--- lib/librte_vhost/Makefile | 2 +- lib/librte_vhost/meson.build | 2 +- lib/librte_vhost/rte_vhost.h | 1 + lib/librte_vhost/rte_vhost_async.h | 136 +++++++++++++++++++++++++++++++++ lib/librte_vhost/rte_vhost_version.map | 4 + lib/librte_vhost/socket.c | 20 +++++ lib/librte_vhost/vhost.c | 127 +++++++++++++++++++++++++++++- lib/librte_vhost/vhost.h | 30 +++++++- lib/librte_vhost/vhost_user.c | 27 ++++++- 9 files changed, 342 insertions(+), 7 deletions(-) create mode 100644 lib/librte_vhost/rte_vhost_async.h
lib/librte_vhost/Makefile +1 -1
@@ -42,7 +42,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := fd_man.c iotlb.c socket.c vhost.c \ # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_VHOST)-include += rte_vhost.h rte_vdpa.h \ - rte_vdpa_dev.h + rte_vdpa_dev.h rte_vhost_async.h # only compile vhost crypto when cryptodev is enabled ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y)
lib/librte_vhost/meson.build +1 -1
@@ -22,5 +22,5 @@ sources = files('fd_man.c', 'iotlb.c', 'socket.c', 'vdpa.c', 'vhost.c', 'vhost_user.c', 'virtio_net.c', 'vhost_crypto.c') headers = files('rte_vhost.h', 'rte_vdpa.h', 'rte_vdpa_dev.h', - 'rte_vhost_crypto.h') + 'rte_vhost_crypto.h', 'rte_vhost_async.h') deps += ['ethdev', 'cryptodev', 'hash', 'pci']
lib/librte_vhost/rte_vhost.h +1 -0
@@ -35,6 +35,7 @@#define RTE_VHOST_USER_EXTBUF_SUPPORT (1ULL << 5)/* support only linear buffers (no chained mbufs) */ #define RTE_VHOST_USER_LINEARBUF_SUPPORT (1ULL << 6) +#define RTE_VHOST_USER_ASYNC_COPY (1ULL << 7)/** Protocol features. */ #ifndef VHOST_USER_PROTOCOL_F_MQ
lib/librte_vhost/rte_vhost_async.h new file +136 -0
136 lines not on this page · raw message
lib/librte_vhost/rte_vhost_version.map +4 -0
@@ -72,4 +72,8 @@ EXPERIMENTAL { rte_vdpa_get_queue_num; rte_vdpa_get_features; rte_vdpa_get_protocol_features; + rte_vhost_async_channel_register; + rte_vhost_async_channel_unregister; + rte_vhost_submit_enqueue_burst; + rte_vhost_poll_enqueue_completed; };
lib/librte_vhost/socket.c +20 -0
@@ -42,6 +42,7 @@ struct vhost_user_socket {bool use_builtin_virtio_net;bool extbuf;bool linearbuf; +bool async_copy;/* * The "supported_features" indicates the feature bits the
@@ -205,6 +206,7 @@ struct vhost_user {size_t size;struct vhost_user_connection *conn;int ret; +struct virtio_net *dev;if (vsocket ==NULL )return ;
@@ -236,6 +238,13 @@ struct vhost_user {if (vsocket->linearbuf)vhost_enable_linearbuf (vid); +if (vsocket->async_copy) { + dev =get_device (vid); + +if (dev) + dev->async_copy =1 ; + } +VHOST_LOG_CONFIG (INFO ,"new device, handle is %d\n" , vid);if (vsocket->notify_ops->new_connection) {
@@ -881,6 +890,17 @@ struct rte_vdpa_device * goto out_mutex; } + vsocket->async_copy = flags &RTE_VHOST_USER_ASYNC_COPY ; + +if (vsocket->async_copy && + (flags & (RTE_VHOST_USER_IOMMU_SUPPORT | +RTE_VHOST_USER_POSTCOPY_SUPPORT ))) { +VHOST_LOG_CONFIG (ERR ,"error: enabling async copy and IOMMU " +"or post-copy feature simultaneously is not " +"supported\n" ); + goto out_mutex; + } +/* * Set the supported features correctly for the builtin vhost-user * net driver.
lib/librte_vhost/vhost.c +126 -1
137 lines not on this page · raw message
lib/librte_vhost/vhost.h +29 -1
@@ -24,6 +24,8 @@#include "rte_vdpa.h" #include "rte_vdpa_dev.h" +#include "rte_vhost_async.h" +/* Used to indicate that the device is running on a data core */ #define VIRTIO_DEV_RUNNING 1/* Used to indicate that the device is ready to operate */
@@ -40,6 +42,11 @@#define VHOST_LOG_CACHE_NR 32 +#define MAX_PKT_BURST 32 + +#define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2) +#define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2) +#define PACKED_DESC_ENQUEUE_USED_FLAG (w) \ ((w) ? (VRING_DESC_F_AVAIL | VRING_DESC_F_USED | VRING_DESC_F_WRITE) : \ VRING_DESC_F_WRITE)
@@ -201,6 +208,25 @@ struct vhost_virtqueue { TAILQ_HEAD(, vhost_iotlb_entry) iotlb_list;int iotlb_cache_nr; TAILQ_HEAD(, vhost_iotlb_entry) iotlb_pending_list; + +/* operation callbacks for async dma */ +struct rte_vhost_async_channel_ops async_ops; + +struct rte_vhost_iov_iter it_pool[VHOST_MAX_ASYNC_IT ]; +struct iovec vec_pool[VHOST_MAX_ASYNC_VEC ]; + +/* async data transfer status */ +uintptr_t **async_pkts_pending; +#define ASYNC_PENDING_INFO_N_MSK 0xFFFF +#define ASYNC_PENDING_INFO_N_SFT 16 +uint64_t *async_pending_info; +uint16_t async_pkts_idx; +uint16_t async_pkts_inflight_n; + +/* vq async features */ +bool async_inorder; +bool async_registered; +uint16_t async_threshold; } __rte_cache_aligned;/* Old kernels have no such macros defined */
@@ -354,6 +380,7 @@ struct virtio_net {int16_t broadcast_rarp;uint32_t nr_vring;int dequeue_zero_copy; +int async_copy;int extbuf;int linearbuf;struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS *2 ];
@@ -699,7 +726,8 @@ uint64_t translate_log_addr(struct virtio_net *dev, struct vhost_virtqueue *vq,/* Don't kick guest if we don't reach index specified by guest. */ if (dev->features & (1ULL <<VIRTIO_RING_F_EVENT_IDX )) {uint16_t old = vq->signalled_used; -uint16_t new = vq->last_used_idx; +uint16_t new = vq->async_pkts_inflight_n ? + vq->used->idx:vq->last_used_idx;bool signalled_used_valid = vq->signalled_used_valid; vq->signalled_used = new;
lib/librte_vhost/vhost_user.c +24 -3
@@ -462,12 +462,18 @@ }else {if (vq->shadow_used_split)rte_free (vq->shadow_used_split); +if (vq->async_pkts_pending) +rte_free (vq->async_pkts_pending); +if (vq->async_pending_info) +rte_free (vq->async_pending_info); + vq->shadow_used_split =rte_malloc (NULL , vq->size *sizeof (struct vring_used_elem ),RTE_CACHE_LINE_SIZE ); +if (!vq->shadow_used_split) {VHOST_LOG_CONFIG (ERR , -"failed to allocate memory for shadow used ring.\n" ); +"failed to allocate memory for vq internal data.\n" );return RTE_VHOST_MSG_RESULT_ERR ; } }
@@ -1145,7 +1151,8 @@ goto err_mmap; } - populate = (dev->dequeue_zero_copy) ?MAP_POPULATE :0 ; + populate = (dev->dequeue_zero_copy || dev->async_copy) ? +MAP_POPULATE :0 ; mmap_addr =mmap (NULL , mmap_size,PROT_READ |PROT_WRITE ,MAP_SHARED | populate, fd,0 );
@@ -1160,7 +1167,7 @@ reg->host_user_addr = (uint64_t )(uintptr_t )mmap_addr + mmap_offset; -if (dev->dequeue_zero_copy) +if (dev->dequeue_zero_copy || dev->async_copy)if (add_guest_pages (dev, reg, alignment) <0 ) { VHOST_LOG_CONFIG(ERR, "adding guest pages to region %u failed.\n",
@@ -1943,6 +1950,12 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused, }else {rte_free (vq->shadow_used_split); vq->shadow_used_split =NULL ; +if (vq->async_pkts_pending) +rte_free (vq->async_pkts_pending); +if (vq->async_pending_info) +rte_free (vq->async_pending_info); + vq->async_pkts_pending =NULL ; + vq->async_pending_info =NULL ; }rte_free (vq->batch_copy_elems);
@@ -1977,6 +1990,14 @@ static int vhost_user_set_vring_err(struct virtio_net **pdev __rte_unused,"set queue enable: %d to qp idx: %d\n" , enable, index); +if (!enable && dev->virtqueue[index]->async_registered) { +if (dev->virtqueue[index]->async_pkts_inflight_n) { +VHOST_LOG_CONFIG (ERR ,"failed to disable vring. " +"async inflight packets must be completed first\n" ); +return RTE_VHOST_MSG_RESULT_ERR ; + } + } + vdpa_dev = dev->vdpa_dev;if (vdpa_dev && vdpa_dev->ops->set_vring_state) vdpa_dev->ops->set_vring_state (dev->vid, index, enable);
Signature · 1 line
1.8.3.1
PATCH v2 2/2 vhost: introduce async enqueue for split ring
In reply to [email protected]From: Patrick Fu <[email protected]>
This patch implements async enqueue data path for split ring. 2 new async data path APIs are defined, by which applications can submit and poll packets to/from async engines. The async enqueue data leverages callback functions registered by applications to work with the async engine.
--- lib/librte_vhost/rte_vhost_async.h | 40 +++ lib/librte_vhost/virtio_net.c | 539 ++++++++++++++++++++++++++++++++++++- 2 files changed, 577 insertions(+), 2 deletions(-)
lib/librte_vhost/rte_vhost_async.h +40 -0
@@ -133,4 +133,44 @@ int rte_vhost_async_channel_register(int vid, uint16_t queue_id,__rte_experimental int rte_vhost_async_channel_unregister(int vid, uint16_t queue_id); +/** +* This function submit enqueue data to async engine. This function has +* no guranttee to the transfer completion upon return. Applications +* should poll transfer status by rte_vhost_poll_enqueue_completed() +* +* @param vid +* id of vhost device to enqueue data +* @param queue_id +* queue id to enqueue data +* @param pkts +* array of packets to be enqueued +* @param count +* packets num to be enqueued +* @return +* num of packets enqueued +*/ +__rte_experimental +uint16_t rte_vhost_submit_enqueue_burst(int vid, uint16_t queue_id, +struct rte_mbuf **pkts,uint16_t count); + +/** +* This function check async completion status for a specific vhost +* device queue. Packets which finish copying (enqueue) operation +* will be returned in an array. +* +* @param vid +* id of vhost device to enqueue data +* @param queue_id +* queue id to enqueue data +* @param pkts +* blank array to get return packet pointer +* @param count +* size of the packet array +* @return +* num of packets returned +*/ +__rte_experimental +uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, +struct rte_mbuf **pkts,uint16_t count); +#endif /* _RTE_VHOST_ASYNC_H_ */
lib/librte_vhost/virtio_net.c +537 -2
569 lines not on this page · raw message
Signature · 1 line
1.8.3.1
Re: [dpdk-dev] [PATCH v2 2/2] vhost: introduce async enqueue for split ring
lib/librte_vhost/virtio_net.c line 28
Quoted 3 lines
lib/librte_vhost/virtio_net.c +2
-0+#define VHOST_ASYNC_BATCH_THRESHOLD 8 +
Not very clear about why batch number is 8. It is better to save it in rte_vhost_async_features if the value come from hardware requirement.
Quoted 61 lines
lib/librte_vhost/virtio_net.c +39
-0+ +static __rte_noinline uint32_t +virtio_dev_rx_async_submit_split (struct virtio_net *dev, +struct vhost_virtqueue *vq,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) +{ +uint32_t pkt_idx =0 , pkt_burst_idx =0 ; +uint16_t num_buffers; +struct buf_vector buf_vec[BUF_VECTOR_MAX ]; +uint16_t avail_head, last_idx, shadow_idx; + +struct rte_vhost_iov_iter *it_pool = vq->it_pool; +struct iovec *vec_pool = vq->vec_pool; +struct rte_vhost_async_desc tdes[MAX_PKT_BURST ]; +struct iovec *src_iovec = vec_pool; +struct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >>1 ); +struct rte_vhost_iov_iter *src_it = it_pool; +struct rte_vhost_iov_iter *dst_it = it_pool +1 ; +uint16_t n_free_slot, slot_idx; +int n_pkts =0 ; + + avail_head = *((volatile uint16_t *)&vq->avail->idx); + last_idx = vq->last_avail_idx; + shadow_idx = vq->shadow_used_idx; + +/* +* The ordering between avail index and +* desc reads needs to be enforced. +*/ +rte_smp_rmb (); + +rte_prefetch0 (&vq->avail->ring[vq->last_avail_idx & (vq->size -1 )]); + +for (pkt_idx =0 ; pkt_idx < count; pkt_idx++) { +uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; +uint16_t nr_vec =0 ; + + if (unlikely(reserve_avail_buf_split(dev, vq, + pkt_len, buf_vec,
&num_buffers,lib/librte_vhost/virtio_net.c +3
-0+ avail_head, &nr_vec) < 0)) { + VHOST_LOG_DATA(DEBUG, + "(%d) failed to get enough desc fromvring\n",
lib/librte_vhost/virtio_net.c +6
-0+ dev->vid); + vq->shadow_used_idx -= num_buffers; +break ; + } + + VHOST_LOG_DATA(DEBUG, "(%d) current index %d | endindex %d\n",
lib/librte_vhost/virtio_net.c +10
-0+ dev->vid, vq->last_avail_idx, + vq->last_avail_idx + num_buffers); + +if (async_mbuf_to_desc (dev, vq, pkts[pkt_idx], + buf_vec, nr_vec, num_buffers, + src_iovec, dst_iovec, src_it, dst_it) <0 ) { + vq->shadow_used_idx -= num_buffers; +break ; + } +
lib/librte_vhost/virtio_net.c line 1550
Quoted 10 lines
lib/librte_vhost/virtio_net.c +10
-0+ slot_idx = (vq->async_pkts_idx + pkt_idx) & (vq->size -1 ); +if (src_it->count) { +async_fill_des (&tdes[pkt_burst_idx], src_it, dst_it); + pkt_burst_idx++; + vq->async_pending_info[slot_idx] = + num_buffers | (src_it->nr_segs <<16 ); + src_iovec += src_it->nr_segs; + dst_iovec += dst_it->nr_segs; + src_it +=2 ; + dst_it +=2 ;
Patrick,
In my understanding, nr_segs type definition can follow nr_vec type definition (uint16_t). By that can short the data saved in async_pkts_pending from 64bit to 32bit. Since those information will be used in datapath, the smaller size will get the better perf.
It is better to replace integer 2 with macro.
Thanks,
Marvin
Re: [dpdk-dev] [PATCH v2 2/2] vhost: introduce async enqueue for split ring
Thanks Marvin, my comments inline:
Quoted 13 lines
-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, July 1, 2020 4:51 PM
To: Fu, Patrick <[email protected]>; [email protected]; [email protected]; Xia, Chenbo <[email protected]>; Wang, Zhihong <[email protected]>
Cc: Fu, Patrick <[email protected]>; Wang, Yinan <[email protected]>; Jiang, Cheng1 <[email protected]>; Liang, Cunming <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v2 2/2] vhost: introduce async enqueue for split ringQuoted 1 line
Quoted 6 lines
lib/librte_vhost/virtio_net.c line 28
Quoted 2 lines
lib/librte_vhost/virtio_net.c +2
-0+#define VHOST_ASYNC_BATCH_THRESHOLD 8 +Not very clear about why batch number is 8. It is better to save it in
rte_vhost_async_featuresif the value come from hardware requirement.
We are in the progress of benchmarking how this value will have impact to the final performance, and we will have a more reasonable manner to handle this macro.
Quoted 79 lines
Quoted 61 lines
lib/librte_vhost/virtio_net.c +39
-0+ +static __rte_noinline uint32_t +virtio_dev_rx_async_submit_split (struct virtio_net *dev, +struct vhost_virtqueue *vq,uint16_t queue_id, +struct rte_mbuf **pkts,uint32_t count) { +uint32_t pkt_idx =0 , pkt_burst_idx =0 ; +uint16_t num_buffers; +struct buf_vector buf_vec[BUF_VECTOR_MAX ]; +uint16_t avail_head, last_idx, shadow_idx; + +struct rte_vhost_iov_iter *it_pool = vq->it_pool; +struct iovec *vec_pool = vq->vec_pool; +struct rte_vhost_async_desc tdes[MAX_PKT_BURST ]; +struct iovec *src_iovec = vec_pool; +struct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >>1 ); +struct rte_vhost_iov_iter *src_it = it_pool; +struct rte_vhost_iov_iter *dst_it = it_pool +1 ; +uint16_t n_free_slot, slot_idx; +int n_pkts =0 ; + + avail_head = *((volatile uint16_t *)&vq->avail->idx); + last_idx = vq->last_avail_idx; + shadow_idx = vq->shadow_used_idx; + +/* +* The ordering between avail index and +* desc reads needs to be enforced. +*/ +rte_smp_rmb (); + +rte_prefetch0 (&vq->avail->ring[vq->last_avail_idx & (vq->size - +1 )]); + +for (pkt_idx =0 ; pkt_idx < count; pkt_idx++) { +uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; +uint16_t nr_vec =0 ; + + if (unlikely(reserve_avail_buf_split(dev, vq, + pkt_len, buf_vec,
&num_buffers,lib/librte_vhost/virtio_net.c +3
-0+ avail_head, &nr_vec) < 0)) { + VHOST_LOG_DATA(DEBUG, + "(%d) failed to get enough desc fromvring\n",
lib/librte_vhost/virtio_net.c +6
-0+ dev->vid); + vq->shadow_used_idx -= num_buffers; +break ; + } + + VHOST_LOG_DATA(DEBUG, "(%d) current index %d | endindex %d\n",
lib/librte_vhost/virtio_net.c +10
-0+ dev->vid, vq->last_avail_idx, + vq->last_avail_idx + num_buffers); + +if (async_mbuf_to_desc (dev, vq, pkts[pkt_idx], + buf_vec, nr_vec, num_buffers, + src_iovec, dst_iovec, src_it, dst_it) <0 ) { + vq->shadow_used_idx -= num_buffers; +break ; + } +lib/librte_vhost/virtio_net.c line 1550
Quoted 10 lines
lib/librte_vhost/virtio_net.c +10
-0+ slot_idx = (vq->async_pkts_idx + pkt_idx) & (vq->size -1 ); +if (src_it->count) { +async_fill_des (&tdes[pkt_burst_idx], src_it, dst_it); + pkt_burst_idx++; + vq->async_pending_info[slot_idx] = + num_buffers | (src_it->nr_segs <<16 ); + src_iovec += src_it->nr_segs; + dst_iovec += dst_it->nr_segs; + src_it +=2 ; + dst_it +=2 ;Patrick,
In my understanding,nr_segstype definition can follownr_vectype definition (uint16_t). By that can short the data saved inasync_pkts_pendingfrom 64bit to 32bit.
Since those information will be used in datapath, the smaller size will get the better perf.
Quoted 2 lines
It is better to replace integer 2 with macro.
will update the code as you suggested.
Thanks,
Patrick