Skip to contentSkip to conversation

virtio: one way barrier for packed vring flags

RFC PATCH Superseded Joyce Kong 6 participants 30 messages Aug 27, 2019 – Oct 16, 2019 Network drivers

lore
Share & tools
lore

Joyce Kong

author
#0 lore raw

cover virtio: one way barrier for packed vring flags

This patch set replaces the two-way barriers with C11 one-way barriers for packed vring flags, when the frontend and backend are implemented in software.

By doing vhost-user + virtio-user case benchmarking, 9% performance gain in the RFC2544 test was measured on Thunderx2 platform.[1]

[1]https://doc.dpdk.org/dts/test_plans/pvp_multi_paths_performance_test_plan.html
PVP test with virtio 1.1 mergeable path

Joyce Kong (2):
virtio: one way barrier for packed vring desc avail flags
virtio: one way barrier for packed vring desc used flags

 drivers/net/virtio/virtio_rxtx.c                 | 38 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c | 10 +++++--
 drivers/net/virtio/virtqueue.h                   |  7 ++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 16 +++++-----
 5 files changed, 50 insertions(+), 23 deletions(-)
Signature · 1 line
2.7.4
author
#1 lore raw

RFC PATCH 1/2 virtio: one way barrier for packed vring desc avail flags

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance. For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 26 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----
 4 files changed, 31 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +20 -6
index 27ead19..2a2153c 100644
@@ -456,8 +456,14 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		if (hw->weak_barriers)
+			__atomic_store_n(&start_dp[idx].flags, flags,
+					 __ATOMIC_RELEASE);
+		else {
+			rte_cio_wmb();
+			start_dp[idx].flags = flags;
+		}
 		if (++vq->vq_avail_idx >= vq->vq_nentries) {
 			vq->vq_avail_idx -= vq->vq_nentries;
 			vq->vq_packed.cached_flags ^=
@@ -671,8 +677,12 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	dp->flags = flags;
+	if (vq->hw->weak_barriers)
+		__atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
+	else {
+		rte_cio_wmb();
+		dp->flags = flags;
+	}
 }
 
 static inline void
@@ -763,8 +773,12 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	head_dp->flags = head_flags;
+	if (vq->hw->weak_barriers)
+		__atomic_store_n(&head_dp->flags, head_flags, __ATOMIC_RELEASE);
+	else {
+		rte_cio_wmb();
+		head_dp->flags = head_flags;
+	}
 }
 
 static inline void
drivers/net/virtio/virtio_user/virtio_user_dev.c +5 -1
index fab87eb..7911c39 100644
@@ -624,7 +624,7 @@ virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 static inline int
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = desc->flags;
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
@@ -684,6 +684,10 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 	uint16_t n_descs, flags;
 
+	/* Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
+	 */
 	while (desc_is_avail(&vring->desc[vq->used_idx],
 			     vq->used_wrap_counter)) {
 
lib/librte_vhost/vhost.h +1 -1
index 884befa..d294ed1 100644
@@ -344,7 +344,7 @@ vq_is_packed(struct virtio_net *dev)
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = *((volatile uint16_t *) &desc->flags);
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
lib/librte_vhost/virtio_net.c +5 -6
index 5b85b83..e7463ff 100644
@@ -503,14 +503,13 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (avail_idx < vq->last_avail_idx)
 		wrap_counter ^= 1;
 
-	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
-		return -1;
-
 	/*
-	 * The ordering between desc flags and desc
-	 * content reads need to be enforced.
+	 * Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
 	 */
-	rte_smp_rmb();
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
 
 	*desc_count = 0;
 	*len = 0;
Signature · 1 line
2.7.4
author
#2 lore raw

RFC PATCH 2/2 virtio: one way barrier for packed vring desc used flags

In reply to Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   |  7 ++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 19 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +9 -3
index 2a2153c..1d818c8 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
 		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
@@ -233,8 +235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;
 		if (vq->used_idx >= dev->queue_size) {
drivers/net/virtio/virtqueue.h +6 -1
index c6dd4a3..ee6fcbb 100644
@@ -286,7 +286,12 @@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
 {
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	if (vq->hw->weak_barriers)
+		flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
+	else {
+		flags = desc->flags;
+		rte_cio_rmb();
+	}
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
 
lib/librte_vhost/virtio_net.c +2 -3
index e7463ff..241d467 100644
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
Signature · 1 line
2.7.4
author
#3 lore raw

v2 cover virtio: one way barrier for packed vring flags

In reply to Joyce Kong

This patch set replaces the two-way barriers with C11 one-way barriers for packed vring flags, when the frontend and backend are implemented in software.

By doing vhost-user + virtio-user case benchmarking, 9% performance gain in the RFC2544 test was measured on Thunderx2 platform.[1]

[1]https://doc.dpdk.org/dts/test_plans/pvp_multi_paths_performance_test_plan.html
PVP test with virtio 1.1 mergeable path

v2: convert RFC to patch

Joyce Kong (2):
virtio: one way barrier for packed vring desc avail flags
virtio: one way barrier for packed vring desc used flags

 drivers/net/virtio/virtio_rxtx.c                 | 38 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c | 10 +++++--
 drivers/net/virtio/virtqueue.h                   |  7 ++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 16 +++++-----
 5 files changed, 50 insertions(+), 23 deletions(-)
Signature · 1 line
2.7.4
author
#4 lore raw

PATCH v2 1/2 virtio: one way barrier for packed vring desc avail flags

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance. For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 26 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----
 4 files changed, 31 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +20 -6
index 27ead19..2a2153c 100644
@@ -456,8 +456,14 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		if (hw->weak_barriers)
+			__atomic_store_n(&start_dp[idx].flags, flags,
+					 __ATOMIC_RELEASE);
+		else {
+			rte_cio_wmb();
+			start_dp[idx].flags = flags;
+		}
 		if (++vq->vq_avail_idx >= vq->vq_nentries) {
 			vq->vq_avail_idx -= vq->vq_nentries;
 			vq->vq_packed.cached_flags ^=
@@ -671,8 +677,12 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	dp->flags = flags;
+	if (vq->hw->weak_barriers)
+		__atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
+	else {
+		rte_cio_wmb();
+		dp->flags = flags;
+	}
 }
 
 static inline void
@@ -763,8 +773,12 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	head_dp->flags = head_flags;
+	if (vq->hw->weak_barriers)
+		__atomic_store_n(&head_dp->flags, head_flags, __ATOMIC_RELEASE);
+	else {
+		rte_cio_wmb();
+		head_dp->flags = head_flags;
+	}
 }
 
 static inline void
drivers/net/virtio/virtio_user/virtio_user_dev.c +5 -1
index fab87eb..7911c39 100644
@@ -624,7 +624,7 @@ virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 static inline int
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = desc->flags;
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
@@ -684,6 +684,10 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 	uint16_t n_descs, flags;
 
+	/* Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
+	 */
 	while (desc_is_avail(&vring->desc[vq->used_idx],
 			     vq->used_wrap_counter)) {
 
lib/librte_vhost/vhost.h +1 -1
index 884befa..d294ed1 100644
@@ -344,7 +344,7 @@ vq_is_packed(struct virtio_net *dev)
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = *((volatile uint16_t *) &desc->flags);
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
lib/librte_vhost/virtio_net.c +5 -6
index 5b85b83..e7463ff 100644
@@ -503,14 +503,13 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (avail_idx < vq->last_avail_idx)
 		wrap_counter ^= 1;
 
-	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
-		return -1;
-
 	/*
-	 * The ordering between desc flags and desc
-	 * content reads need to be enforced.
+	 * Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
 	 */
-	rte_smp_rmb();
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
 
 	*desc_count = 0;
 	*len = 0;
Signature · 1 line
2.7.4
#5 lore raw

Re: [dpdk-dev] [PATCH v2 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Joyce,

Quoted 32 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance. For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 26 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----
 4 files changed, 31 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +0 -0
index 27ead19..2a2153c 100644
@@ -456,8 +456,14 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;

drivers/net/virtio/virtio_rxtx.c virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, · line 466

Quoted 10 lines
drivers/net/virtio/virtio_rxtx.c +8 -2
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		if (hw->weak_barriers)
+			__atomic_store_n(&start_dp[idx].flags, flags,
+					 __ATOMIC_RELEASE);
+		else {
+			rte_cio_wmb();
+			start_dp[idx].flags = flags;
+		}

It looks good to me.
I just wonder whether it would be cleaner to put that in an inline function:

static inline void
virtqueue_store_flags_packed()

Same for the fetch.

#6 lore raw

Re: [dpdk-dev] [PATCH v2 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Maxime,

Quoted 65 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Saturday, September 7, 2019 12:02 AM
To: Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: Re: [PATCH v2 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Joyce,

Quoted 5 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better

performance.

drivers/net/virtio/virtio_rxtx.c virtqueue_enqueue_recv_refill_packed(struct

Quoted 25 lines

For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 26 ++++++++++++++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----
 4 files changed, 31 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +0 -0
index 27ead19..2a2153c 100644
@@ -456,8 +456,14 @@ virtqueue_enqueue_recv_refill_packed(struct

virtqueue *vq,

drivers/net/virtio/virtio_rxtx.c virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq, · line 466

Quoted 13 lines
drivers/net/virtio/virtio_rxtx.c +8 -2
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		if (hw->weak_barriers)
+			__atomic_store_n(&start_dp[idx].flags, flags,
+					 __ATOMIC_RELEASE);
+		else {
+			rte_cio_wmb();
+			start_dp[idx].flags = flags;
+		}

It looks good to me.
I just wonder whether it would be cleaner to put that in an inline function:

Quoted 4 lines

static inline void
virtqueue_store_flags_packed()

Same for the fetch.

Have wrapped the store/fetch operation in inline functions in v3.

Best Regards,
Joyce

author
#7 lore raw

PATCH v2 2/2 virtio: one way barrier for packed vring desc used flags

In reply to Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   |  7 ++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 19 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +9 -3
index 2a2153c..1d818c8 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
 		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
@@ -233,8 +235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;
 		if (vq->used_idx >= dev->queue_size) {
drivers/net/virtio/virtqueue.h +6 -1
index c6dd4a3..ee6fcbb 100644
@@ -286,7 +286,12 @@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
 {
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	if (vq->hw->weak_barriers)
+		flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
+	else {
+		flags = desc->flags;
+		rte_cio_rmb();
+	}
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
 
lib/librte_vhost/virtio_net.c +2 -3
index e7463ff..241d467 100644
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
Signature · 1 line
2.7.4
author
#8 lore raw

v3 cover virtio: one way barrier for packed vring flags

In reply to Joyce Kong

This patch set replaces the two-way barriers with C11 one-way barriers for packed vring flags, when the frontend and backend are implemented in software.

By doing vhost-user + virtio-user case benchmarking, 9% performance gain in the RFC2544 test was measured on Thunderx2 platform.[1]

[1]https://doc.dpdk.org/dts/test_plans/pvp_multi_paths_performance_test_plan.html
PVP test with virtio 1.1 mergeable path

v3:
Wrap C11 one-way barriers and DMA barriers(rte_cio_*) together with an inline fuction.

v2:
Convert RFC to patch.

Joyce Kong (2):
virtio: one way barrier for packed vring desc avail flags
virtio: one way barrier for packed vring desc used flags

 drivers/net/virtio/virtio_rxtx.c                 | 25 +++++++++++++--------
 drivers/net/virtio/virtio_user/virtio_user_dev.c | 10 ++++++---
 drivers/net/virtio/virtqueue.h                   | 28 +++++++++++++++++++++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 16 ++++++--------
 5 files changed, 58 insertions(+), 23 deletions(-)
Signature · 1 line
2.7.4
author
#9 lore raw

PATCH v3 1/2 virtio: one way barrier for packed vring desc avail flags

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +7 -6
index 27ead19..a87ffe1 100644
@@ -456,8 +456,10 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		virtqueue_store_flags_packed(&start_dp[idx], flags,
+					     hw->weak_barriers);
+
 		if (++vq->vq_avail_idx >= vq->vq_nentries) {
 			vq->vq_avail_idx -= vq->vq_nentries;
 			vq->vq_packed.cached_flags ^=
@@ -671,8 +673,7 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	dp->flags = flags;
+	virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
 }
 
 static inline void
@@ -763,8 +764,8 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	head_dp->flags = head_flags;
+	virtqueue_store_flags_packed(head_dp, head_flags,
+				     vq->hw->weak_barriers);
 }
 
 static inline void
drivers/net/virtio/virtio_user/virtio_user_dev.c +5 -1
index fab87eb..7911c39 100644
@@ -624,7 +624,7 @@ virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 static inline int
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = desc->flags;
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
@@ -684,6 +684,10 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 	uint16_t n_descs, flags;
 
+	/* Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
+	 */
 	while (desc_is_avail(&vring->desc[vq->used_idx],
 			     vq->used_wrap_counter)) {
 
drivers/net/virtio/virtqueue.h +11 -0
index c6dd4a3..8d93ccb 100644
@@ -54,6 +54,17 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline void
+virtqueue_store_flags_packed(struct vring_packed_desc *dp,
+			      uint16_t flags, uint8_t weak_barriers)
+{
+	if (weak_barriers) {
+		__atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
+	} else {
+		rte_cio_wmb();
+		dp->flags = flags;
+	}
+}
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
 #else
lib/librte_vhost/vhost.h +1 -1
index 884befa..d294ed1 100644
@@ -344,7 +344,7 @@ vq_is_packed(struct virtio_net *dev)
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = *((volatile uint16_t *) &desc->flags);
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
lib/librte_vhost/virtio_net.c +5 -6
index 5b85b83..e7463ff 100644
@@ -503,14 +503,13 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (avail_idx < vq->last_avail_idx)
 		wrap_counter ^= 1;
 
-	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
-		return -1;
-
 	/*
-	 * The ordering between desc flags and desc
-	 * content reads need to be enforced.
+	 * Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
 	 */
-	rte_smp_rmb();
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
 
 	*desc_count = 0;
 	*len = 0;
Signature · 1 line
2.7.4
Reviewed-by on v3 1/2
#10 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 19 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#11 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base commit id: d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3 patch set , seven paths of vhost/virtio pvp test shows performance drop as below:

PVP vhost/virtio 1c1q test          before apply patch apply patch
test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068 6.905
test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 45 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime Coquelin
Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 19 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#12 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Yinan,

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb and rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not saved, which requir additional cpu cycles, this caused slight degradation on x86.

The patches uplifted the performance on aarch64 about 9% as indicated in the cover letter. While I am thinking over a solution to the degradation on x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost never used, it is rte_wmb that is more widely used to notify the NIC device, any difference between the virtio ring compatible smartNIC device(or vDPA?) and i40e like devices?
2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by the compiler, then the problem becomes with the correct use of barriers, other than the degradation.

Any comments are welcome!

Best Regards,
Gavin

Quoted 80 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base commit id:
d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3 patch set , seven paths of vhost/virtio pvp test shows performance drop as below:

PVP vhost/virtio 1c1q test          before apply patch apply patch
test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068 6.905
test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 8 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime Coquelin
Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 36 lines

desc avail flags

Quoted 19 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#13 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Thanks Yinan for reporting the regresion and Gavin for the analysis.

Quoted 6 lines · Gavin Hu (Arm Technology China)

Hi Yinan,

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb and rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not saved, which requir additional cpu cycles, this caused slight degradation on x86.

Quoted 6 lines

The patches uplifted the performance on aarch64 about 9% as indicated in the cover letter. While I am thinking over a solution to the degradation on x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost never used, it is rte_wmb that is more widely used to notify the NIC device, any difference between the virtio ring compatible smartNIC device(or vDPA?) and i40e like devices?
2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by the compiler, then the problem becomes with the correct use of barriers, other than the degradation.

Any comments are welcome!

It may we worth that Yinan tries with rte_wmb instead of rte_cio_wmb without the series applied, just to confirm this is caused by the etra branch.

Maxime

Quoted 83 lines

Best Regards,
Gavin

Quoted 80 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base commit id:
d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3 patch set , seven paths of vhost/virtio pvp test shows performance drop as below:

PVP vhost/virtio 1c1q test          before apply patch apply patch
test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068 6.905
test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 8 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime Coquelin
Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 36 lines

desc avail flags

Quoted 19 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#14 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

In reply to Gavin Hu (Arm Technology China)
Quoted 25 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Tuesday, September 10, 2019 5:49 PM
To: Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Yinan,

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb and rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not saved, which requir additional cpu cycles, this caused slight degradation on x86.

Quoted 9 lines

The patches uplifted the performance on aarch64 about 9% as indicated in the cover letter. While I am thinking over a solution to the degradation on x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost never
used, it is rte_wmb that is more widely used to notify the NIC device, any difference between the virtio ring compatible smartNIC device(or vDPA?) and i40e like devices?

Hi Gavin,
X86 architecture can guarantee that young store happen later than old store. So rte_cio_wmb is just compiler memory barrier in x86.

I think compiler barrier is also enough in pmd, rte_wmb is in pmd because of it was inherit from first implementation :)

Thanks,
Marvin

Quoted 93 lines

2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by the compiler, then the problem becomes with the correct use of barriers, other than the degradation.

Any comments are welcome!

Best Regards,
Gavin

Quoted 4 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Quoted 8 lines

Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 8 lines

desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base commit id:
d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3 patch set , seven paths of vhost/virtio pvp test shows performance drop as

below:

Quoted 21 lines

PVP vhost/virtio 1c1q test          before apply patch apply patch
test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068 6.905
test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 8 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime Coquelin
Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

Quoted 2 lines

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 37 lines

vring

Quoted 36 lines

desc avail flags

Quoted 19 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#15 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Marvin,

Thanks for your answers, one more question for x86:
1. For CIO memory alone or MMIO memory(eg PCI BAR) alone, the compiler barrier is enough to keep ordering, that's why both rte_io_mb and rte_cio_mb are defined as compiler barriers, right?
2. How about the ordering of interleaved CIO and MMIO accesses, for example, a young store to MMIO can be reordered before an older store to CIO? CIO may be faster than devices, but store buffers or caching may cause the CIO update not visible to the device(in a common doorbell case)?

Best regards,
Gavin

Quoted 158 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 10:39 AM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 10 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Tuesday, September 10, 2019 5:49 PM
To: Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli

<[email protected]>;

Quoted 8 lines

Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Yinan,

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb

and

Quoted 3 lines

rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not

saved,

Quoted 10 lines

which requir additional cpu cycles, this caused slight degradation on x86.

The patches uplifted the performance on aarch64 about 9% as indicated in the cover letter. While I am thinking over a solution to the degradation on x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost never
used, it is rte_wmb that is more widely used to notify the NIC device, any difference between the virtio ring compatible smartNIC device(or vDPA?)

and

Quoted 1 line

i40e like devices?

Hi Gavin,
X86 architecture can guarantee that young store happen later than old store. So rte_cio_wmb is just compiler memory barrier in x86.

I think compiler barrier is also enough in pmd, rte_wmb is in pmd because of it was inherit from first implementation :)

Thanks,
Marvin

Quoted 22 lines

2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by the compiler, then the problem becomes with the correct use of barriers, other than the degradation.

Any comments are welcome!

Best Regards,
Gavin

Quoted 4 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Quoted 8 lines

Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

Quoted 8 lines

vring

Quoted 7 lines

desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base commit id:
d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3

patch

Quoted 17 lines
Quoted 1 line

set , seven paths of vhost/virtio pvp test shows performance drop as

below:

Quoted 15 lines

PVP vhost/virtio 1c1q test          before apply patch apply patch
test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068 6.905
test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 2 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime

Coquelin

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

Quoted 11 lines
Quoted 2 lines

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 8 lines

vring

Quoted 7 lines

desc avail flags

Quoted 2 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software,

that

Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 17 lines

is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 7 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11 +++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#16 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Thanks Gavin, my answers are inline.

Quoted 13 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Wednesday, September 11, 2019 11:35 AM
To: Liu, Yong <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 6 lines

Hi Marvin,

Thanks for your answers, one more question for x86:
1. For CIO memory alone or MMIO memory(eg PCI BAR) alone, the compiler barrier is enough to keep ordering, that's why both rte_io_mb and rte_cio_mb are defined as compiler barriers, right?

Yes, that's right for x86.

Quoted 5 lines

2. How about the ordering of interleaved CIO and MMIO accesses, for example, a young store to MMIO can be reordered before an older store to CIO? CIO may be faster than devices, but store buffers or caching may cause the CIO update not visible to the device(in a common doorbell case)?

There's always one kind of cache coherent engine in x86 uncore sub-system.
When CIO write instruction was retried, data will be in CPU LLC.
When device doing inbound read, request will go to cache engine first and then check memory state and retrieve latest value.

Quoted 184 lines

Best regards,
Gavin

Quoted 11 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 10:39 AM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 14 lines

desc avail flags

Quoted 10 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Tuesday, September 10, 2019 5:49 PM
To: Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli

Quoted 3 lines

<[email protected]>;

Quoted 2 lines

Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 4 lines
Quoted 4 lines

vring desc avail flags

Hi Yinan,

Quoted 8 lines
Quoted 2 lines

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb

and

Quoted 3 lines

rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not

saved,

Quoted 1 line

which requir additional cpu cycles, this caused slight degradation on

x86.

Quoted 2 lines
Quoted 2 lines

The patches uplifted the performance on aarch64 about 9% as indicated

in

Quoted 1 line
Quoted 1 line

the cover letter. While I am thinking over a solution to the

degradation on

Quoted 4 lines
Quoted 4 lines

x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost

never

Quoted 1 line
Quoted 1 line

used, it is rte_wmb that is more widely used to notify the NIC device,

any

Quoted 6 lines
Quoted 1 line

difference between the virtio ring compatible smartNIC device(or vDPA?)

and

Quoted 1 line

i40e like devices?

Hi Gavin,
X86 architecture can guarantee that young store happen later than old

store.

Quoted 3 lines

So rte_cio_wmb is just compiler memory barrier in x86.

I think compiler barrier is also enough in pmd, rte_wmb is in pmd because

of

Quoted 7 lines

it was inherit from first implementation :)

Thanks,
Marvin

Quoted 2 lines

2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by

the

Quoted 1 line
Quoted 1 line

compiler, then the problem becomes with the correct use of barriers,

other

Quoted 7 lines
Quoted 7 lines

than the degradation.

Any comments are welcome!

Best Regards,
Gavin

Quoted 12 lines
Quoted 4 lines
Quoted 4 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Quoted 8 lines
Quoted 8 lines

Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 7 lines
Quoted 7 lines

vring

Quoted 6 lines

desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base

commit id:

Quoted 6 lines
Quoted 1 line
Quoted 1 line

d03d8622db48918d14bfe805641b1766ecc40088, after applying your v3

patch

Quoted 4 lines
Quoted 1 line

set , seven paths of vhost/virtio pvp test shows performance drop as

below:

Quoted 2 lines

PVP vhost/virtio 1c1q test          before apply patch apply

patch

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068

6.905

Quoted 18 lines
Quoted 7 lines
Quoted 7 lines

test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 2 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime

Coquelin

Quoted 10 lines
Quoted 8 lines
Quoted 6 lines

Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 1 line

vring

Quoted 10 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

desc avail flags

Quoted 2 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software,

that

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other

than

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers

== 1)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

and yields better performance.
For the above case, this patch helps yielding even better

performance

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

by replacing the two-way barriers with C11 one-way barriers for

avail

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire

can

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++---

---

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11

+++++++++++

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----

-

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#17 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Thanks Marvin, my inline comments.

Quoted 45 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 2:30 PM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Thanks Gavin, my answers are inline.

Quoted 5 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Wednesday, September 11, 2019 11:35 AM
To: Liu, Yong <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Technology

Quoted 8 lines

China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 6 lines

Hi Marvin,

Thanks for your answers, one more question for x86:
1. For CIO memory alone or MMIO memory(eg PCI BAR) alone, the compiler barrier is enough to keep ordering, that's why both rte_io_mb and rte_cio_mb are defined as compiler barriers, right?

Yes, that's right for x86.

Quoted 1 line

2. How about the ordering of interleaved CIO and MMIO accesses, for

example,

Quoted 4 lines

a young store to MMIO can be reordered before an older store to CIO? CIO may be faster than devices, but store buffers or caching may cause the CIO update not visible to the device(in a common doorbell case)?

Quoted 4 lines

There's always one kind of cache coherent engine in x86 uncore sub-system. When CIO write instruction was retried, data will be in CPU LLC. When device doing inbound read, request will go to cache engine first and then check memory state and retrieve latest value.

I understand your words that the cache coherent engine is working like a hub/coordinator/arbiter for all the accesses to three types of memory: 1 - normal memory, 2 - CIO memory, 3 - MMIO memory, and the ordering behaviors are no different? Then in what scenarios mfence/sfence/lfence should be used? Maybe just mfence is enough to keep orderings of store/load(which is the only one might reordered on x86)?

Quoted 193 lines
Quoted 9 lines

Best regards,
Gavin

Quoted 5 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 10:39 AM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin

<[email protected]>;

Quoted 1 line
Quoted 1 line

Joyce Kong (Arm Technology China) <[email protected]>;

[email protected]

Quoted 4 lines
Quoted 4 lines

Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper

<[email protected]>

Quoted 13 lines
Quoted 1 line

Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 11 lines

desc avail flags

Quoted 7 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Tuesday, September 10, 2019 5:49 PM
To: Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang,

Zhihong

Quoted 45 lines
Quoted 6 lines
Quoted 3 lines

<[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli

<[email protected]>;

Quoted 2 lines

Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 4 lines
Quoted 4 lines

vring desc avail flags

Hi Yinan,

Quoted 8 lines
Quoted 2 lines

We have done a comparative analysis and found with the old code the if(weak_barriers) and else branches were saved on x86 as rte_smp_wmb

and

Quoted 3 lines

rte_cio_wmb are identical.
http://git.dpdk.org/dpdk/tree/drivers/net/virtio/virtqueue.h#n49
For the new code, with Joyce's patches applied, the branches were not

saved,

Quoted 1 line

which requir additional cpu cycles, this caused slight degradation on

x86.

Quoted 2 lines
Quoted 2 lines

The patches uplifted the performance on aarch64 about 9% as indicated

in

Quoted 1 line
Quoted 1 line

the cover letter. While I am thinking over a solution to the

degradation on

Quoted 4 lines
Quoted 4 lines

x86,could you help answer:
1. Is rte_cio_wmb is sufficient for the non weak-barrier case(HW offloading)?
I got this question because I see in Intel NIC PMDs, it is almost

never

Quoted 1 line
Quoted 1 line

used, it is rte_wmb that is more widely used to notify the NIC device,

any

Quoted 6 lines
Quoted 1 line

difference between the virtio ring compatible smartNIC device(or vDPA?)

and

Quoted 1 line

i40e like devices?

Hi Gavin,
X86 architecture can guarantee that young store happen later than old

store.

Quoted 3 lines

So rte_cio_wmb is just compiler memory barrier in x86.

I think compiler barrier is also enough in pmd, rte_wmb is in pmd because

of

Quoted 2 lines

it was inherit from first implementation :)

Quoted 19 lines
Quoted 5 lines

Thanks,
Marvin

Quoted 2 lines

2. If the rte_cio_wmb is not sufficient for this case and replaced by stronger barriers, like sfence, then the branches will not be saved by

the

Quoted 1 line
Quoted 1 line

compiler, then the problem becomes with the correct use of barriers,

other

Quoted 11 lines
Quoted 11 lines

than the degradation.

Any comments are welcome!

Best Regards,
Gavin

Quoted 4 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Tuesday, September 10, 2019 11:54 AM
To: Maxime Coquelin <[email protected]>; Joyce Kong

(Arm

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang,

Zhihong

Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

<[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 9 lines
Quoted 7 lines
Quoted 7 lines

vring

Quoted 6 lines

desc avail flags

Hi Joyce,

I just test performance impact of your patch set with code base

commit id:

Quoted 1 line
Quoted 1 line
Quoted 1 line

d03d8622db48918d14bfe805641b1766ecc40088, after applying your

v3

Quoted 5 lines
Quoted 5 lines

patch

Quoted 4 lines
Quoted 1 line

set , seven paths of vhost/virtio pvp test shows performance drop as

below:

Quoted 2 lines

PVP vhost/virtio 1c1q test          before apply patch apply

Quoted 32 lines

patch

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

test_perf_pvp_inorder_mergeable 7.603 7.474
test_perf_pvp_inorder_no_mergeable 7.642 7.525
test_perf_pvp_mergeable 7.556 7.431
test_perf_pvp_normal 7.554 7.478
test_perf_pvp_vector_rx 7.581 7.469
test_perf_pvp_virtio11_mergeable 7.068

6.905

Quoted 24 lines
Quoted 7 lines
Quoted 7 lines

test_perf_pvp_virtio11_normal 7.088 6.888

Thanks,
Yinan

Quoted 2 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Maxime

Coquelin

Quoted 16 lines
Quoted 8 lines
Quoted 6 lines

Sent: 2019年9月9日 18:10
To: Joyce Kong <[email protected]>; [email protected]
Cc: [email protected]; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected];

[email protected]

Quoted 1 line

Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for

packed

Quoted 7 lines

vring

Quoted 6 lines

desc avail flags

Quoted 1 line · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then

the

Quoted 39 lines
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

frontend and backend are assumed to be implemented in software,

that

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

is they can run on identical CPUs in an SMP configuration. Thus a weak form of memory barriers like rte_smp_r/wmb, other

than

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers

== 1)

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

and yields better performance.
For the above case, this patch helps yielding even better

performance

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

by replacing the two-way barriers with C11 one-way barriers for

avail

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire

can

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 9 lines

enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++---

---

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 11

+++++++++++

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++-----

-

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 1 line
 5 files changed, 29 insertions(+), 14 deletions(-)

Thanks,
Maxime

#18 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 47 lines · Gavin Hu (Arm Technology China)

Thanks Marvin, my inline comments.

Quoted 45 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 2:30 PM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Thanks Gavin, my answers are inline.

Quoted 5 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Wednesday, September 11, 2019 11:35 AM
To: Liu, Yong <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Technology

Quoted 8 lines

China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 6 lines

Hi Marvin,

Thanks for your answers, one more question for x86:
1. For CIO memory alone or MMIO memory(eg PCI BAR) alone, the compiler barrier is enough to keep ordering, that's why both rte_io_mb and rte_cio_mb are defined as compiler barriers, right?

Yes, that's right for x86.

Quoted 1 line

2. How about the ordering of interleaved CIO and MMIO accesses, for

example,

Quoted 4 lines

a young store to MMIO can be reordered before an older store to CIO? CIO may be faster than devices, but store buffers or caching may cause the CIO update not visible to the device(in a common doorbell case)?

Quoted 7 lines
Quoted 4 lines

There's always one kind of cache coherent engine in x86 uncore sub-system. When CIO write instruction was retried, data will be in CPU LLC. When device doing inbound read, request will go to cache engine first and then check memory state and retrieve latest value.

I understand your words that the cache coherent engine is working like a hub/coordinator/arbiter for all the accesses to three types of memory: 1 - normal memory, 2 - CIO memory, 3 - MMIO memory, and the ordering behaviors are no different? Then in what scenarios mfence/sfence/lfence should be used? Maybe just mfence is enough to keep orderings of store/load(which is the only one might reordered on x86)?

Quoted 1 line

The fence types needed will depend on the memory types used, for example, any memory mapped as write-combining will have different behaviour and need different fences to the regular write-back memory we are most familiar with. For the situations we deal with in DPDK, for regular memory writes and MMIO writes, reads won't be reordered with other reads, and writes won't be reordered with other writes, so therefore, as you point out, the mfence instruction is only rarely needed, and barriers to prevent compiler reordering are sufficient in nearly all cases.

/Bruce

#19 lore raw

Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Bruce,

Quoted 88 lines

-----Original Message-----
From: Bruce Richardson <[email protected]>
Sent: Wednesday, September 11, 2019 6:03 PM
To: Gavin Hu (Arm Technology China) <[email protected]>
Cc: Liu, Yong <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin <[email protected]>; Joyce Kong (Arm Technology China) <[email protected]>; [email protected]; nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 7 lines · Gavin Hu (Arm Technology China)

Thanks Marvin, my inline comments.

Quoted 5 lines

-----Original Message-----
From: Liu, Yong <[email protected]>
Sent: Wednesday, September 11, 2019 2:30 PM
To: Gavin Hu (Arm Technology China) <[email protected]>; Wang, Yinan <[email protected]>; Maxime Coquelin

<[email protected]>;

Quoted 1 line
Quoted 1 line

Joyce Kong (Arm Technology China) <[email protected]>;

[email protected]

Quoted 4 lines
Quoted 4 lines

Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Steve Capper

<[email protected]>

Quoted 1 line
Quoted 1 line

Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed

vring

Quoted 8 lines
Quoted 8 lines

desc avail flags

Thanks Gavin, my answers are inline.

Quoted 4 lines

-----Original Message-----
From: Gavin Hu (Arm Technology China) [mailto:[email protected]]
Sent: Wednesday, September 11, 2019 11:35 AM
To: Liu, Yong <[email protected]>; Wang, Yinan

<[email protected]>;

Quoted 4 lines
Quoted 4 lines
Quoted 1 line

Maxime Coquelin <[email protected]>; Joyce Kong (Arm

Technology

Quoted 2 lines

China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; Bie, Tiwei <[email protected]>; Wang,

Zhihong

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

<[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; [email protected]; Honnappa

Nagarahalli

Quoted 1 line
Quoted 1 line
Quoted 1 line

<[email protected]>; Steve Capper

<[email protected]>

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Subject: RE: [dpdk-dev] [PATCH v3 1/2] virtio: one way barrier for packed vring desc avail flags

Hi Marvin,

Thanks for your answers, one more question for x86:
1. For CIO memory alone or MMIO memory(eg PCI BAR) alone, the

compiler

Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

barrier is enough to keep ordering, that's why both rte_io_mb and rte_cio_mb are defined as compiler barriers, right?

Yes, that's right for x86.

Quoted 1 line

2. How about the ordering of interleaved CIO and MMIO accesses, for

example,

Quoted 1 line

a young store to MMIO can be reordered before an older store to CIO?

CIO

Quoted 1 line
Quoted 1 line
Quoted 1 line

may be faster than devices, but store buffers or caching may cause the

CIO

Quoted 4 lines
Quoted 4 lines
Quoted 2 lines

update not visible to the device(in a common doorbell case)?

There's always one kind of cache coherent engine in x86 uncore sub-

system.

Quoted 4 lines
Quoted 3 lines

When CIO write instruction was retried, data will be in CPU LLC. When device doing inbound read, request will go to cache engine first and then check memory state and retrieve latest value.

I understand your words that the cache coherent engine is working like a

hub/coordinator/arbiter for all the accesses to three types of memory: 1 - normal memory, 2 - CIO memory, 3 - MMIO memory, and the ordering behaviors are no different?

Quoted 1 line

Then in what scenarios mfence/sfence/lfence should be used? Maybe just

mfence is enough to keep orderings of store/load(which is the only one might reordered on x86)?

Quoted 1 line
Quoted 1 line
Quoted 8 lines

The fence types needed will depend on the memory types used, for example, any memory mapped as write-combining will have different behaviour and need different fences to the regular write-back memory we are most familiar with. For the situations we deal with in DPDK, for regular memory writes and MMIO writes, reads won't be reordered with other reads, and writes won't be reordered with other writes, so therefore, as you point out, the mfence instruction is only rarely needed, and barriers to prevent compiler reordering are sufficient in nearly all cases.

Thanks for your explanation about the barriers on x86, it is really helpful for us to optimize PMDs for aarch64 by using less restrictive barriers while not breaking x86 platforms.

Quoted 2 lines

/Bruce

author
#20 lore raw

PATCH v3 2/2 virtio: one way barrier for packed vring desc used flags

In reply to Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in packed ring.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 17 ++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 29 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +9 -3
index a87ffe1..2f0879c 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
 		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
@@ -233,8 +235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;
 		if (vq->used_idx >= dev->queue_size) {
drivers/net/virtio/virtqueue.h +16 -1
index 8d93ccb..bac52e3 100644
@@ -54,6 +54,21 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline uint16_t
+virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
+			      uint8_t weak_barriers)
+{
+	uint16_t flags;
+
+	if (weak_barriers) {
+		flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE);
+	} else {
+		flags = dp->flags;
+		rte_cio_rmb();
+	}
+	return flags;
+}
+
 static inline void
 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
 			      uint16_t flags, uint8_t weak_barriers)
@@ -297,7 +312,7 @@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
 {
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
 
lib/librte_vhost/virtio_net.c +2 -3
index e7463ff..241d467 100644
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
Signature · 1 line
2.7.4
Reviewed-by on v3 2/2
#21 lore raw

Re: [dpdk-dev] [PATCH v3 2/2] virtio: one way barrier for packed vring desc used flags

Quoted 13 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in packed ring.

Quoted 6 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 17 ++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 29 insertions(+), 9 deletions(-)

Thanks,
Maxime

author
#22 lore raw

v4 cover virtio: one way barrier for packed vring flags

In reply to Joyce Kong

This patch set replaces the two-way barriers with C11 one-way barriers for packed vring flags, when the frontend and backend are implemented in software.

By doing vhost-user + virtio-user case benchmarking, 9% performance gain in the RFC2544 test was measured on Thunderx2 platform.[1] And by doing VM2VM case benchmarking, 11% perf gain was measured on Ampere platform.

[1]https://doc.dpdk.org/dts/test_plans/pvp_multi_paths_performance_test_plan.html
PVP test with virtio 1.1 mergeable path

v4:
Use rte_smp_rmb/wmb instead of __atomic_load/store_n on x86 as it reports a better perf(~1.5%), which comes from the saved branch by the compiler. The if and else branch are identical with the smp and cio barriers both defined as compiler barriers on x86.
http://inbox.dpdk.org/dev/E0CBA5A1980F1F408E1F28F9991B5B1D50EFF246@SHSMSX104.ccr.corp.intel.com/

v3:
Wrap C11 one-way barriers and DMA barriers(rte_cio_*) together with an inline fuction.

v2:
Convert RFC to patch.

Joyce Kong (2):
virtio: one way barrier for packed vring desc avail flags
virtio: one way barrier for packed vring desc used flags

 drivers/net/virtio/virtio_rxtx.c                 | 25 +++++++-----
 drivers/net/virtio/virtio_user/virtio_user_dev.c | 10 +++--
 drivers/net/virtio/virtqueue.h                   | 49 +++++++++++++++++++++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 16 ++++----
 5 files changed, 79 insertions(+), 23 deletions(-)
Signature · 1 line
2.7.4
author
#23 lore raw

PATCH v4 1/2 virtio: one way barrier for packed vring desc avail flags

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 21 +++++++++++++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 39 insertions(+), 14 deletions(-)
drivers/net/virtio/virtio_rxtx.c +7 -6
index 27ead19..a87ffe1 100644
@@ -456,8 +456,10 @@ virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
 		vq->vq_desc_head_idx = dxp->next;
 		if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
 			vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
-		virtio_wmb(hw->weak_barriers);
-		start_dp[idx].flags = flags;
+
+		virtqueue_store_flags_packed(&start_dp[idx], flags,
+					     hw->weak_barriers);
+
 		if (++vq->vq_avail_idx >= vq->vq_nentries) {
 			vq->vq_avail_idx -= vq->vq_nentries;
 			vq->vq_packed.cached_flags ^=
@@ -671,8 +673,7 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	dp->flags = flags;
+	virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
 }
 
 static inline void
@@ -763,8 +764,8 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
 			vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
 	}
 
-	virtio_wmb(vq->hw->weak_barriers);
-	head_dp->flags = head_flags;
+	virtqueue_store_flags_packed(head_dp, head_flags,
+				     vq->hw->weak_barriers);
 }
 
 static inline void
drivers/net/virtio/virtio_user/virtio_user_dev.c +5 -1
index fab87eb..7911c39 100644
@@ -624,7 +624,7 @@ virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
 static inline int
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = desc->flags;
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
@@ -684,6 +684,10 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
 	uint16_t n_descs, flags;
 
+	/* Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
+	 */
 	while (desc_is_avail(&vring->desc[vq->used_idx],
 			     vq->used_wrap_counter)) {
 
drivers/net/virtio/virtqueue.h +21 -0
index c6dd4a3..b728ff8 100644
@@ -54,6 +54,27 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline void
+virtqueue_store_flags_packed(struct vring_packed_desc *dp,
+			      uint16_t flags, uint8_t weak_barriers)
+{
+	if (weak_barriers) {
+/* x86 prefers to using rte_smp_wmb over __atomic_store_n as it reports
+ * a better perf(~1.5%), which comes from the saved branch by the compiler.
+ * The if and else branch are identical with the smp and cio barriers both
+ * defined as compiler barriers on x86.
+ */
+#ifdef RTE_ARCH_X86_64
+		rte_smp_wmb();
+		dp->flags = flags;
+#else
+		__atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
+#endif
+	} else {
+		rte_cio_wmb();
+		dp->flags = flags;
+	}
+}
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
 #else
lib/librte_vhost/vhost.h +1 -1
index 884befa..d294ed1 100644
@@ -344,7 +344,7 @@ vq_is_packed(struct virtio_net *dev)
 static inline bool
 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
 {
-	uint16_t flags = *((volatile uint16_t *) &desc->flags);
+	uint16_t flags = __atomic_load_n(&desc->flags, __ATOMIC_ACQUIRE);
 
 	return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) &&
 		wrap_counter != !!(flags & VRING_DESC_F_USED);
lib/librte_vhost/virtio_net.c +5 -6
index 5b85b83..e7463ff 100644
@@ -503,14 +503,13 @@ fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq,
 	if (avail_idx < vq->last_avail_idx)
 		wrap_counter ^= 1;
 
-	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
-		return -1;
-
 	/*
-	 * The ordering between desc flags and desc
-	 * content reads need to be enforced.
+	 * Perform a load-acquire barrier in desc_is_avail to
+	 * enforce the ordering between desc flags and desc
+	 * content.
 	 */
-	rte_smp_rmb();
+	if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter)))
+		return -1;
 
 	*desc_count = 0;
 	*len = 0;
Signature · 1 line
2.7.4
Reviewed-by on v4 1/2
#24 lore raw

Re: [dpdk-dev] [PATCH v4 1/2] virtio: one way barrier for packed vring desc avail flags

Quoted 20 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for avail flags in packed ring.

Meanwhile, a read barrier is required to ensure ordering between descriptor's flags and content reads[1]. With C11, load-acquire can enforce the ordering instead of rmb barrier.

[1]https://patchwork.dpdk.org/patch/49109/

Quoted 8 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 13 +++++++------
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  6 +++++-
 drivers/net/virtio/virtqueue.h                   | 21 +++++++++++++++++++++
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 11 +++++------
 5 files changed, 39 insertions(+), 14 deletions(-)

Thanks,
Maxime

author
#25 lore raw

PATCH v4 2/2 virtio: one way barrier for packed vring desc used flags

In reply to Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in packed ring.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 28 +++++++++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 40 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +9 -3
index a87ffe1..2f0879c 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
 		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
@@ -233,8 +235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;
 		if (vq->used_idx >= dev->queue_size) {
drivers/net/virtio/virtqueue.h +27 -1
index b728ff8..8d7f197 100644
@@ -54,6 +54,32 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline uint16_t
+virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
+			      uint8_t weak_barriers)
+{
+	uint16_t flags;
+
+	if (weak_barriers) {
+/* x86 prefers to using rte_smp_rmb over __atomic_load_n as it reports
+ * a better perf(~1.5%), which comes from the saved branch by the compiler.
+ * The if and else branch are identical with the smp and cio barriers both
+ * defined as compiler barriers on x86.
+ */
+#ifdef RTE_ARCH_X86_64
+		flags = dp->flags;
+		rte_smp_rmb();
+#else
+		flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE);
+#endif
+	} else {
+		flags = dp->flags;
+		rte_cio_rmb();
+	}
+
+	return flags;
+}
+
 static inline void
 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
 			      uint16_t flags, uint8_t weak_barriers)
@@ -307,7 +333,7 @@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
 {
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
 
lib/librte_vhost/virtio_net.c +2 -3
index e7463ff..241d467 100644
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
Signature · 1 line
2.7.4
#26 lore raw

Re: [dpdk-dev] [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags

Hi Joyce,

I test performance impact of your patch set with code base commit id: d03d8622db48918d14bfe805641b1766ecc40088, after applying your v4 patch set , packed ring shows small performance drop as below:

PVP vhost/virtio 1c1q test commit:d03d8622db48918 apply v4 patch set

pvp_virtio11_mergeable 7.218 7.147
pvp_virtio11_normal 7.217 7.182

Thanks,
Yinan

Quoted 164 lines

-----Original Message-----
From: Joyce Kong [mailto:[email protected]]
Sent: 2019年9月17日 13:28
To: [email protected]
Cc: [email protected]; [email protected]; Wang, Yinan <[email protected]>; Richardson, Bruce <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in packed ring.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 28
+++++++++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 40 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +3 -1
index a87ffe1..2f0879c 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct virtqueue
 *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
drivers/net/virtio/virtio_rxtx.c +7 -2
 		cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; @@ -233,8
+235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct virtqueue
 *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
drivers/net/virtio/virtio_rxtx.c +0 -0
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct virtio_user_dev
 *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;

if (vq->used_idx >= dev->queue_size) { diff --git
a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h index b728ff8..8d7f197 100644

drivers/net/virtio/virtqueue.h +26 -0
@@ -54,6 +54,32 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }
 
+static inline uint16_t
+virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
+			      uint8_t weak_barriers)
+{
+	uint16_t flags;
+
+	if (weak_barriers) {
+/* x86 prefers to using rte_smp_rmb over __atomic_load_n as it reports
+ * a better perf(~1.5%), which comes from the saved branch by the compiler.
+ * The if and else branch are identical with the smp and cio barriers
+both
+ * defined as compiler barriers on x86.
+ */
+#ifdef RTE_ARCH_X86_64
+		flags = dp->flags;
+		rte_smp_rmb();
+#else
+		flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE); #endif
+	} else {
+		flags = dp->flags;
+		rte_cio_rmb();
+	}
+
+	return flags;
+}
+
 static inline void
 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
 			      uint16_t flags, uint8_t weak_barriers) @@ -307,7 +333,7

@@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq) {

drivers/net/virtio/virtqueue.h +1 -1
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
lib/librte_vhost/virtio_net.c index renamed from lib/librte_vhost/virtio_net.c +0 -0

e7463ff..241d467 100644

lib/librte_vhost/virtio_net.c +2 -4
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net *dev,
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
--

2.7.4

#27 lore raw

Re: [dpdk-dev] [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags

Hi Yinan,

Thanks for verification and reporting this, I will dump the assembly code and make an analysis after I am back from travel. From my understanding the assembly code for x86 should be same, but I am not sure if anything we were missing.

Is this <1 perf drop coming from run to run variances, or system noise?

/Gavin

Quoted 215 lines

-----Original Message-----
From: Wang, Yinan <[email protected]>
Sent: Wednesday, September 18, 2019 7:21 AM
To: Joyce Kong (Arm Technology China) <[email protected]>; [email protected]
Cc: nd <[email protected]>; [email protected]; Richardson, Bruce <[email protected]>; Bie, Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected]; Honnappa Nagarahalli <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>
Subject: RE: [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags

Hi Joyce,

I test performance impact of your patch set with code base commit id:
d03d8622db48918d14bfe805641b1766ecc40088, after applying your v4 patch set , packed ring shows small performance drop as below:

PVP vhost/virtio 1c1q test commit:d03d8622db48918 apply v4
patch set

pvp_virtio11_mergeable 7.218 7.147
pvp_virtio11_normal 7.217 7.182

Thanks,
Yinan

Quoted 6 lines

-----Original Message-----
From: Joyce Kong [mailto:[email protected]]
Sent: 2019年9月17日 13:28
To: [email protected]
Cc: [email protected]; [email protected]; Wang, Yinan <[email protected]>; Richardson, Bruce <[email protected]>;

Bie,

Quoted 3 lines

Tiwei <[email protected]>; Wang, Zhihong <[email protected]>; [email protected]; Wang, Xiao W <[email protected]>; Liu, Yong <[email protected]>; [email protected];

[email protected];

Quoted 2 lines

[email protected]
Subject: [PATCH v4 2/2] virtio: one way barrier for packed vring desc used

flags

Quoted 2 lines

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the

frontend

Quoted 1 line

and backend are assumed to be implemented in software, that is they can

run on

Quoted 6 lines

identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in

packed

drivers/net/virtio/virtio_rxtx.c virtqueue_dequeue_burst_rx_packed(struct

Quoted 19 lines

ring.

---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 28
+++++++++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 40 insertions(+), 9 deletions(-)
drivers/net/virtio/virtio_rxtx.c +0 -0
index a87ffe1..2f0879c 100644
@@ -122,9 +122,11 @@ virtqueue_dequeue_burst_rx_packed(struct

virtqueue

Quoted 12 lines
drivers/net/virtio/virtio_rxtx.c +3 -1
 *vq,
 
 	for (i = 0; i < num; i++) {
 		used_idx = vq->vq_used_cons_idx;
+		/* desc_is_used has a load-acquire or rte_cio_rmb inside
+		 * and wait for used desc in virtqueue.
+		 */
 		if (!desc_is_used(&desc[used_idx], vq))
 			return i;
-		virtio_rmb(vq->hw->weak_barriers);
 		len[i] = desc[used_idx].len;
 		id = desc[used_idx].id;
Quoted 1 line

cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie; @@ -

233,8

Quoted 1 line

+235,10 @@ virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int

num)

drivers/net/virtio/virtio_rxtx.c virtio_xmit_cleanup_normal_packed(struct

Quoted 12 lines
drivers/net/virtio/virtio_rxtx.c +3 -1
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		do {
 			curr_id = used_idx;
@@ -265,8 +269,10 @@ virtio_xmit_cleanup_normal_packed(struct

virtqueue

drivers/net/virtio/virtio_user/virtio_user_dev.c virtio_user_handle_cq_packed(struct

Quoted 18 lines
drivers/net/virtio/virtio_rxtx.c +3 -1
 *vq, int num)
 	struct vq_desc_extra *dxp;
 
 	used_idx = vq->vq_used_cons_idx;
+	/* desc_is_used has a load-acquire or rte_cio_rmb inside
+	 * and wait for used desc in virtqueue.
+	 */
 	while (num-- && desc_is_used(&desc[used_idx], vq)) {
-		virtio_rmb(vq->hw->weak_barriers);
 		id = desc[used_idx].id;
 		dxp = &vq->vq_descx[id];
drivers/net/virtio/virtio_rxtx.c +0 -0
 		vq->vq_used_cons_idx += dxp->ndescs;
drivers/net/virtio/virtio_user/virtio_user_dev.c +0 -0
index 7911c39..1c575d0 100644
@@ -698,8 +698,8 @@ virtio_user_handle_cq_packed(struct

virtio_user_dev

Quoted 18 lines
drivers/net/virtio/virtio_user/virtio_user_dev.c +2 -2
 *dev, uint16_t queue_idx)
 		if (vq->used_wrap_counter)
 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
 
-		rte_smp_wmb();
-		vring->desc[vq->used_idx].flags = flags;
+		__atomic_store_n(&vring->desc[vq->used_idx].flags, flags,
+				 __ATOMIC_RELEASE);
 
 		vq->used_idx += n_descs;

if (vq->used_idx >= dev->queue_size) { diff --git
a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h index b728ff8..8d7f197 100644

drivers/net/virtio/virtqueue.h +0 -0
@@ -54,6 +54,32 @@ virtio_wmb(uint8_t weak_barriers)
 		rte_cio_wmb();
 }

drivers/net/virtio/virtqueue.h virtio_wmb(uint8_t weak_barriers) · line 65

Quoted 10 lines
drivers/net/virtio/virtqueue.h +9 -0
 
+static inline uint16_t
+virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
+			      uint8_t weak_barriers)
+{
+	uint16_t flags;
+
+	if (weak_barriers) {
+/* x86 prefers to using rte_smp_rmb over __atomic_load_n as it reports
+ * a better perf(~1.5%), which comes from the saved branch by the

compiler.

drivers/net/virtio/virtqueue.h virtio_wmb(uint8_t weak_barriers) · line 73

Quoted 9 lines
drivers/net/virtio/virtqueue.h +9 -0
+ * The if and else branch are identical with the smp and cio barriers
+both
+ * defined as compiler barriers on x86.
+ */
+#ifdef RTE_ARCH_X86_64
+		flags = dp->flags;
+		rte_smp_rmb();
+#else
+		flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE);

#endif

drivers/net/virtio/virtqueue.h virtio_wmb(uint8_t weak_barriers)

Quoted 11 lines
drivers/net/virtio/virtqueue.h +8 -0
+	} else {
+		flags = dp->flags;
+		rte_cio_rmb();
+	}
+
+	return flags;
+}
+
 static inline void
 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
 			      uint16_t flags, uint8_t weak_barriers) @@ -307,7

+333,7

lib/librte_vhost/virtio_net.c flush_shadow_used_ring_packed(struct virtio_net

Quoted 13 lines

@@ desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq) {

drivers/net/virtio/virtqueue.h +1 -1
 	uint16_t used, avail, flags;
 
-	flags = desc->flags;
+	flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
 	used = !!(flags & VRING_PACKED_DESC_F_USED);
 	avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
lib/librte_vhost/virtio_net.c index renamed from lib/librte_vhost/virtio_net.c +0 -0

e7463ff..241d467 100644

lib/librte_vhost/virtio_net.c +0 -0
@@ -110,8 +110,6 @@ flush_shadow_used_ring_packed(struct virtio_net

*dev,

lib/librte_vhost/virtio_net.c flush_shadow_used_ring_packed(struct virtio_net

Quoted 9 lines
lib/librte_vhost/virtio_net.c +0 -2
 			used_idx -= vq->size;
 	}
 
-	rte_smp_wmb();
-
 	for (i = 0; i < vq->shadow_used_idx; i++) {
 		uint16_t flags;
 
@@ -147,7 +145,8 @@ flush_shadow_used_ring_packed(struct virtio_net

*dev,

Quoted 11 lines
lib/librte_vhost/virtio_net.c +2 -2
 		}
 	}
 
-	vq->desc_packed[head_idx].flags = head_flags;
+	__atomic_store_n(&vq->desc_packed[head_idx].flags, head_flags,
+			 __ATOMIC_RELEASE);
 
 	vhost_log_cache_used_vring(dev, vq,
 				head_idx *
--

2.7.4

Reviewed-by on v4 2/2
#28 lore raw

Re: [dpdk-dev] [PATCH v4 2/2] virtio: one way barrier for packed vring desc used flags

In reply to Joyce Kong
Quoted 14 lines · Joyce Kong

In case VIRTIO_F_ORDER_PLATFORM(36) is not negotiated, then the frontend and backend are assumed to be implemented in software, that is they can run on identical CPUs in an SMP configuration.
Thus a weak form of memory barriers like rte_smp_r/wmb, other than rte_cio_r/wmb, is sufficient for this case(vq->hw->weak_barriers == 1) and yields better performance.
For the above case, this patch helps yielding even better performance by replacing the two-way barriers with C11 one-way barriers for used flags in packed ring.

Quoted 6 lines
---
 drivers/net/virtio/virtio_rxtx.c                 | 12 +++++++---
 drivers/net/virtio/virtio_user/virtio_user_dev.c |  4 ++--
 drivers/net/virtio/virtqueue.h                   | 28 +++++++++++++++++++++++-
 lib/librte_vhost/virtio_net.c                    |  5 ++---
 4 files changed, 40 insertions(+), 9 deletions(-)

Thanks,
Maxime

#29 lore raw
In reply to Joyce Kong
Quoted 28 lines · Joyce Kong

This patch set replaces the two-way barriers with C11 one-way barriers for packed vring flags, when the frontend and backend are implemented in software.

By doing vhost-user + virtio-user case benchmarking, 9% performance gain in the RFC2544 test was measured on Thunderx2 platform.[1] And by doing VM2VM case benchmarking, 11% perf gain was measured on Ampere platform.

[1]https://doc.dpdk.org/dts/test_plans/pvp_multi_paths_performance_test_plan.html
PVP test with virtio 1.1 mergeable path

v4:
Use rte_smp_rmb/wmb instead of __atomic_load/store_n on x86 as it reports a better perf(~1.5%), which comes from the saved branch by the compiler. The if and else branch are identical with the smp and cio barriers both defined as compiler barriers on x86.
http://inbox.dpdk.org/dev/E0CBA5A1980F1F408E1F28F9991B5B1D50EFF246@SHSMSX104.ccr.corp.intel.com/

v3:
Wrap C11 one-way barriers and DMA barriers(rte_cio_*) together with an inline fuction.

v2:
Convert RFC to patch.

Joyce Kong (2):
virtio: one way barrier for packed vring desc avail flags
virtio: one way barrier for packed vring desc used flags

Quoted 7 lines
 drivers/net/virtio/virtio_rxtx.c                 | 25 +++++++-----
 drivers/net/virtio/virtio_user/virtio_user_dev.c | 10 +++--
 drivers/net/virtio/virtqueue.h                   | 49 +++++++++++++++++++++++-
 lib/librte_vhost/vhost.h                         |  2 +-
 lib/librte_vhost/virtio_net.c                    | 16 ++++----
 5 files changed, 79 insertions(+), 23 deletions(-)

Applied to dpdk-next-virtio/master.

Thanks,
Maxime