Skip to contentSkip to conversation

net/virtio: add platform memory ordering feature support

PATCH Superseded Ilya Maximets 6 participants 28 messages Dec 14, 2018 – Feb 12, 2019 1 message not in the archive Network drivers

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): CGME20181214153817eucas1p19a41cdd791879252e1f3a5d77c427845@eucas1p1.samsung.com

#1 lore raw

PATCH 1/1 net/virtio: add platform memory ordering feature support

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Based on "[RFC] net/virtio: use real barriers for vDPA".

RFC --> PATCH:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 ++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 14 ++++++------
 drivers/net/virtio/virtqueue.h     | 35 +++++++++++++++++++++++++-----
 5 files changed, 48 insertions(+), 13 deletions(-)
drivers/net/virtio/virtio_ethdev.c +2 -0
index cb2b2e0bf..5ae7a9650 100644
@@ -1474,6 +1474,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
 	if (virtio_negotiate_features(hw, req_features) < 0)
 		return -1;
 
+	hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
+
 	if (!hw->virtio_user_dev) {
 		pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
drivers/net/virtio/virtio_ethdev.h +2 -1
index e0f80e5a4..c098e5ac0 100644
@@ -34,7 +34,8 @@
 	 1u << VIRTIO_RING_F_INDIRECT_DESC |    \
 	 1ULL << VIRTIO_F_VERSION_1       |	\
 	 1ULL << VIRTIO_F_IN_ORDER        |	\
-	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
+	 1ULL << VIRTIO_F_IOMMU_PLATFORM  |	\
+	 1ULL << VIRTIO_F_ORDER_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
drivers/net/virtio/virtio_pci.h +7 -0
index e961a58ca..e2f096185 100644
@@ -128,6 +128,12 @@ struct virtnet_ctl;
  */
 #define VIRTIO_F_IN_ORDER 35
 
+/*
+ * This feature indicates that memory accesses by the driver and the device
+ * are ordered in a way described by the platform.
+ */
+#define VIRTIO_F_ORDER_PLATFORM 36
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -240,6 +246,7 @@ struct virtio_hw {
 	uint8_t     use_simple_rx;
 	uint8_t     use_inorder_rx;
 	uint8_t     use_inorder_tx;
+	uint8_t     weak_barriers;
 	bool        has_tx_offload;
 	bool        has_rx_offload;
 	uint16_t    port_id;
drivers/net/virtio/virtio_rxtx.c +7 -7
index cb8f89f18..66195bf47 100644
@@ -906,7 +906,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
 	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
@@ -1017,7 +1017,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 	nb_used = RTE_MIN(nb_used, nb_pkts);
 	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1202,7 +1202,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1365,7 +1365,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup(vq, nb_used);
 
@@ -1407,7 +1407,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup(vq, need);
@@ -1463,7 +1463,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup_inorder(vq, nb_used);
 
@@ -1511,7 +1511,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 		need = slots - vq->vq_free_cnt;
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup_inorder(vq, need);
drivers/net/virtio/virtqueue.h +30 -5
index 26518ed98..6b9055a1f 100644
@@ -19,15 +19,40 @@
 struct rte_mbuf;
 
 /*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.
  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.
  *
+ *     For using virtio to talk to real devices (eg. vDPA) we do need real
+ *     barriers.
  */
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();
+}
 
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
@@ -312,7 +337,7 @@ void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
 static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;
 }
 
Signature · 1 line
2.17.1
#2 lore raw
Quoted 165 lines · Ilya Maximets

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Based on "[RFC] net/virtio: use real barriers for vDPA".

RFC --> PATCH:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 ++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 14 ++++++------
 drivers/net/virtio/virtqueue.h     | 35 +++++++++++++++++++++++++-----
 5 files changed, 48 insertions(+), 13 deletions(-)
drivers/net/virtio/virtio_ethdev.c +2 -0
index cb2b2e0bf..5ae7a9650 100644
@@ -1474,6 +1474,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
 	if (virtio_negotiate_features(hw, req_features) < 0)
 		return -1;
 
+	hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
+
 	if (!hw->virtio_user_dev) {
 		pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
drivers/net/virtio/virtio_ethdev.h +2 -1
index e0f80e5a4..c098e5ac0 100644
@@ -34,7 +34,8 @@
 	 1u << VIRTIO_RING_F_INDIRECT_DESC |    \
 	 1ULL << VIRTIO_F_VERSION_1       |	\
 	 1ULL << VIRTIO_F_IN_ORDER        |	\
-	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
+	 1ULL << VIRTIO_F_IOMMU_PLATFORM  |	\
+	 1ULL << VIRTIO_F_ORDER_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
drivers/net/virtio/virtio_pci.h +7 -0
index e961a58ca..e2f096185 100644
@@ -128,6 +128,12 @@ struct virtnet_ctl;
  */
 #define VIRTIO_F_IN_ORDER 35
 
+/*
+ * This feature indicates that memory accesses by the driver and the device
+ * are ordered in a way described by the platform.
+ */
+#define VIRTIO_F_ORDER_PLATFORM 36
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -240,6 +246,7 @@ struct virtio_hw {
 	uint8_t     use_simple_rx;
 	uint8_t     use_inorder_rx;
 	uint8_t     use_inorder_tx;
+	uint8_t     weak_barriers;
 	bool        has_tx_offload;
 	bool        has_rx_offload;
 	uint16_t    port_id;
drivers/net/virtio/virtio_rxtx.c +7 -7
index cb8f89f18..66195bf47 100644
@@ -906,7 +906,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
 	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
@@ -1017,7 +1017,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 	nb_used = RTE_MIN(nb_used, nb_pkts);
 	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1202,7 +1202,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1365,7 +1365,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup(vq, nb_used);
 
@@ -1407,7 +1407,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup(vq, need);
@@ -1463,7 +1463,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup_inorder(vq, nb_used);
 
@@ -1511,7 +1511,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 		need = slots - vq->vq_free_cnt;
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup_inorder(vq, need);
drivers/net/virtio/virtqueue.h +3 -3
index 26518ed98..6b9055a1f 100644
@@ -19,15 +19,40 @@
 struct rte_mbuf;
 
 /*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.
  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.
  *
+ *     For using virtio to talk to real devices (eg. vDPA) we do need real
+ *     barriers.
  */
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()

drivers/net/virtio/virtqueue.h line 38

Quoted 10 lines
drivers/net/virtio/virtqueue.h +9 -1
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+

Why doesn't rte_cio_rmb exit?

Quoted 31 lines
drivers/net/virtio/virtqueue.h +18 -1
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();
+}
 
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
@@ -312,7 +337,7 @@ void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
 static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;
 }
 
Signature · 1 line
2.17.1
#3 lore raw
Quoted 170 lines · Michael S. Tsirkin
Quoted 169 lines · Ilya Maximets

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Based on "[RFC] net/virtio: use real barriers for vDPA".

RFC --> PATCH:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 ++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 14 ++++++------
 drivers/net/virtio/virtqueue.h     | 35 +++++++++++++++++++++++++-----
 5 files changed, 48 insertions(+), 13 deletions(-)
drivers/net/virtio/virtio_ethdev.c +2 -0
index cb2b2e0bf..5ae7a9650 100644
@@ -1474,6 +1474,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
 	if (virtio_negotiate_features(hw, req_features) < 0)
 		return -1;
 
+	hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
+
 	if (!hw->virtio_user_dev) {
 		pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
drivers/net/virtio/virtio_ethdev.h +2 -1
index e0f80e5a4..c098e5ac0 100644
@@ -34,7 +34,8 @@
 	 1u << VIRTIO_RING_F_INDIRECT_DESC |    \
 	 1ULL << VIRTIO_F_VERSION_1       |	\
 	 1ULL << VIRTIO_F_IN_ORDER        |	\
-	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
+	 1ULL << VIRTIO_F_IOMMU_PLATFORM  |	\
+	 1ULL << VIRTIO_F_ORDER_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
drivers/net/virtio/virtio_pci.h +7 -0
index e961a58ca..e2f096185 100644
@@ -128,6 +128,12 @@ struct virtnet_ctl;
  */
 #define VIRTIO_F_IN_ORDER 35
 
+/*
+ * This feature indicates that memory accesses by the driver and the device
+ * are ordered in a way described by the platform.
+ */
+#define VIRTIO_F_ORDER_PLATFORM 36
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -240,6 +246,7 @@ struct virtio_hw {
 	uint8_t     use_simple_rx;
 	uint8_t     use_inorder_rx;
 	uint8_t     use_inorder_tx;
+	uint8_t     weak_barriers;
 	bool        has_tx_offload;
 	bool        has_rx_offload;
 	uint16_t    port_id;
drivers/net/virtio/virtio_rxtx.c +7 -7
index cb8f89f18..66195bf47 100644
@@ -906,7 +906,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
 	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
@@ -1017,7 +1017,7 @@ virtio_recv_mergeable_pkts_inorder(void *rx_queue,
 	nb_used = RTE_MIN(nb_used, nb_pkts);
 	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1202,7 +1202,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1365,7 +1365,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup(vq, nb_used);
 
@@ -1407,7 +1407,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup(vq, need);
@@ -1463,7 +1463,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup_inorder(vq, nb_used);
 
@@ -1511,7 +1511,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 		need = slots - vq->vq_free_cnt;
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup_inorder(vq, need);
drivers/net/virtio/virtqueue.h +6 -4
index 26518ed98..6b9055a1f 100644
@@ -19,15 +19,40 @@
 struct rte_mbuf;
 
 /*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.
  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.
  *
+ *     For using virtio to talk to real devices (eg. vDPA) we do need real
+ *     barriers.
  */
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
Quoted 8 lines

drivers/net/virtio/virtqueue.h line 38

Quoted 6 lines
drivers/net/virtio/virtqueue.h +6 -0
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+

Why doesn't rte_cio_rmb exit?

Assuming your question was "Why doesn't rte_cio_mb exist?". I guess 'cio' barriers was copied from 'dma_' barriers from kernel. And 'rte_cio_mb' does not exist because there is no 'dma_mb' in kernel.

OTOH, maybe we can use 'rte_io_mb' here to be more consistent, but it equals to the 'rte_mb' on all supported architectures. So, I'm not sure if it's needed.

Quoted 34 lines
Quoted 32 lines
drivers/net/virtio/virtqueue.h +18 -1
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();
+}
 
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
@@ -312,7 +337,7 @@ void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
 static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;
 }
 
Signature · 1 line
2.17.1
#4 lore raw

PATCH v2 1/1 net/virtio: add platform memory ordering feature support

In reply to Ilya Maximets

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 ++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++++------
 drivers/net/virtio/virtqueue.h     | 39 ++++++++++++++++++++++++------
 5 files changed, 51 insertions(+), 16 deletions(-)
drivers/net/virtio/virtio_ethdev.c +2 -0
index 446c338fc..6d461180c 100644
@@ -1613,6 +1613,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
 	if (virtio_negotiate_features(hw, req_features) < 0)
 		return -1;
 
+	hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
+
 	if (!hw->virtio_user_dev) {
 		pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
drivers/net/virtio/virtio_ethdev.h +2 -1
index f8d8a56ab..b8aab7da4 100644
@@ -35,7 +35,8 @@
 	 1ULL << VIRTIO_F_VERSION_1       |	\
 	 1ULL << VIRTIO_F_IN_ORDER        |	\
 	 1ULL << VIRTIO_F_RING_PACKED	  |	\
-	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
+	 1ULL << VIRTIO_F_IOMMU_PLATFORM  |	\
+	 1ULL << VIRTIO_F_ORDER_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
drivers/net/virtio/virtio_pci.h +7 -0
index b22b62dad..38a0261da 100644
@@ -129,6 +129,12 @@ struct virtnet_ctl;
  */
 #define VIRTIO_F_IN_ORDER 35
 
+/*
+ * This feature indicates that memory accesses by the driver and the device
+ * are ordered in a way described by the platform.
+ */
+#define VIRTIO_F_ORDER_PLATFORM 36
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -241,6 +247,7 @@ struct virtio_hw {
 	uint8_t     use_simple_rx;
 	uint8_t     use_inorder_rx;
 	uint8_t     use_inorder_tx;
+	uint8_t     weak_barriers;
 	bool        has_tx_offload;
 	bool        has_rx_offload;
 	uint16_t    port_id;
drivers/net/virtio/virtio_rxtx.c +8 -8
index 2309b71d6..ebb86ef70 100644
@@ -1152,7 +1152,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
 	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
@@ -1361,7 +1361,7 @@ virtio_recv_pkts_inorder(void *rx_queue,
 	nb_used = RTE_MIN(nb_used, nb_pkts);
 	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1549,7 +1549,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1940,7 +1940,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_pkts);
 			virtio_xmit_cleanup_packed(vq, need);
 			need = slots - vq->vq_free_cnt;
@@ -1988,7 +1988,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup(vq, nb_used);
 
@@ -2030,7 +2030,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup(vq, need);
@@ -2086,7 +2086,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup_inorder(vq, nb_used);
 
@@ -2134,7 +2134,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 		need = slots - vq->vq_free_cnt;
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup_inorder(vq, need);
drivers/net/virtio/virtqueue.h +32 -7
index d8ae5cdec..a66a37f61 100644
@@ -19,15 +19,40 @@
 struct rte_mbuf;
 
 /*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.
  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.
  *
+ *     For using virtio to talk to real devices (eg. vDPA) we do need real
+ *     barriers.
  */
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();
+}
 
 #ifdef RTE_PMD_PACKET_PREFETCH
 #define rte_packet_prefetch(p)  rte_prefetch1(p)
@@ -325,7 +350,7 @@ virtqueue_enable_intr_packed(struct virtqueue *vq)
 
 
 	if (vq->event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {
-		virtio_wmb();
+		virtio_wmb(vq->hw->weak_barriers);
 		vq->event_flags_shadow = RING_EVENT_FLAGS_ENABLE;
 		*event_flags = vq->event_flags_shadow;
 	}
@@ -391,7 +416,7 @@ void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
 static inline void
 vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;
 }
 
@@ -423,7 +448,7 @@ virtqueue_kick_prepare_packed(struct virtqueue *vq)
 {
 	uint16_t flags;
 
-	virtio_mb();
+	virtio_mb(vq->hw->weak_barriers);
 	flags = vq->ring_packed.device_event->desc_event_flags;
 
 	return flags != RING_EVENT_FLAGS_DISABLE;
Signature · 1 line
2.17.1
#5 lore raw

Hi Ilya,

Wednesday, December 26, 2018 6:37 PM, Ilya Maximets:

Quoted 36 lines

Subject: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:

https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.m
ail-archive.com%2Fvirtio-dev%40lists.oasis- open.org%2Fmsg04114.html&amp;data=02%7C01%7Cshahafs%40mellanox.co m%7C147684bb9b754e30781408d66b506965%7Ca652971c7d2e4d9ba6a4d149 256f461b%7C0%7C0%7C636814390449088148&amp;sdata=f4W1YLftBtZ4zAQ3 %2Bv7LV5w%2FDhM5aWpROV2c2gHY8iU%3D&amp;reserved=0

 drivers/net/virtio/virtio_ethdev.c |  2 ++  drivers/net/virtio/virtio_ethdev.h |  3
++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++++------
 drivers/net/virtio/virtqueue.h     | 39 ++++++++++++++++++++++++------
 5 files changed, 51 insertions(+), 16 deletions(-)

[...]

Quoted 29 lines

/*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.

  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.

*
+ * For using virtio to talk to real devices (eg. vDPA) we do need real
+ * barriers.
*/

drivers/net/virtio/virtqueue.h +15 -3
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else

drivers/net/virtio/virtqueue.h line 54

Quoted 10 lines
drivers/net/virtio/virtqueue.h +10 -0
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();

Just wondering if the cio barrier is enough here.
This virtio_wmb will be called, for example in the following sequence for transmit (not of packed queue):

if (likely(nb_tx)) {                                                 
        vq_update_avail_idx(vq);                <--- virito_wmb()                     
                                                                     
        if (unlikely(virtqueue_kick_prepare(vq))) {                 <--- no barrier 
                virtqueue_notify(vq);                                
                PMD_TX_LOG(DEBUG, "Notified backend after xmit");    
        }                                                            
}                                                                    

Assuming the backhand has vDPA device. The notify will be most likely mapped to the device PCI bar as write combining.
This means we need to keep ordering here between two memory domains - the PCI and the host local memory. We must make sure that when the notify reaches the device, the store to the host local memory is already written to memory (and not in the write buffer). Is complier barrier is enough for such purpose? Or we need something stronger like sfence?

Note #1 - This issue (if exists) is not caused directly by your code, however you are making things right here w/ respect to memory ordering so worth taking care also on this one Note #2 - if indeed there is an issue here, for packed queue we are somewhat OK, since virtio_mb() will be called before the kick. I am not sure why we need such hard barrier and sfence is not enough. Do you know?

Quoted 30 lines

+}

#ifdef RTE_PMD_PACKET_PREFETCH
#define rte_packet_prefetch(p) rte_prefetch1(p) @@ -325,7 +350,7 @@
virtqueue_enable_intr_packed(struct virtqueue *vq)

drivers/net/virtio/virtqueue.h +3 -4
 	if (vq->event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {
-		virtio_wmb();
+		virtio_wmb(vq->hw->weak_barriers);
 		vq->event_flags_shadow = RING_EVENT_FLAGS_ENABLE;
 		*event_flags = vq->event_flags_shadow;
 	}
@@ -391,7 +416,7 @@ void vq_ring_free_inorder(struct virtqueue *vq,
 uint16_t desc_idx,  static inline void  vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;  }
 
@@ -423,7 +448,7 @@ virtqueue_kick_prepare_packed(struct virtqueue *vq)  {
 	uint16_t flags;
 
-	virtio_mb();
+	virtio_mb(vq->hw->weak_barriers);
 	flags = vq->ring_packed.device_event->desc_event_flags;
 
 	return flags != RING_EVENT_FLAGS_DISABLE;
--

2.17.1

#6 lore raw
Quoted 93 lines · Shahaf Shuler

Hi Ilya,

Wednesday, December 26, 2018 6:37 PM, Ilya Maximets:

Quoted 36 lines

Subject: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:

https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww.m
ail-archive.com%2Fvirtio-dev%40lists.oasis- open.org%2Fmsg04114.html&amp;data=02%7C01%7Cshahafs%40mellanox.co m%7C147684bb9b754e30781408d66b506965%7Ca652971c7d2e4d9ba6a4d149 256f461b%7C0%7C0%7C636814390449088148&amp;sdata=f4W1YLftBtZ4zAQ3 %2Bv7LV5w%2FDhM5aWpROV2c2gHY8iU%3D&amp;reserved=0

 drivers/net/virtio/virtio_ethdev.c |  2 ++  drivers/net/virtio/virtio_ethdev.h |  3
++-
 drivers/net/virtio/virtio_pci.h    |  7 ++++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++++------
 drivers/net/virtio/virtqueue.h     | 39 ++++++++++++++++++++++++------
 5 files changed, 51 insertions(+), 16 deletions(-)

[...]

Quoted 29 lines

/*
- * Per virtio_config.h in Linux.
+ * Per virtio_ring.h in Linux.

  *     For virtio_pci on SMP, we don't need to order with respect to MMIO
  *     accesses through relaxed memory I/O windows, so smp_mb() et al are
  *     sufficient.

*
+ * For using virtio to talk to real devices (eg. vDPA) we do need real
+ * barriers.
*/

drivers/net/virtio/virtqueue.h +15 -3
-#define virtio_mb()	rte_smp_mb()
-#define virtio_rmb()	rte_smp_rmb()
-#define virtio_wmb()	rte_smp_wmb()
+static inline void
+virtio_mb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_mb();
+	else
+		rte_mb();
+}
+
+static inline void
+virtio_rmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_rmb();
+	else

drivers/net/virtio/virtqueue.h line 54

Quoted 10 lines
drivers/net/virtio/virtqueue.h +10 -0
+		rte_cio_rmb();
+}
+
+static inline void
+virtio_wmb(uint8_t weak_barriers)
+{
+	if (weak_barriers)
+		rte_smp_wmb();
+	else
+		rte_cio_wmb();

Just wondering if the cio barrier is enough here.
This virtio_wmb will be called, for example in the following sequence for transmit (not of packed queue):

if (likely(nb_tx)) {                                                 
        vq_update_avail_idx(vq);                <--- virito_wmb()                     
                                                                     
        if (unlikely(virtqueue_kick_prepare(vq))) {                 <--- no barrier 
                virtqueue_notify(vq);                                
                PMD_TX_LOG(DEBUG, "Notified backend after xmit");    
        }                                                            
}                                                                    
Quoted 6 lines

Assuming the backhand has vDPA device. The notify will be most likely mapped to the device PCI bar as write combining.
This means we need to keep ordering here between two memory domains - the PCI and the host local memory. We must make sure that when the notify reaches the device, the store to the host local memory is already written to memory (and not in the write buffer). Is complier barrier is enough for such purpose? Or we need something stronger like sfence?

Note #1 - This issue (if exists) is not caused directly by your code, however you are making things right here w/ respect to memory ordering so worth taking care also on this one Note #2 - if indeed there is an issue here, for packed queue we are somewhat OK, since virtio_mb() will be called before the kick. I am not sure why we need such hard barrier and sfence is not enough. Do you know?

Hmm. Thanks for spotting this.
The issue exists, but it's a bit different.
Looks like we have missed virtio_mb() for the split case inside the virtqueue_kick_prepare(). It's required because we must ensure the ordering between writing the data (updating avail->idx) and reading the used->flags. Otherwise we could catch the case where data written, but virtqueue wasn't notified and vhost waits for notification. This is not the vDPA related issue. This could happen with kernel vhost. Adding the virtio_mb() to virtqueue_kick_prepare() will solve the vDPA case automatically. I'll prepare the patches.

Beside that, Jens, between v10 and v11 of the packed queues support patch-set you added the virtio_mb() to kick_prepare function. I guess, this could be the root cause of the performance drop you spotted in compare with split queues. Maybe you could check and prove that point?
I didn't found why you done that change (there are no such review comments on the list), but it was right decision.

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Quoted 33 lines
Quoted 32 lines

+}

#ifdef RTE_PMD_PACKET_PREFETCH
#define rte_packet_prefetch(p) rte_prefetch1(p) @@ -325,7 +350,7 @@
virtqueue_enable_intr_packed(struct virtqueue *vq)

drivers/net/virtio/virtqueue.h +3 -4
 	if (vq->event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {
-		virtio_wmb();
+		virtio_wmb(vq->hw->weak_barriers);
 		vq->event_flags_shadow = RING_EVENT_FLAGS_ENABLE;
 		*event_flags = vq->event_flags_shadow;
 	}
@@ -391,7 +416,7 @@ void vq_ring_free_inorder(struct virtqueue *vq,
 uint16_t desc_idx,  static inline void  vq_update_avail_idx(struct virtqueue *vq)
 {
-	virtio_wmb();
+	virtio_wmb(vq->hw->weak_barriers);
 	vq->vq_ring.avail->idx = vq->vq_avail_idx;  }
 
@@ -423,7 +448,7 @@ virtqueue_kick_prepare_packed(struct virtqueue *vq)  {
 	uint16_t flags;
 
-	virtio_mb();
+	virtio_mb(vq->hw->weak_barriers);
 	flags = vq->ring_packed.device_event->desc_event_flags;
 
 	return flags != RING_EVENT_FLAGS_DISABLE;
--

2.17.1

#7 lore raw
Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

Signature · 1 line
MST
#8 lore raw

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory. The only write which is to uncachable memory (device memory) is the notify itself.

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

[1]
https://patches.dpdk.org/patch/49545/

Quoted 3 lines

--
MST

#9 lore raw

Thursday, January 10, 2019 10:37 PM, Shahaf Shuler:

Quoted 26 lines

Subject: RE: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory.
The only write which is to uncachable memory (device memory) is the notify itself.

Quoted 3 lines

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

Any answer?
It is pity if we add redundant barriers which will degrade the driver performance.

Quoted 7 lines

[1]
https://patches.dpdk.org/patch/49545/

Quoted 3 lines

--
MST

#10 lore raw
Quoted 27 lines · Shahaf Shuler

Thursday, January 10, 2019 10:37 PM, Shahaf Shuler:

Quoted 26 lines

Subject: RE: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory.
The only write which is to uncachable memory (device memory) is the notify itself.

Quoted 6 lines
Quoted 3 lines

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

Any answer?
It is pity if we add redundant barriers which will degrade the driver performance.

Sorry for late responses. Have a lot of work with OVS right now.

Regarding your question.
Current code looks like this:

1. Update ring.
2. virtio_wmb()
3. Update idx.
4. virtio_mb()
5. read flags.
6. notify.

virtio_mb() is here to avoid reordering of steps 3 and 5.
i.e. we need a full barrier to ensure the order between store (idx update) and load (reading the flags). Otherwise we could miss the notification. We can't avoid the barrier here, because even x86 does not guarantee the ordering of the local load with earlier local store.

Quoted 10 lines
Quoted 8 lines

[1]
https://patches.dpdk.org/patch/49545/

Quoted 3 lines

--
MST

#11 lore raw

Tuesday, January 15, 2019 10:29 AM, Ilya Maximets:

Quoted 51 lines

Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 27 lines · Shahaf Shuler

Thursday, January 10, 2019 10:37 PM, Shahaf Shuler:

Quoted 26 lines

Subject: RE: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory.
The only write which is to uncachable memory (device memory) is the notify itself.

Quoted 6 lines
Quoted 3 lines

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

Any answer?
It is pity if we add redundant barriers which will degrade the driver

performance.

Sorry for late responses. Have a lot of work with OVS right now.

Regarding your question.
Current code looks like this:

1. Update ring.
2. virtio_wmb()
3. Update idx.
4. virtio_mb()
5. read flags.
6. notify.

Quoted 5 lines

virtio_mb() is here to avoid reordering of steps 3 and 5.
i.e. we need a full barrier to ensure the order between store (idx update) and load (reading the flags). Otherwise we could miss the notification. We can't avoid the barrier here, because even x86 does not guarantee the ordering of the local load with earlier local store.

This is clear. You need the rte_smp_mb() here. My question is why you need the rte_mb() in case of vDPA? As you said, all accesses are local.

Pasting you commit code:
/*
* Per virtio_ring.h in Linux.

 *     For virtio_pci on SMP, we don't need to order with respect to MMIO 
 *     accesses through relaxed memory I/O windows, so smp_mb() et al are 
 *     sufficient.                                                        

*
* For using virtio to talk to real devices (eg. vDPA) we do need real
* barriers.
*/
static inline void
virtio_mb(uint8_t weak_barriers)

{                                                                         
        if (weak_barriers)                                                
                rte_smp_mb();                                             
        else                                                              
                rte_mb();                                                 
}                                                                         
Quoted 22 lines
Quoted 5 lines
Quoted 5 lines

[1]

https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpa

Quoted 1 line
Quoted 1 line

tches.dpdk.org%2Fpatch%2F49545%2F&amp;data=02%7C01%7Cshahafs%40 mellan

Quoted 1 line
Quoted 1 line

ox.com%7C01907f1b2e0e4002cb7508d67ac38a98%7Ca652971c7d2e4d9ba6a4 d1492

Quoted 1 line
Quoted 1 line

56f461b%7C0%7C0%7C636831377591864200&amp;sdata=TSpc%2Fzyq2aq0N3 %2Bh4o

Quoted 7 lines
Quoted 5 lines

ro4std8ut%2FQU6%2BOeMDeeaQdsM%3D&amp;reserved=0

Quoted 3 lines

--
MST

#12 lore raw
Quoted 52 lines · Shahaf Shuler

Tuesday, January 15, 2019 10:29 AM, Ilya Maximets:

Quoted 51 lines

Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 27 lines · Shahaf Shuler

Thursday, January 10, 2019 10:37 PM, Shahaf Shuler:

Quoted 26 lines

Subject: RE: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory.
The only write which is to uncachable memory (device memory) is the notify itself.

Quoted 6 lines
Quoted 3 lines

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

Any answer?
It is pity if we add redundant barriers which will degrade the driver

performance.

Sorry for late responses. Have a lot of work with OVS right now.

Regarding your question.
Current code looks like this:

1. Update ring.
2. virtio_wmb()
3. Update idx.
4. virtio_mb()
5. read flags.
6. notify.

Quoted 8 lines
Quoted 5 lines

virtio_mb() is here to avoid reordering of steps 3 and 5.
i.e. we need a full barrier to ensure the order between store (idx update) and load (reading the flags). Otherwise we could miss the notification. We can't avoid the barrier here, because even x86 does not guarantee the ordering of the local load with earlier local store.

This is clear. You need the rte_smp_mb() here. My question is why you need the rte_mb() in case of vDPA? As you said, all accesses are local.

Sorry, I misunderstood your question.
Memory accesses are not local here. We want the ring data/idx be visible to vDPA HW before reading the flags.

Quoted 42 lines

Pasting you commit code:
/*
* Per virtio_ring.h in Linux.

 *     For virtio_pci on SMP, we don't need to order with respect to MMIO 
 *     accesses through relaxed memory I/O windows, so smp_mb() et al are 
 *     sufficient.                                                        

*
* For using virtio to talk to real devices (eg. vDPA) we do need real
* barriers.
*/
static inline void
virtio_mb(uint8_t weak_barriers)

{                                                                         
        if (weak_barriers)                                                
                rte_smp_mb();                                             
        else                                                              
                rte_mb();                                                 
}                                                                         
Quoted 22 lines
Quoted 5 lines
Quoted 5 lines

[1]

https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpa

Quoted 1 line
Quoted 1 line

tches.dpdk.org%2Fpatch%2F49545%2F&amp;data=02%7C01%7Cshahafs%40 mellan

Quoted 1 line
Quoted 1 line

ox.com%7C01907f1b2e0e4002cb7508d67ac38a98%7Ca652971c7d2e4d9ba6a4 d1492

Quoted 1 line
Quoted 1 line

56f461b%7C0%7C0%7C636831377591864200&amp;sdata=TSpc%2Fzyq2aq0N3 %2Bh4o

Quoted 7 lines
Quoted 5 lines

ro4std8ut%2FQU6%2BOeMDeeaQdsM%3D&amp;reserved=0

Quoted 3 lines

--
MST

#13 lore raw
In reply to Shahaf Shuler
Quoted 52 lines · Shahaf Shuler

Tuesday, January 15, 2019 10:29 AM, Ilya Maximets:

Quoted 51 lines

Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 27 lines · Shahaf Shuler

Thursday, January 10, 2019 10:37 PM, Shahaf Shuler:

Quoted 26 lines

Subject: RE: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Wednesday, January 9, 2019 5:50 PM, Michael S. Tsirkin:

Quoted 10 lines

[email protected]; Daniel Marcovitch <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2] net/virtio: add platform memory ordering feature support

Quoted 2 lines · Ilya Maximets

virtio_mb() is really heavy. I'd like to avoid it somehow, but I don't know how to do this yet.

Linux driver doesn't avoid it either.

I understand v3 was merged but still would like to continue the discuss and make sure all is clear and agreed.

Form patch [1] description it is very clear why we need the rte_smp_mb() barrier.
However I am not sure why this barrier is interoperate into rte_mb in case of vDPA. In vDPA case, both read of the user ring and write of the avail index are for local cached memory.
The only write which is to uncachable memory (device memory) is the notify itself.

Quoted 6 lines
Quoted 3 lines

As I mentioned, there is a need to have a store fence before doing the notify, but from different reasons. So vDPA use case and need Is a bit different than what presented in [1].

Any answer?
It is pity if we add redundant barriers which will degrade the driver

performance.

Sorry for late responses. Have a lot of work with OVS right now.

Regarding your question.
Current code looks like this:

1. Update ring.
2. virtio_wmb()
3. Update idx.
4. virtio_mb()
5. read flags.
6. notify.

Quoted 8 lines
Quoted 5 lines

virtio_mb() is here to avoid reordering of steps 3 and 5.
i.e. we need a full barrier to ensure the order between store (idx update) and load (reading the flags). Otherwise we could miss the notification. We can't avoid the barrier here, because even x86 does not guarantee the ordering of the local load with earlier local store.

This is clear. You need the rte_smp_mb() here. My question is why you need the rte_mb() in case of vDPA? As you said, all accesses are local.

Are you asking why the different barriers? Or as you asking why is a barrier needed at all?

The barriers themselves are clearly needed.

But in my opinion some dpdk barrier implementations are sub-optimal and too strong. For example on intel: the big question is whether anyone does any non-temporals. In absence of these, and with non-cacheable mappings on x86 wmb and rmb should be a nop, and mb should be a locked instruction. It might make sense to add rte_dma_rmb/wmb/mb.

Quoted 41 lines

Pasting you commit code:
/*
* Per virtio_ring.h in Linux.

 *     For virtio_pci on SMP, we don't need to order with respect to MMIO 
 *     accesses through relaxed memory I/O windows, so smp_mb() et al are 
 *     sufficient.                                                        

*
* For using virtio to talk to real devices (eg. vDPA) we do need real
* barriers.
*/
static inline void
virtio_mb(uint8_t weak_barriers)

{                                                                         
        if (weak_barriers)                                                
                rte_smp_mb();                                             
        else                                                              
                rte_mb();                                                 
}                                                                         
Quoted 22 lines
Quoted 5 lines
Quoted 5 lines

[1]

https://emea01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpa

Quoted 1 line
Quoted 1 line

tches.dpdk.org%2Fpatch%2F49545%2F&amp;data=02%7C01%7Cshahafs%40 mellan

Quoted 1 line
Quoted 1 line

ox.com%7C01907f1b2e0e4002cb7508d67ac38a98%7Ca652971c7d2e4d9ba6a4 d1492

Quoted 1 line
Quoted 1 line

56f461b%7C0%7C0%7C636831377591864200&amp;sdata=TSpc%2Fzyq2aq0N3 %2Bh4o

Quoted 7 lines
Quoted 5 lines

ro4std8ut%2FQU6%2BOeMDeeaQdsM%3D&amp;reserved=0

Quoted 3 lines

--
MST

#14 lore raw

v3 cover Missing barriers and VIRTIO_F_ORDER_PLATFORM.

In reply to Ilya Maximets

Version 3:
* Added 2 patches with fixes for current virtio driver. Not directly connected with the new feature.

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

Ilya Maximets (3):
net/virtio: add missing barrier before reading the flags net/virtio: update memory ordering comment for vq notify net/virtio: add platform memory ordering feature support

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 +-
 drivers/net/virtio/virtio_pci.h    |  7 ++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++-----
 drivers/net/virtio/virtqueue.h     | 56 +++++++++++++++++++++++-------
 5 files changed, 63 insertions(+), 21 deletions(-)
Signature · 1 line
2.17.1
#15 lore raw

PATCH v3 1/3 net/virtio: add missing barrier before reading the flags

Reading the used->flags could be reordered with avail->idx update. vhost in kernel disables notifications for the time of packets receiving, like this:

1. disable notify
2. process packets
3. enable notify
4. has more packets ? goto 1

In case of reordering, virtio driver could read the flags on step 2 while notifications disabled and update avail->idx after the step 4, i.e. vhost will exit the loop on step 4 with notifications enabled, but virtio will not notify.

Fixes c1f86306a026 ("virtio: add new driver")
---
 drivers/net/virtio/virtqueue.h | 8 ++++++++
 1 file changed, 8 insertions(+)
drivers/net/virtio/virtqueue.h +8 -0
index d8ae5cdec..dffa03669 100644
@@ -415,6 +415,11 @@ vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
 static inline int
 virtqueue_kick_prepare(struct virtqueue *vq)
 {
+	/*
+	 * Ensure updated avail->idx is visible to vhost before reading
+	 * the used->flags.
+	 */
+	virtio_mb();
 	return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY);
 }
 
@@ -423,6 +428,9 @@ virtqueue_kick_prepare_packed(struct virtqueue *vq)
 {
 	uint16_t flags;
 
+	/*
+	 * Ensure updated data is visible to vhost before reading the flags.
+	 */
 	virtio_mb();
 	flags = vq->ring_packed.device_event->desc_event_flags;
 
Signature · 1 line
2.17.1
Reviewed-by on v3 1/3
#16 lore raw

Re: [PATCH v3 1/3] net/virtio: add missing barrier before reading the flags

Quoted 14 lines · Ilya Maximets

Reading the used->flags could be reordered with avail->idx update. vhost in kernel disables notifications for the time of packets receiving, like this:

1. disable notify
2. process packets
3. enable notify
4. has more packets ? goto 1

In case of reordering, virtio driver could read the flags on step 2 while notifications disabled and update avail->idx after the step 4, i.e. vhost will exit the loop on step 4 with notifications enabled, but virtio will not notify.

Quoted 9 lines
Fixes c1f86306a026 ("virtio: add new driver")
---
  drivers/net/virtio/virtqueue.h | 8 ++++++++
  1 file changed, 8 insertions(+)

Thanks,
Maxime

#17 lore raw

PATCH v3 2/3 net/virtio: update memory ordering comment for vq notify

In reply to Ilya Maximets

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

---
 drivers/net/virtio/virtqueue.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)
drivers/net/virtio/virtqueue.h +4 -5
index dffa03669..53aeac238 100644
@@ -437,14 +437,13 @@ virtqueue_kick_prepare_packed(struct virtqueue *vq)
 	return flags != RING_EVENT_FLAGS_DISABLE;
 }
 
+/*
+ * virtqueue_kick_prepare*() or the virtio_wmb() should be called
+ * before this function to be sure that all the data is visible to vhost.
+ */
 static inline void
 virtqueue_notify(struct virtqueue *vq)
 {
-	/*
-	 * Ensure updated avail->idx is visible to host.
-	 * For virtio on IA, the notificaiton is through io port operation
-	 * which is a serialization instruction itself.
-	 */
 	VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
 }
 
Signature · 1 line
2.17.1
#18 lore raw

Re: [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Ilya Maximets
Sent: Wednesday, January 9, 2019 10:50 PM
To: [email protected]; Maxime Coquelin <[email protected]>; Michael S . Tsirkin <[email protected]>; Xiao Wang <[email protected]>
Cc: Tiwei Bie <[email protected]>; Zhihong Wang <[email protected]>; [email protected]; Jason Wang <[email protected]>; [email protected]; [email protected]; Ilya Maximets <[email protected]>
Subject: [dpdk-dev] [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 2 lines

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

Agree, it should be architecture neutral.

Quoted 11 lines
---
 drivers/net/virtio/virtqueue.h | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)
drivers/net/virtio/virtqueue.h +0 -0
index dffa03669..53aeac238 100644

drivers/net/virtio/virtqueue.h virtqueue_kick_prepare_packed(struct virtqueue · line 444

Quoted 8 lines
drivers/net/virtio/virtqueue.h +4 -0
@@ -437,14 +437,13 @@ virtqueue_kick_prepare_packed(struct virtqueue
 *vq)
 	return flags != RING_EVENT_FLAGS_DISABLE;
 }
 
+/*
+ * virtqueue_kick_prepare*() or the virtio_wmb() should be called
+ * before this function to be sure that all the data is visible to vhost.
+ */

C11 _atomic APIs are preferred for new code, other than wmb or rmb, could you work on that?

Quoted 13 lines
drivers/net/virtio/virtqueue.h +0 -6
 static inline void
 virtqueue_notify(struct virtqueue *vq)
 {
-	/*
-	 * Ensure updated avail->idx is visible to host.
-	 * For virtio on IA, the notificaiton is through io port operation
-	 * which is a serialization instruction itself.
-	 */
 	VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
 }
 
--

2.17.1

#19 lore raw

Re: [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Hi Gavin,

Quoted 36 lines · Gavin Hu (Arm Technology China)
Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Ilya Maximets
Sent: Wednesday, January 9, 2019 10:50 PM
To: [email protected]; Maxime Coquelin <[email protected]>; Michael S . Tsirkin <[email protected]>; Xiao Wang <[email protected]>
Cc: Tiwei Bie <[email protected]>; Zhihong Wang <[email protected]>; [email protected]; Jason Wang <[email protected]>; [email protected]; [email protected]; Ilya Maximets <[email protected]>
Subject: [dpdk-dev] [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 2 lines

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

Agree, it should be architecture neutral.

Quoted 15 lines
---
  drivers/net/virtio/virtqueue.h | 9 ++++-----
  1 file changed, 4 insertions(+), 5 deletions(-)
drivers/net/virtio/virtqueue.h +0 -0
index dffa03669..53aeac238 100644
@@ -437,14 +437,13 @@ virtqueue_kick_prepare_packed(struct virtqueue
 *vq)
  	return flags != RING_EVENT_FLAGS_DISABLE;
  }
 
Quoted 6 lines

drivers/net/virtio/virtqueue.h virtqueue_kick_prepare_packed(struct virtqueue *vq) · line 443

Quoted 4 lines
drivers/net/virtio/virtqueue.h +4 -0
+/*
+ * virtqueue_kick_prepare*() or the virtio_wmb() should be called
+ * before this function to be sure that all the data is visible to vhost.
+ */

C11 _atomic APIs are preferred for new code, other than wmb or rmb, could you work on that?

Thanks for the review.
-rc2 deadline is today, so I may apply this series as is if no reply from Ilya today. If a change is agreed, it could be made on top for -rc3.

Regards,
Maxime

Quoted 14 lines
Quoted 13 lines
drivers/net/virtio/virtqueue.h +0 -6
  static inline void
  virtqueue_notify(struct virtqueue *vq)
  {
-	/*
-	 * Ensure updated avail->idx is visible to host.
-	 * For virtio on IA, the notificaiton is through io port operation
-	 * which is a serialization instruction itself.
-	 */
  	VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
  }
 
--

2.17.1

#20 lore raw

Re: [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 46 lines · Maxime Coquelin

Hi Gavin,

Quoted 36 lines · Gavin Hu (Arm Technology China)
Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Ilya Maximets
Sent: Wednesday, January 9, 2019 10:50 PM
To: [email protected]; Maxime Coquelin <[email protected]>; Michael S . Tsirkin <[email protected]>; Xiao Wang <[email protected]>
Cc: Tiwei Bie <[email protected]>; Zhihong Wang <[email protected]>; [email protected]; Jason Wang <[email protected]>; [email protected]; [email protected]; Ilya Maximets <[email protected]>
Subject: [dpdk-dev] [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 2 lines

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

Agree, it should be architecture neutral.

Quoted 15 lines

---
  drivers/net/virtio/virtqueue.h | 9 ++++-----

  1 file changed, 4 insertions(+), 5 deletions(-)
drivers/net/virtio/virtqueue.h +0 -0
index dffa03669..53aeac238 100644
@@ -437,14 +437,13 @@ virtqueue_kick_prepare_packed(struct virtqueue
 *vq)
       return flags != RING_EVENT_FLAGS_DISABLE;
   }
 
Quoted 6 lines

drivers/net/virtio/virtqueue.h virtqueue_kick_prepare_packed(struct virtqueue *vq) · line 443

Quoted 4 lines
drivers/net/virtio/virtqueue.h +4 -0
+/*
+ * virtqueue_kick_prepare*() or the virtio_wmb() should be called
+ * before this function to be sure that all the data is visible to vhost.
+ */

C11 _atomic APIs are preferred for new code, other than wmb or rmb,  could you work on that?

Quoted 4 lines

Thanks for the review.
-rc2 deadline is today, so I may apply this series as is if no reply from Ilya today. If a change is agreed, it could be made on top for -rc3.

IMHO, If we'll decide to move to C11 atomics, we'll need to rewrite all the significant memory accesses in virtio driver at once. We can't do this partially. This will require significant amount of work to understand how to do that and will require a lot of testing. So, it's definitely not for current release. Also, there are possible performance concerns about such solution.

But I have even more significant concern: C11 atomics are designed for inter-thread synchronization on multi-core systems. But in case of vDPA we have real hardware and I'm not sure if we can use C11 atomics for cross-domain synchronizations. Do you know if some of the memory ordering types in C11 provide outer domain sync on ARMv8, for example ?

Quoted 19 lines

Regards,
Maxime

Quoted 14 lines
Quoted 13 lines

  static inline void
  virtqueue_notify(struct virtqueue *vq)
  {
-    /*
-     * Ensure updated avail->idx is visible to host.
-     * For virtio on IA, the notificaiton is through io port operation -     * which is a serialization instruction itself.
-     */
      VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
  }

Signature · 1 line
2.17.1
#21 lore raw

Re: [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 59 lines · Ilya Maximets
Quoted 46 lines · Maxime Coquelin

Hi Gavin,

Quoted 36 lines · Gavin Hu (Arm Technology China)
Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Ilya Maximets
Sent: Wednesday, January 9, 2019 10:50 PM
To: [email protected]; Maxime Coquelin <[email protected]>; Michael S . Tsirkin <[email protected]>; Xiao Wang <[email protected]>
Cc: Tiwei Bie <[email protected]>; Zhihong Wang <[email protected]>; [email protected]; Jason Wang <[email protected]>; [email protected]; [email protected]; Ilya Maximets <[email protected]>
Subject: [dpdk-dev] [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

Quoted 2 lines

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

Agree, it should be architecture neutral.

Quoted 15 lines

---
  drivers/net/virtio/virtqueue.h | 9 ++++-----

  1 file changed, 4 insertions(+), 5 deletions(-)
drivers/net/virtio/virtqueue.h +0 -0
index dffa03669..53aeac238 100644
@@ -437,14 +437,13 @@ virtqueue_kick_prepare_packed(struct virtqueue
 *vq)
       return flags != RING_EVENT_FLAGS_DISABLE;
   }
 
Quoted 6 lines

drivers/net/virtio/virtqueue.h virtqueue_kick_prepare_packed(struct virtqueue *vq) · line 443

Quoted 4 lines
drivers/net/virtio/virtqueue.h +4 -0
+/*
+ * virtqueue_kick_prepare*() or the virtio_wmb() should be called
+ * before this function to be sure that all the data is visible to vhost.
+ */

C11 _atomic APIs are preferred for new code, other than wmb or rmb,  could you work on that?

Quoted 4 lines

Thanks for the review.
-rc2 deadline is today, so I may apply this series as is if no reply from Ilya today. If a change is agreed, it could be made on top for -rc3.

IMHO, If we'll decide to move to C11 atomics, we'll need to rewrite all the significant memory accesses in virtio driver at once. We can't do this partially. This will require significant amount of work to understand how to do that and will require a lot of testing. So, it's definitely not for current release. Also, there are possible performance concerns about such solution.

Quoted 5 lines

But I have even more significant concern: C11 atomics are designed for inter-thread synchronization on multi-core systems. But in case of vDPA we have real hardware and I'm not sure if we can use C11 atomics for cross-domain synchronizations. Do you know if some of the memory ordering types in C11 provide outer domain sync on ARMv8, for example ?

I would add to that - compiler support might not work well in all versions which can be used to build dpdk.
E.g. this article https://lwn.net/Articles/691128/
says that "there will be some seriously suboptimal code production before gcc-7.1".

HTH

Quoted 19 lines
Quoted 19 lines

Regards,
Maxime

Quoted 14 lines
Quoted 13 lines

  static inline void
  virtqueue_notify(struct virtqueue *vq)
  {
-    /*
-     * Ensure updated avail->idx is visible to host.
-     * For virtio on IA, the notificaiton is through io port operation -     * which is a serialization instruction itself.
-     */
      VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
  }

Signature · 1 line
2.17.1
Reviewed-by on v3 2/3
#22 lore raw

Re: [PATCH v3 2/3] net/virtio: update memory ordering comment for vq notify

In reply to Ilya Maximets
Quoted 7 lines · Ilya Maximets

We're not using io ports in case of modern device even on IA. Also, this comment useless for other architectures.

---
  drivers/net/virtio/virtqueue.h | 9 ++++-----
  1 file changed, 4 insertions(+), 5 deletions(-)

Thanks,
Maxime

#23 lore raw

PATCH v3 3/3 net/virtio: add platform memory ordering feature support

In reply to Ilya Maximets

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

---
 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 ++-
 drivers/net/virtio/virtio_pci.h    |  7 +++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++++------
 drivers/net/virtio/virtqueue.h     | 41 ++++++++++++++++++++++++------
 5 files changed, 52 insertions(+), 17 deletions(-)
drivers/net/virtio/virtio_ethdev.c +2 -0
index 446c338fc..6d461180c 100644
@@ -1613,6 +1613,8 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
 	if (virtio_negotiate_features(hw, req_features) < 0)
 		return -1;
 
+	hw->weak_barriers = !vtpci_with_feature(hw, VIRTIO_F_ORDER_PLATFORM);
+
 	if (!hw->virtio_user_dev) {
 		pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
drivers/net/virtio/virtio_ethdev.h +2 -1
index f8d8a56ab..b8aab7da4 100644
@@ -35,7 +35,8 @@
 	 1ULL << VIRTIO_F_VERSION_1       |	\
 	 1ULL << VIRTIO_F_IN_ORDER        |	\
 	 1ULL << VIRTIO_F_RING_PACKED	  |	\
-	 1ULL << VIRTIO_F_IOMMU_PLATFORM)
+	 1ULL << VIRTIO_F_IOMMU_PLATFORM  |	\
+	 1ULL << VIRTIO_F_ORDER_PLATFORM)
 
 #define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES	\
 	(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |	\
drivers/net/virtio/virtio_pci.h +7 -0
index b22b62dad..38a0261da 100644
@@ -129,6 +129,12 @@ struct virtnet_ctl;
  */
 #define VIRTIO_F_IN_ORDER 35
 
+/*
+ * This feature indicates that memory accesses by the driver and the device
+ * are ordered in a way described by the platform.
+ */
+#define VIRTIO_F_ORDER_PLATFORM 36
+
 /* The Guest publishes the used index for which it expects an interrupt
  * at the end of the avail ring. Host should ignore the avail->flags field. */
 /* The Host publishes the avail index for which it expects a kick
@@ -241,6 +247,7 @@ struct virtio_hw {
 	uint8_t     use_simple_rx;
 	uint8_t     use_inorder_rx;
 	uint8_t     use_inorder_tx;
+	uint8_t     weak_barriers;
 	bool        has_tx_offload;
 	bool        has_rx_offload;
 	uint16_t    port_id;
drivers/net/virtio/virtio_rxtx.c +8 -8
index 2309b71d6..ebb86ef70 100644
@@ -1152,7 +1152,7 @@ virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
 	if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
@@ -1361,7 +1361,7 @@ virtio_recv_pkts_inorder(void *rx_queue,
 	nb_used = RTE_MIN(nb_used, nb_pkts);
 	nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1549,7 +1549,7 @@ virtio_recv_mergeable_pkts(void *rx_queue,
 
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 
 	PMD_RX_LOG(DEBUG, "used:%d", nb_used);
 
@@ -1940,7 +1940,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_pkts);
 			virtio_xmit_cleanup_packed(vq, need);
 			need = slots - vq->vq_free_cnt;
@@ -1988,7 +1988,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup(vq, nb_used);
 
@@ -2030,7 +2030,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
 		/* Positive value indicates it need free vring descriptors */
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup(vq, need);
@@ -2086,7 +2086,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
 	nb_used = VIRTQUEUE_NUSED(vq);
 
-	virtio_rmb();
+	virtio_rmb(hw->weak_barriers);
 	if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
 		virtio_xmit_cleanup_inorder(vq, nb_used);
 
@@ -2134,7 +2134,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 		need = slots - vq->vq_free_cnt;
 		if (unlikely(need > 0)) {
 			nb_used = VIRTQUEUE_NUSED(vq);
-			virtio_rmb();
+			virtio_rmb(hw->weak_barriers);
 			need = RTE_MIN(need, (int)nb_used);
 
 			virtio_xmit_cleanup_inorder(vq, need);
drivers/net/virtio/virtqueue.h +33 -8

76 lines not on this page · raw message

Signature · 1 line
2.17.1
Reviewed-by on v3 3/3
#24 lore raw
Quoted 7 lines · Ilya Maximets

VIRTIO_F_ORDER_PLATFORM is required to use proper memory barriers in case of HW vhost implementations like vDPA.

DMA barriers (rte_cio_*) are sufficent for that purpose.

Previously known as VIRTIO_F_IO_BARRIER.

Quoted 8 lines
---
  drivers/net/virtio/virtio_ethdev.c |  2 ++
  drivers/net/virtio/virtio_ethdev.h |  3 ++-
  drivers/net/virtio/virtio_pci.h    |  7 +++++
  drivers/net/virtio/virtio_rxtx.c   | 16 ++++++------
  drivers/net/virtio/virtqueue.h     | 41 ++++++++++++++++++++++++------
  5 files changed, 52 insertions(+), 17 deletions(-)

Thanks,
Maxime

Acked-by on the v3 series
#25 lore raw

Re: [PATCH v3 0/3] Missing barriers and VIRTIO_F_ORDER_PLATFORM.

In reply to Ilya Maximets
Quoted 3 lines · Ilya Maximets

Version 3:
* Added 2 patches with fixes for current virtio driver. Not directly connected with the new feature.

New version shouldn't be reply-to old one really :). But the patches are good I think.

Quoted 25 lines

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

Ilya Maximets (3):
net/virtio: add missing barrier before reading the flags net/virtio: update memory ordering comment for vq notify net/virtio: add platform memory ordering feature support

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 +-
 drivers/net/virtio/virtio_pci.h    |  7 ++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++-----
 drivers/net/virtio/virtqueue.h     | 56 +++++++++++++++++++++++-------
 5 files changed, 63 insertions(+), 21 deletions(-)
Signature · 1 line
2.17.1
#26 lore raw

Re: [PATCH v3 0/3] Missing barriers and VIRTIO_F_ORDER_PLATFORM.

Quoted 6 lines · Michael S. Tsirkin
Quoted 3 lines · Ilya Maximets

Version 3:
* Added 2 patches with fixes for current virtio driver. Not directly connected with the new feature.

New version shouldn't be reply-to old one really :).

It's kind of a common practice and a recommended thing for the dpdk mail-list. At least it was.
I'm not doing that on other lists. =)

Quoted 3 lines

But the patches are good I think.

Thanks.

Quoted 30 lines
Quoted 28 lines

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

Ilya Maximets (3):
net/virtio: add missing barrier before reading the flags net/virtio: update memory ordering comment for vq notify net/virtio: add platform memory ordering feature support

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 +-
 drivers/net/virtio/virtio_pci.h    |  7 ++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++-----
 drivers/net/virtio/virtqueue.h     | 56 +++++++++++++++++++++++-------
 5 files changed, 63 insertions(+), 21 deletions(-)
Signature · 1 line
2.17.1
#27 lore raw

Re: [PATCH v3 0/3] Missing barriers and VIRTIO_F_ORDER_PLATFORM.

Quoted 11 lines · Ilya Maximets
Quoted 6 lines · Michael S. Tsirkin
Quoted 3 lines · Ilya Maximets

Version 3:
* Added 2 patches with fixes for current virtio driver. Not directly connected with the new feature.

New version shouldn't be reply-to old one really :).

It's kind of a common practice and a recommended thing for the dpdk mail-list. At least it was.
I'm not doing that on other lists. =)

Yes this is something recommended to do. It helps a lot to be able to see all versions and comments in same email thread.

Quoted 37 lines
Quoted 4 lines

But the patches are good I think.

Thanks.

Quoted 30 lines
Quoted 28 lines

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

Ilya Maximets (3):
net/virtio: add missing barrier before reading the flags net/virtio: update memory ordering comment for vq notify net/virtio: add platform memory ordering feature support

 drivers/net/virtio/virtio_ethdev.c |  2 ++
 drivers/net/virtio/virtio_ethdev.h |  3 +-
 drivers/net/virtio/virtio_pci.h    |  7 ++++
 drivers/net/virtio/virtio_rxtx.c   | 16 ++++-----
 drivers/net/virtio/virtqueue.h     | 56 +++++++++++++++++++++++-------
 5 files changed, 63 insertions(+), 21 deletions(-)
Signature · 1 line
2.17.1
#28 lore raw

Re: [PATCH v3 0/3] Missing barriers and VIRTIO_F_ORDER_PLATFORM.

In reply to Michael S. Tsirkin
Quoted 37 lines · Michael S. Tsirkin
Quoted 3 lines · Ilya Maximets

Version 3:
* Added 2 patches with fixes for current virtio driver. Not directly connected with the new feature.

New version shouldn't be reply-to old one really :). But the patches are good I think.

Quoted 25 lines

Version 2:
* rebased on current master (packed rings).

RFC --> Version 1:
* Dropped vendor-specific hack to determine if we need real barriers.
* Added VIRTIO_F_ORDER_PLATFORM feature definition and checking.

Note: Patch to change the name of the feature from VIRTIO_F_IO_BARRIER
to VIRTIO_F_ORDER_PLATFORM is not merged yet:
https://www.mail-archive.com/[email protected]/msg04114.html

Ilya Maximets (3):
net/virtio: add missing barrier before reading the flags net/virtio: update memory ordering comment for vq notify net/virtio: add platform memory ordering feature support

  drivers/net/virtio/virtio_ethdev.c |  2 ++
  drivers/net/virtio/virtio_ethdev.h |  3 +-
  drivers/net/virtio/virtio_pci.h    |  7 ++++
  drivers/net/virtio/virtio_rxtx.c   | 16 ++++-----
  drivers/net/virtio/virtqueue.h     | 56 +++++++++++++++++++++++-------
  5 files changed, 63 insertions(+), 21 deletions(-)
Signature · 1 line
2.17.1

Applied to dpdk-next-virtio/master.

Thanks,
Maxime