Skip to contentSkip to conversation

net/memif: optimized with one-way barrier

PATCH Applied Phil Yang 4 participants 9 messages Aug 26, 2019 – Oct 14, 2019 Network drivers

lore
Share & tools
lore

Phil Yang

author
#0 lore raw

PATCH 1/1 net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

---
 drivers/net/memif/memif.h         |  4 +--
 drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++------------------
 2 files changed, 31 insertions(+), 26 deletions(-)
drivers/net/memif/memif.h +2 -2
index 3948b1f..a4d88c0 100644
@@ -169,9 +169,9 @@ typedef struct {
 	uint32_t cookie;			/**< MEMIF_COOKIE */
 	uint16_t flags;				/**< flags */
 #define MEMIF_RING_FLAG_MASK_INT 1		/**< disable interrupt mode */
-	volatile uint16_t head;			/**< pointer to ring buffer head */
+	uint16_t head;			/**< pointer to ring buffer head */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline1);
-	volatile uint16_t tail;			/**< pointer to ring buffer tail */
+	uint16_t tail;			/**< pointer to ring buffer tail */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline2);
 	memif_desc_t desc[0];			/**< buffer descriptors */
 } memif_ring_t;
drivers/net/memif/rte_eth_memif.c +29 -24

117 lines not on this page · raw message

Signature · 1 line
2.7.4
#1 lore raw

+ Gavin

Quoted 198 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd <[email protected]>
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

---
 drivers/net/memif/memif.h         |  4 +--
 drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++--------
----------
 2 files changed, 31 insertions(+), 26 deletions(-)
drivers/net/memif/memif.h +0 -0
index 3948b1f..a4d88c0 100644
@@ -169,9 +169,9 @@ typedef struct {
 	uint32_t cookie;			/**< MEMIF_COOKIE */
 	uint16_t flags;				/**< flags */
 #define MEMIF_RING_FLAG_MASK_INT 1		/**< disable interrupt

mode */

drivers/net/memif/memif.h +0 -1
-	volatile uint16_t head;			/**< pointer to ring buffer

head */

drivers/net/memif/memif.h +1 -1
+	uint16_t head;			/**< pointer to ring buffer head */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline1);
-	volatile uint16_t tail;			/**< pointer to ring buffer

tail */

drivers/net/memif/memif.h +1 -0
+	uint16_t tail;			/**< pointer to ring buffer tail */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline2);
 	memif_desc_t desc[0];			/**< buffer descriptors */
 } memif_ring_t;
drivers/net/memif/rte_eth_memif.c +6 -2
index a59f809..b1c871e 100644
@@ -275,8 +275,14 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs,
 uint16_t nb_pkts)
 	ring_size = 1 << mq->log2_ring_size;
 	mask = ring_size - 1;
 
-	cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq-
 >last_tail;
-	last_slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
+	if (type == MEMIF_RING_S2M) {
+		cur_slot = mq->last_head;
+		last_slot = __atomic_load_n(&ring->head,
 __ATOMIC_ACQUIRE);
+	} else {
+		cur_slot = mq->last_tail;
+		last_slot = __atomic_load_n(&ring->tail,

__ATOMIC_ACQUIRE);

drivers/net/memif/rte_eth_memif.c +3 -2
+	}
+
 	if (cur_slot == last_slot)
 		goto refill;
 	n_slots = last_slot - cur_slot;
@@ -344,8 +350,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs,
 uint16_t nb_pkts)
 
 no_free_bufs:
 	if (type == MEMIF_RING_S2M) {
-		rte_mb();
-		ring->tail = cur_slot;
+		__atomic_store_n(&ring->tail, cur_slot,
 __ATOMIC_RELEASE);
 		mq->last_head = cur_slot;
drivers/net/memif/rte_eth_memif.c +1 -1
 	} else {
 		mq->last_tail = cur_slot;
@@ -353,7 +358,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs,
 uint16_t nb_pkts)
 
 refill:
 	if (type == MEMIF_RING_M2S) {
-		head = ring->head;
+		head = __atomic_load_n(&ring->head,
 __ATOMIC_ACQUIRE);
 		n_slots = ring_size - head + mq->last_tail;
drivers/net/memif/rte_eth_memif.c +1 -2
 		while (n_slots--) {
@@ -361,8 +366,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs,
 uint16_t nb_pkts)
 			d0 = &ring->desc[s0];
 			d0->length = pmd->run.pkt_buffer_size;
 		}
-		rte_mb();
-		ring->head = head;
+		__atomic_store_n(&ring->head, head, __ATOMIC_RELEASE);
 	}
 
drivers/net/memif/rte_eth_memif.c +8 -6
 	mq->n_pkts += n_rx_pkts;
@@ -398,14 +402,16 @@ eth_memif_tx(void *queue, struct rte_mbuf
 **bufs, uint16_t nb_pkts)
 	ring_size = 1 << mq->log2_ring_size;
 	mask = ring_size - 1;
 
-	n_free = ring->tail - mq->last_tail;
+	n_free = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE) - mq-
 >last_tail;
 	mq->last_tail += n_free;
-	slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail;
 
-	if (type == MEMIF_RING_S2M)
-		n_free = ring_size - ring->head + mq->last_tail;
-	else
-		n_free = ring->head - ring->tail;
+	if (type == MEMIF_RING_S2M) {
+		slot = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE);
+		n_free = ring_size - slot + mq->last_tail;
+	} else {
+		slot = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE);
+		n_free = __atomic_load_n(&ring->head,
 __ATOMIC_ACQUIRE) - slot;
+	}
drivers/net/memif/rte_eth_memif.c +2 -3
 	while (n_tx_pkts < nb_pkts && n_free) {
 		mbuf_head = *bufs++;
@@ -464,11 +470,10 @@ eth_memif_tx(void *queue, struct rte_mbuf
 **bufs, uint16_t nb_pkts)
 	}
 
 no_free_slots:
-	rte_mb();
 	if (type == MEMIF_RING_S2M)
-		ring->head = slot;
+		__atomic_store_n(&ring->head, slot, __ATOMIC_RELEASE);
 	else
-		ring->tail = slot;
+		__atomic_store_n(&ring->tail, slot, __ATOMIC_RELEASE);
 
 	if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) {
drivers/net/memif/rte_eth_memif.c +2 -2
 		a = 1;
@@ -611,8 +616,8 @@ memif_init_rings(struct rte_eth_dev *dev)
 
 	for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 		ring = memif_get_ring(pmd, proc_private,
 MEMIF_RING_S2M, i);
-		ring->head = 0;
-		ring->tail = 0;
+		__atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
+		__atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
 		ring->cookie = MEMIF_COOKIE;
 		ring->flags = 0;
drivers/net/memif/rte_eth_memif.c +2 -2
 		for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) {
@@ -627,8 +632,8 @@ memif_init_rings(struct rte_eth_dev *dev)
 
 	for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 		ring = memif_get_ring(pmd, proc_private,
 MEMIF_RING_M2S, i);
-		ring->head = 0;
-		ring->tail = 0;
+		__atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
+		__atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
 		ring->cookie = MEMIF_COOKIE;
 		ring->flags = 0;
drivers/net/memif/rte_eth_memif.c +2 -2
 		for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) {
@@ -734,8 +739,8 @@ memif_connect(struct rte_eth_dev *dev)
 				MIF_LOG(ERR, "Wrong ring");
 				return -1;
 			}
-			ring->head = 0;
-			ring->tail = 0;
+			__atomic_store_n(&ring->head, 0,
 __ATOMIC_RELAXED);
+			__atomic_store_n(&ring->tail, 0,
 __ATOMIC_RELAXED);
 			mq->last_head = 0;
drivers/net/memif/rte_eth_memif.c +2 -2
 			mq->last_tail = 0;
 			/* enable polling mode */
@@ -750,8 +755,8 @@ memif_connect(struct rte_eth_dev *dev)
 				MIF_LOG(ERR, "Wrong ring");
 				return -1;
 			}
-			ring->head = 0;
-			ring->tail = 0;
+			__atomic_store_n(&ring->head, 0,
 __ATOMIC_RELAXED);
+			__atomic_store_n(&ring->tail, 0,
 __ATOMIC_RELAXED);
 			mq->last_head = 0;
drivers/net/memif/rte_eth_memif.c +0 -1
 			mq->last_tail = 0;
 			/* enable polling mode */
--

2.7.4

#2 lore raw
Quoted 7 lines
Quoted 6 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd

<[email protected]>

Quoted 15 lines
Quoted 7 lines

Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered between

CPUs on IA.

Quoted 7 lines

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

The patch is looking good, but 'MEMIF_VERSION_MAJOR' in memif.h needs to be set to 3 as ring pointers are no longer volatile.

#3 lore raw
Quoted 32 lines

-----Original Message-----
From: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>
Sent: Tuesday, October 8, 2019 7:05 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Damjan Marion (damarion) <[email protected]>; nd <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Quoted 6 lines
Quoted 6 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd

Quoted 8 lines

<[email protected]>

Quoted 7 lines

Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered

between

Quoted 4 lines

CPUs on IA.

Quoted 3 lines

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd

throughput

Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

The patch is looking good, but 'MEMIF_VERSION_MAJOR' in memif.h needs to
be set to 3 as ring pointers are no longer volatile.

Updated in v2.
Thanks for your comments.

Thanks,
Phil

#4 lore raw
Quoted 28 lines
Quoted 28 lines
Quoted 17 lines
Quoted 8 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd

<[email protected]>

Quoted 8 lines

Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered

between

Quoted 4 lines

CPUs on IA.

Quoted 3 lines

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd

throughput

Quoted 4 lines
Quoted 4 lines

boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

Quoted 8 lines
Quoted 2 lines

The patch is looking good, but 'MEMIF_VERSION_MAJOR' in memif.h needs to be set to 3 as ring pointers are no longer volatile.

Updated in v2.
Thanks for your comments.

Thanks,
Phil

I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.

Sorry for the inconvenience.

Applied
#5 lore raw
Quoted 34 lines · On 10/9/2019 12:17 PM, Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at…
Quoted 34 lines
Quoted 22 lines
Quoted 17 lines
Quoted 8 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd

<[email protected]>

Quoted 8 lines

Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered

between

Quoted 4 lines

CPUs on IA.

Quoted 3 lines

Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd

Quoted 8 lines

throughput

Quoted 4 lines
Quoted 4 lines

boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

The patch is looking good, but 'MEMIF_VERSION_MAJOR' in memif.h needs to be set to 3 as ring pointers are no longer volatile.

Updated in v2.
Thanks for your comments.

Quoted 9 lines
Quoted 2 lines

Thanks,
Phil

I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.

Sorry for the inconvenience.

Applied to dpdk-next-net/master, thanks.

author
#6 lore raw

PATCH v2 1/1 net/memif: optimized with one-way barrier

In reply to Phil Yang

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.

Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)

v1:
Initial version.

 drivers/net/memif/memif.h         |  6 ++---
 drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++------------------
 2 files changed, 32 insertions(+), 27 deletions(-)
drivers/net/memif/memif.h +3 -3
index 3948b1f..8000aa1 100644
@@ -6,7 +6,7 @@
 #define _MEMIF_H_
 
 #define MEMIF_COOKIE		0x3E31F20
-#define MEMIF_VERSION_MAJOR	2
+#define MEMIF_VERSION_MAJOR	3
 #define MEMIF_VERSION_MINOR	0
 #define MEMIF_VERSION		((MEMIF_VERSION_MAJOR << 8) | MEMIF_VERSION_MINOR)
 #define MEMIF_NAME_SZ		32
@@ -169,9 +169,9 @@ typedef struct {
 	uint32_t cookie;			/**< MEMIF_COOKIE */
 	uint16_t flags;				/**< flags */
 #define MEMIF_RING_FLAG_MASK_INT 1		/**< disable interrupt mode */
-	volatile uint16_t head;			/**< pointer to ring buffer head */
+	uint16_t head;			/**< pointer to ring buffer head */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline1);
-	volatile uint16_t tail;			/**< pointer to ring buffer tail */
+	uint16_t tail;			/**< pointer to ring buffer tail */
 	MEMIF_CACHELINE_ALIGN_MARK(cacheline2);
 	memif_desc_t desc[0];			/**< buffer descriptors */
 } memif_ring_t;
drivers/net/memif/rte_eth_memif.c +29 -24

117 lines not on this page · raw message

Signature · 1 line
2.7.4
#7 lore raw
Quoted 21 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Wednesday, October 9, 2019 4:05 AM
To: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Damjan Marion (damarion) <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: [PATCH v2] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.

Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

Quoted 8 lines

---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)

v1:
Initial version.

I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.

Sorry for the inconvenience.

#8 lore raw
Quoted 28 lines · On Wed, Oct 9, 2019 at 1:15 PM Jakub Grajciar -X (jgrajcia - PANTHEON…
Quoted 28 lines

-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Wednesday, October 9, 2019 4:05 AM
To: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Damjan Marion (damarion) <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: [PATCH v2] net/memif: optimized with one-way barrier

Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.

Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.

---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)

Quoted 6 lines
Quoted 2 lines

v1:
Initial version.

I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.

Sorry for the inconvenience.

Ok, marking v2 as rejected then.

Signature · 1 line
David Marchand