net/memif: optimized with one-way barrier
PATCH 1/1 net/memif: optimized with one-way barrier
Using 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.
Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
--- drivers/net/memif/memif.h | 4 +-- drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 26 deletions(-)
drivers/net/memif/memif.h +2 -2
@@ -169,9 +169,9 @@ typedef struct {uint32_t cookie;/**< MEMIF_COOKIE */ uint16_t flags;/**< flags */ #define MEMIF_RING_FLAG_MASK_INT 1/**< disable interrupt mode */ -volatile uint16_t head;/**< pointer to ring buffer head */ +uint16_t head;/**< pointer to ring buffer head */ MEMIF_CACHELINE_ALIGN_MARK (cacheline1); -volatile uint16_t tail;/**< pointer to ring buffer tail */ +uint16_t tail;/**< pointer to ring buffer tail */ MEMIF_CACHELINE_ALIGN_MARK (cacheline2);memif_desc_t desc[0 ];/**< buffer descriptors */ } memif_ring_t;
drivers/net/memif/rte_eth_memif.c +29 -24
117 lines not on this page · raw message
Signature · 1 line
2.7.4
+ Gavin
Quoted 198 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd <[email protected]>
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrierUsing '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>--- drivers/net/memif/memif.h | 4 +-- drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++-------- ---------- 2 files changed, 31 insertions(+), 26 deletions(-)drivers/net/memif/memif.h +0
-0@@ -169,9 +169,9 @@ typedef struct {uint32_t cookie;/**< MEMIF_COOKIE */ uint16_t flags;/**< flags */ #define MEMIF_RING_FLAG_MASK_INT 1/**< disable interrupt mode */
drivers/net/memif/memif.h +0
-1-volatile uint16_t head;/**< pointer to ring buffer head */
drivers/net/memif/memif.h +1
-1+uint16_t head;/**< pointer to ring buffer head */ MEMIF_CACHELINE_ALIGN_MARK (cacheline1); -volatile uint16_t tail;/**< pointer to ring buffer tail */
drivers/net/memif/memif.h +1
-0+uint16_t tail;/**< pointer to ring buffer tail */ MEMIF_CACHELINE_ALIGN_MARK (cacheline2);memif_desc_t desc[0 ];/**< buffer descriptors */ } memif_ring_t;drivers/net/memif/rte_eth_memif.c +6
-2@@ -275,8 +275,14 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) ring_size =1 << mq->log2_ring_size; mask = ring_size -1 ; - cur_slot = (type ==MEMIF_RING_S2M ) ? mq->last_head : mq- >last_tail; - last_slot = (type ==MEMIF_RING_S2M ) ? ring->head : ring->tail; +if (type ==MEMIF_RING_S2M ) { + cur_slot = mq->last_head; + last_slot =__atomic_load_n (&ring->head, __ATOMIC_ACQUIRE); + }else { + cur_slot = mq->last_tail; + last_slot = __atomic_load_n(&ring->tail,
__ATOMIC_ACQUIRE);drivers/net/memif/rte_eth_memif.c +3
-2+ } +if (cur_slot == last_slot) goto refill; n_slots = last_slot - cur_slot;@@ -344,8 +350,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) no_free_bufs:if (type ==MEMIF_RING_S2M ) { -rte_mb (); - ring->tail = cur_slot; +__atomic_store_n (&ring->tail, cur_slot, __ATOMIC_RELEASE); mq->last_head = cur_slot;drivers/net/memif/rte_eth_memif.c +1
-1}else { mq->last_tail = cur_slot;@@ -353,7 +358,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) refill:if (type ==MEMIF_RING_M2S ) { - head = ring->head; + head =__atomic_load_n (&ring->head, __ATOMIC_ACQUIRE); n_slots = ring_size - head + mq->last_tail;drivers/net/memif/rte_eth_memif.c +1
-2while (n_slots--) {@@ -361,8 +366,7 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) d0 = &ring->desc[s0]; d0->length = pmd->run.pkt_buffer_size; } -rte_mb (); - ring->head = head; +__atomic_store_n (&ring->head, head, __ATOMIC_RELEASE); }drivers/net/memif/rte_eth_memif.c +8
-6mq->n_pkts += n_rx_pkts;@@ -398,14 +402,16 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) ring_size =1 << mq->log2_ring_size; mask = ring_size -1 ; - n_free = ring->tail - mq->last_tail; + n_free =__atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE) - mq- >last_tail; mq->last_tail += n_free; - slot = (type ==MEMIF_RING_S2M ) ? ring->head : ring->tail; -if (type ==MEMIF_RING_S2M ) - n_free = ring_size - ring->head + mq->last_tail; -else - n_free = ring->head - ring->tail; +if (type ==MEMIF_RING_S2M ) { + slot =__atomic_load_n (&ring->head, __ATOMIC_ACQUIRE); + n_free = ring_size - slot + mq->last_tail; + }else { + slot =__atomic_load_n (&ring->tail, __ATOMIC_ACQUIRE); + n_free =__atomic_load_n (&ring->head, __ATOMIC_ACQUIRE) - slot; + }drivers/net/memif/rte_eth_memif.c +2
-3while (n_tx_pkts < nb_pkts && n_free) { mbuf_head = *bufs++;@@ -464,11 +470,10 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } no_free_slots: -rte_mb ();if (type ==MEMIF_RING_S2M ) - ring->head = slot; +__atomic_store_n (&ring->head, slot, __ATOMIC_RELEASE);else - ring->tail = slot; +__atomic_store_n (&ring->tail, slot, __ATOMIC_RELEASE);if ((ring->flags &MEMIF_RING_FLAG_MASK_INT ) ==0 ) {drivers/net/memif/rte_eth_memif.c +2
-2a =1 ;@@ -611,8 +616,8 @@ memif_init_rings(struct rte_eth_dev *dev)for (i =0 ; i < pmd->run.num_s2m_rings; i++) { ring =memif_get_ring (pmd, proc_private,MEMIF_RING_S2M , i); - ring->head =0 ; - ring->tail =0 ; +__atomic_store_n (&ring->head,0 , __ATOMIC_RELAXED); +__atomic_store_n (&ring->tail,0 , __ATOMIC_RELAXED); ring->cookie =MEMIF_COOKIE ; ring->flags =0 ;drivers/net/memif/rte_eth_memif.c +2
-2for (j =0 ; j < (1 << pmd->run.log2_ring_size); j++) {@@ -627,8 +632,8 @@ memif_init_rings(struct rte_eth_dev *dev)for (i =0 ; i < pmd->run.num_m2s_rings; i++) { ring =memif_get_ring (pmd, proc_private,MEMIF_RING_M2S , i); - ring->head =0 ; - ring->tail =0 ; +__atomic_store_n (&ring->head,0 , __ATOMIC_RELAXED); +__atomic_store_n (&ring->tail,0 , __ATOMIC_RELAXED); ring->cookie =MEMIF_COOKIE ; ring->flags =0 ;drivers/net/memif/rte_eth_memif.c +2
-2for (j =0 ; j < (1 << pmd->run.log2_ring_size); j++) {@@ -734,8 +739,8 @@ memif_connect(struct rte_eth_dev *dev)MIF_LOG (ERR ,"Wrong ring" );return -1 ; } - ring->head =0 ; - ring->tail =0 ; +__atomic_store_n (&ring->head,0 , __ATOMIC_RELAXED); +__atomic_store_n (&ring->tail,0 , __ATOMIC_RELAXED); mq->last_head =0 ;drivers/net/memif/rte_eth_memif.c +2
-2mq->last_tail =0 ;/* enable polling mode */ @@ -750,8 +755,8 @@ memif_connect(struct rte_eth_dev *dev)MIF_LOG (ERR ,"Wrong ring" );return -1 ; } - ring->head =0 ; - ring->tail =0 ; +__atomic_store_n (&ring->head,0 , __ATOMIC_RELAXED); +__atomic_store_n (&ring->tail,0 , __ATOMIC_RELAXED); mq->last_head =0 ;drivers/net/memif/rte_eth_memif.c +0
-1mq->last_tail =0 ;/* enable polling mode */ --2.7.4
Quoted 7 lines
Quoted 6 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; nd
Quoted 15 lines
Quoted 7 lines
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier
Using '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and ordered betweenCPUs on IA.
Quoted 7 lines
Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
The patch is looking good, but 'MEMIF_VERSION_MAJOR' in memif.h needs to be set to 3 as ring pointers are no longer volatile.
Quoted 32 lines
-----Original Message-----
From: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>
Sent: Tuesday, October 8, 2019 7:05 PM
To: Phil Yang (Arm Technology China) <[email protected]>; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Damjan Marion (damarion) <[email protected]>; nd <[email protected]>; Gavin Hu (Arm Technology China) <[email protected]>; nd <[email protected]>
Subject: RE: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrierQuoted 6 lines
Quoted 6 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; ndQuoted 8 lines
Quoted 7 lines
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier
Using '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and orderedbetween
Quoted 4 lines
CPUs on IA.
Quoted 3 lines
Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd
throughput
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines
boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>The patch is looking good, but '
MEMIF_VERSION_MAJOR' inmemif.hneeds to
be set to 3 as ring pointers are no longer volatile.
Updated in v2.
Thanks for your comments.
Thanks,
Phil
Quoted 28 lines
Quoted 28 lines
Quoted 17 lines
Quoted 8 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; ndQuoted 8 lines
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier
Using '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and orderedbetween
Quoted 4 lines
CPUs on IA.
Quoted 3 lines
Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd
throughput
Quoted 4 lines
Quoted 4 lines
boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>
Quoted 8 lines
Quoted 2 lines
The patch is looking good, but '
MEMIF_VERSION_MAJOR' inmemif.hneeds to be set to 3 as ring pointers are no longer volatile.Updated in v2.
Thanks for your comments.Thanks,
Phil
I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.
Sorry for the inconvenience.
Ferruh Yigit
AppliedQuoted 34 lines · On 10/9/2019 12:17 PM, Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at…
Quoted 34 lines
Quoted 22 lines
Quoted 17 lines
Quoted 8 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Phil Yang
Sent: Monday, August 26, 2019 7:00 PM
To: [email protected]; [email protected]
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; [email protected]; ndQuoted 8 lines
Subject: [dpdk-dev] [PATCH v1] net/memif: optimized with one-way barrier
Using '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platforms, such like aarch64. Meanwhile update the shared ring head and tail are observable and orderedbetween
Quoted 4 lines
CPUs on IA.
Quoted 3 lines
Optimized this full barrier with the one-way barrier can improve the throughput. On aarch64 n1sdp server this patch make testpmd
Quoted 8 lines
throughput
Quoted 4 lines
Quoted 4 lines
boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>The patch is looking good, but '
MEMIF_VERSION_MAJOR' inmemif.hneeds to be set to 3 as ring pointers are no longer volatile.Updated in v2.
Thanks for your comments.
Quoted 9 lines
Quoted 2 lines
Thanks,
PhilI jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.
Sorry for the inconvenience.
Reviewed-by Jakub Grajciar <[email protected]>
Applied to dpdk-next-net/master, thanks.
PATCH v2 1/1 net/memif: optimized with one-way barrier
In reply to Phil YangUsing 'rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.
Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)
v1:
Initial version.
drivers/net/memif/memif.h | 6 ++--- drivers/net/memif/rte_eth_memif.c | 53 +++++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 27 deletions(-)
drivers/net/memif/memif.h +3 -3
@@ -6,7 +6,7 @@#define _MEMIF_H_#define MEMIF_COOKIE 0x3E31F20 -#define MEMIF_VERSION_MAJOR 2 +#define MEMIF_VERSION_MAJOR 3#define MEMIF_VERSION_MINOR 0#define MEMIF_VERSION ((MEMIF_VERSION_MAJOR << 8) | MEMIF_VERSION_MINOR)#define MEMIF_NAME_SZ 32
@@ -169,9 +169,9 @@ typedef struct {uint32_t cookie;/**< MEMIF_COOKIE */ uint16_t flags;/**< flags */ #define MEMIF_RING_FLAG_MASK_INT 1/**< disable interrupt mode */ -volatile uint16_t head;/**< pointer to ring buffer head */ +uint16_t head;/**< pointer to ring buffer head */ MEMIF_CACHELINE_ALIGN_MARK (cacheline1); -volatile uint16_t tail;/**< pointer to ring buffer tail */ +uint16_t tail;/**< pointer to ring buffer tail */ MEMIF_CACHELINE_ALIGN_MARK (cacheline2);memif_desc_t desc[0 ];/**< buffer descriptors */ } memif_ring_t;
drivers/net/memif/rte_eth_memif.c +29 -24
117 lines not on this page · raw message
Signature · 1 line
2.7.4
Quoted 21 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Wednesday, October 9, 2019 4:05 AM
To: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Damjan Marion (damarion) <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: [PATCH v2] net/memif: optimized with one-way barrierUsing '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Quoted 8 lines
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)v1:
Initial version.
I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.
Sorry for the inconvenience.
Quoted 28 lines · On Wed, Oct 9, 2019 at 1:15 PM Jakub Grajciar -X (jgrajcia - PANTHEON…
Quoted 28 lines
-----Original Message-----
From: Phil Yang <[email protected]>
Sent: Wednesday, October 9, 2019 4:05 AM
To: Jakub Grajciar -X (jgrajcia - PANTHEON TECHNOLOGIES at Cisco) <[email protected]>; [email protected]; [email protected]
Cc: [email protected]; Damjan Marion (damarion) <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: [PATCH v2] net/memif: optimized with one-way barrierUsing '
rte_mb' to synchronize the shared ring head/tail between producer and consumer will stall the pipeline and damage performance on the weak memory model platform such like AArch64. Meanwhile update the shared ring head and tail are observable and ordered between CPUs on IA.Optimized this full barrier with the one-way barrier can improve the throughput. On AArch64 n1sdp server this patch make testpmd throughput boost 2.1%. On Intel E5-2640, testpmd got 3.98% performance gain.
Signed-off-by Phil Yang <[email protected]>Reviewed-by Gavin Hu <[email protected]>---
v2:
Upgrade 'MEMIF_VERSION_MAJOR' number to 3. (Jakub Grajciar)
Quoted 6 lines
Quoted 2 lines
v1:
Initial version.I jumped the gun with the version bump. The change doesn't break compatibility. I'm putting reviewed label on v1.
Sorry for the inconvenience.
Ok, marking v2 as rejected then.
Signature · 1 line
David Marchand