i40e vPMD optimization on aarch64
cover i40e vPMD optimization on aarch64
This series is to optimize the i40e vPMD performance on aarch64.
The patches were benchmarked by running the following command on Marvell ThunderX2 and Arm N1SDP and showed positive performance results.
sudo ./build/app/testpmd -l 1,3 -w 0001:01:00.0 -w 0001:01:00.1 --master-lcore 1 -- -i --rxq=4 --txq=4 --nb-cores=1 --nb-ports=2 -a
Gavin Hu (3):
net/i40e: relax barrier in the Tx fastpath of vPMD net/i40e: restrict pointer aliasing for neon vec net/i40e: auto-vectorization to speed up Tx free
drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ drivers/net/i40e/i40e_rxtx_vec_neon.c | 24 +++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/3 net/i40e: relax barrier in the Tx fastpath of vPMD
To keep ordering of mixed accesses, rte_cio is sufficient. The rte_io barrier inside the I40E_PCI_REG_WRITE is overkill.[1]
This patch fixes by replacing with just sufficient barriers in the normal PMD and vPMD.
It showed 7% performance uplift on ThunderX2 and 4% on Arm N1SDP. The test case is the RFC2544 zero-loss test running testpmd.
[1] http://inbox.dpdk.org/dev/CALBAE1M-ezVWCjqCZDBw+MMDEC4O9
[email protected]
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +4 -2
@@ -72,8 +72,9 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) rx_id = (uint16_t )((rxq->rxrearm_start ==0 ) ? (rxq->nb_rx_desc -1 ) : (rxq->rxrearm_start -1 )); +rte_cio_wmb ();/* Update the tail pointer on the NIC */ -I40E_PCI_REG_WRITE (rxq->qrx_tail, rx_id); +I40E_PCI_REG_WRITE_RELAXED (rxq->qrx_tail, rx_id); } static inline void
@@ -564,7 +565,8 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -I40E_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail); +rte_cio_wmb (); +I40E_PCI_REG_WRITE_RELAXED (txq->qtx_tail, tx_id);return nb_pkts; }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v1 1/3] net/i40e: relax barrier in the Tx fastpath of vPMD
Quoted 10 lines · Gavin Hu
To keep ordering of mixed accesses,
rte_ciois sufficient. Therte_iobarrier inside theI40E_PCI_REG_WRITEis overkill.[1]This patch fixes by replacing with just sufficient barriers in the normal PMD and vPMD.
It showed 7% performance uplift on ThunderX2 and 4% on Arm N1SDP. The test case is the RFC2544 zero-loss test running testpmd.
Quoted 7 lines
[1] http://inbox.dpdk.org/dev/CALBAE1M-ezVWCjqCZDBw+MMDEC4O9
[email protected]Fixes 4861cde46116 ("i40e: new poll mode driver")Signed-off-by Gavin Hu <[email protected]>
Quoted 32 lines
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)drivers/net/i40e/i40e_rxtx_vec_neon.c +4
-3@@ -72,8 +72,9 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) rx_id = (uint16_t )((rxq->rxrearm_start ==0 ) ? (rxq->nb_rx_desc -1 ) : (rxq->rxrearm_start -1 )); +rte_cio_wmb ();/* Update the tail pointer on the NIC */ -I40E_PCI_REG_WRITE (rxq->qrx_tail, rx_id); +I40E_PCI_REG_WRITE_RELAXED (rxq->qrx_tail, rx_id); } static inline void@@ -564,7 +565,8 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -I40E_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail); +rte_cio_wmb (); +I40E_PCI_REG_WRITE_RELAXED (txq->qtx_tail, tx_id);return nb_pkts; } --2.17.1
PATCH 2/3 net/i40e: restrict pointer aliasing for neon vec
In reply to Gavin Hurestrict pointer aliasing to optimize the code generated.
The patch showed ~3% performnace uplift on Arm N1SDP platform, and no degradation on ThunderX2. The tet case is RFC2544 zero-loss L2 forwarding running testpmd.
[1] https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Restricted-Pointers.html
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +9 -9
@@ -172,8 +172,8 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],#define I40E_UINT16_BIT (CHAR_BIT * sizeof(uint16_t))static inline void -desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **rx_pkts, -uint32_t *ptype_tbl) +desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **restrict rx_pkts, +uint32_t *restrict ptype_tbl) {int i;uint8_t ptype;
@@ -194,8 +194,8 @@ desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts,* numbers of DD bits */ static inline uint16_t -_recv_raw_pkts_vec (struct i40e_rx_queue *rxq,struct rte_mbuf **rx_pkts, -uint16_t nb_pkts,uint8_t *split_packet) +_recv_raw_pkts_vec (struct i40e_rx_queue *restrict rxq,struct rte_mbuf + **restrict rx_pkts,uint16_t nb_pkts,uint8_t *split_packet) {volatile union i40e_rx_desc *rxdp;struct i40e_rx_entry *sw_ring;
@@ -432,7 +432,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,* numbers of DD bits */ uint16_t -i40e_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts, +i40e_recv_pkts_vec (void *restrict rx_queue,struct rte_mbuf **restrict rx_pkts,uint16_t nb_pkts) {return _recv_raw_pkts_vec (rx_queue, rx_pkts, nb_pkts,NULL );
@@ -494,8 +494,8 @@ vtx1(volatile struct i40e_tx_desc *txdp, }static inline void -vtx (volatile struct i40e_tx_desc *txdp, -struct rte_mbuf **pkt,uint16_t nb_pkts,uint64_t flags) +vtx (volatile struct i40e_tx_desc *txdp,struct rte_mbuf **pkt, +uint16_t nb_pkts,uint64_t flags) {int i;
@@ -504,8 +504,8 @@ vtx(volatile struct i40e_tx_desc *txdp, }uint16_t -i40e_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts, -uint16_t nb_pkts) +i40e_xmit_fixed_burst_vec (void *restrict tx_queue, +struct rte_mbuf **restrict tx_pkts,uint16_t nb_pkts) {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;
Signature · 1 line
2.17.1
PATCH 3/3 net/i40e: auto-vectorization to speed up Tx free
In reply to Gavin HuTx mbuf free is a hotspot for i40e on aarch64, as there are no inter-loop dependencies, it is safe to enable auto-vectorization to speed up.
This patch showed 2~3% performance lift on ThunderX2 and no degradation on Arm N1SDP. The test case is single core RFC2544 zero-loss test.
--- drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ 1 file changed, 5 insertions(+)
drivers/net/i40e/i40e_rxtx_vec_common.h +5 -0
@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety) +#elif defined(__GNUC__) +#pragma GCC ivdep +#endif for (i =1 ; i < n; i++) { m =rte_pktmbuf_prefree_seg (txep[i].mbuf);if (likely (m !=NULL )) {
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx free
Quoted 19 lines · Gavin Hu
Tx mbuf free is a hotspot for i40e on aarch64, as there are no inter-loop dependencies, it is safe to enable auto-vectorization to speed up.
This patch showed 2~3% performance lift on ThunderX2 and no degradation on Arm N1SDP. The test case is single core RFC2544 zero-loss test.
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>--- drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ 1 file changed, 5 insertions(+)drivers/net/i40e/i40e_rxtx_vec_common.h +0
-0
drivers/net/i40e/i40e_rxtx_vec_common.h i40e_tx_free_bufs(struct i40e_tx_queue *txq) · line 105
Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +5
-0@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety) +#elif defined(__GNUC__) +#pragma GCC ivdep +#endif
IMO, It is better to abstract the compiler features (above compiler feature and __restrict__) as macros inrte_common.h or so. It will help to support other compilers(ICC or Windows) and enable them to have "changes" in one place.
Quoted 6 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +0
-1for (i =1 ; i < n; i++) { m =rte_pktmbuf_prefree_seg (txep[i].mbuf);if (likely (m !=NULL )) { --2.17.1
Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx free
06/03/2020 08:44, Jerin Jacob:
Quoted 9 lines · Gavin Hu
Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +2
-0@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety)
Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h i40e_tx_free_bufs(struct i40e_tx_queue *txq) · line 105
Quoted 3 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +3
-0+#elif defined(__GNUC__) +#pragma GCC ivdep +#endif IMO, It is better to abstract the compiler features (above compiler feature and
__restrict__) as macros inrte_common.hor so. It will help to support other compilers(ICC or Windows) and enable them to have "changes" in one place.
I agree with the need for common abstraction.
Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx free
In reply to Jerin JacobHi Jerin,
Quoted 40 lines
-----Original Message-----
From: Jerin Jacob <[email protected]>
Sent: Friday, March 6, 2020 3:45 PM
To: Gavin Hu <[email protected]>
Cc: dpdk-dev <[email protected]>; nd <[email protected]>; David Marchand <[email protected]>; [email protected]; [email protected]; Ye, Xiaolong <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>; Joyce Kong <[email protected]>; Steve Capper <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx freeQuoted 6 lines · Gavin Hu
Tx mbuf free is a hotspot for i40e on aarch64, as there are no inter-loop dependencies, it is safe to enable auto-vectorization to speed up.
This patch showed 2~3% performance lift on ThunderX2 and no
degradation
Review
Quoted 9 lines
on Arm N1SDP. The test case is single core RFC2544 zero-loss test.
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>--- drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ 1 file changed, 5 insertions(+)(quoted diff) +0
-0
b/drivers/net/i40e/i40e_rxtx_vec_common.hQuoted 9 lines
index
0e6ffa007..fc0fa45d4100644drivers/net/i40e/i40e_rxtx_vec_common.h +2
-0@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety)
Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h i40e_tx_free_bufs(struct i40e_tx_queue *txq) · line 105
Quoted 3 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +3
-0+#elif defined(__GNUC__) +#pragma GCC ivdep +#endif IMO, It is better to abstract the compiler features (above compiler feature and
__restrict__) as macros inrte_common.hor so. It will help to support other compilers(ICC or Windows) and enable them to have "changes" in one place.
How about defining RTE_LOOP_AUTO_VECTORIZATION in the rte_common.h?
#if defined(__clang__) define RTE_LOOP_AUTO_VECTORIZATION \ #pragma clang loop vectorize(assume_safety) #elif defined(__GNUC__) define RTE_LOOP_AUTO_VECTORIZATION \ #pragma GCC ivdep
#else
define RTE_LOOP_AUTO_VECTORIZATION
#endif
If you agree, I will submit a v2. Thanks for your comments! /Gavin
Quoted 9 lines
Quoted 9 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +0
-1for (i =1 ; i < n; i++) { m =rte_pktmbuf_prefree_seg (txep[i].mbuf);if (likely (m !=NULL )) { --2.17.1
Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx free
Quoted 52 lines · Gavin Hu
Hi Jerin,
Quoted 40 lines
-----Original Message-----
From: Jerin Jacob <[email protected]>
Sent: Friday, March 6, 2020 3:45 PM
To: Gavin Hu <[email protected]>
Cc: dpdk-dev <[email protected]>; nd <[email protected]>; David Marchand <[email protected]>; [email protected]; [email protected]; Ye, Xiaolong <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>; Joyce Kong <[email protected]>; Steve Capper <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx freeQuoted 6 lines · Gavin Hu
Tx mbuf free is a hotspot for i40e on aarch64, as there are no inter-loop dependencies, it is safe to enable auto-vectorization to speed up.
This patch showed 2~3% performance lift on ThunderX2 and no
degradation
Review
Quoted 9 lines
on Arm N1SDP. The test case is single core RFC2544 zero-loss test.
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>--- drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ 1 file changed, 5 insertions(+)(quoted diff) +0
-0
b/drivers/net/i40e/i40e_rxtx_vec_common.hQuoted 9 lines
index
0e6ffa007..fc0fa45d4100644drivers/net/i40e/i40e_rxtx_vec_common.h +2
-0@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety)Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h i40e_tx_free_bufs(struct i40e_tx_queue *txq) · line 105
Quoted 3 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +3
-0+#elif defined(__GNUC__) +#pragma GCC ivdep +#endif IMO, It is better to abstract the compiler features (above compiler feature and
__restrict__) as macros inrte_common.hor so. It will help to support other compilers(ICC or Windows) and enable them to have "changes" in one place.
Quoted 1 line
How about defining
RTE_LOOP_AUTO_VECTORIZATIONin therte_common.h?
Other compiler stuff in rte_common.h are starting with __rte in small letter(__rte_packed, __rte_unused) etc.
I think, a better name would be __rte_loop_auto_vectorize or so. No strong opinion for the name though.
# Probably it is worth checking and add performance result of x86 testing in git commit as well as it
is common code.
Quoted 20 lines
#if
defined(__clang__)
defineRTE_LOOP_AUTO_VECTORIZATION\
#pragma clang loopvectorize(assume_safety)
#elifdefined(__GNUC__)
defineRTE_LOOP_AUTO_VECTORIZATION\
#pragma GCC ivdep
#else
defineRTE_LOOP_AUTO_VECTORIZATION
#endif
If you agree, I will submit a v2. Thanks for your comments! /GavinQuoted 9 lines
Quoted 9 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +0
-1for (i =1 ; i < n; i++) { m =rte_pktmbuf_prefree_seg (txep[i].mbuf);if (likely (m !=NULL )) { --2.17.1
Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx free
Hi Jerin,
Quoted 69 lines
-----Original Message-----
From: Jerin Jacob <[email protected]>
Sent: Monday, March 9, 2020 3:36 PM
To: Gavin Hu <[email protected]>
Cc: dpdk-dev <[email protected]>; nd <[email protected]>; David Marchand <[email protected]>; [email protected]; [email protected]; Ye, Xiaolong <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>; Joyce Kong <[email protected]>; Steve Capper <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx freeQuoted 52 lines · Gavin Hu
Hi Jerin,
Quoted 40 lines
-----Original Message-----
From: Jerin Jacob <[email protected]>
Sent: Friday, March 6, 2020 3:45 PM
To: Gavin Hu <[email protected]>
Cc: dpdk-dev <[email protected]>; nd <[email protected]>; David Marchand <[email protected]>; [email protected]; [email protected]; Ye, Xiaolong <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>; Joyce Kong <[email protected]>; Steve Capper <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v1 3/3] net/i40e: auto-vectorization to speed up Tx freeQuoted 6 lines · Gavin Hu
Tx mbuf free is a hotspot for i40e on aarch64, as there are no inter-loop dependencies, it is safe to enable auto-vectorization to speed up.
This patch showed 2~3% performance lift on ThunderX2 and no
degradation
Review
Quoted 9 lines
on Arm N1SDP. The test case is single core RFC2544 zero-loss test.
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>--- drivers/net/i40e/i40e_rxtx_vec_common.h | 5 +++++ 1 file changed, 5 insertions(+)(quoted diff) +0
-0
b/drivers/net/i40e/i40e_rxtx_vec_common.hQuoted 9 lines
index
0e6ffa007..fc0fa45d4100644drivers/net/i40e/i40e_rxtx_vec_common.h +2
-0@@ -98,6 +98,11 @@ i40e_tx_free_bufs(struct i40e_tx_queue *txq)if (likely (m !=NULL )) { free[0 ] = m; nb_free =1 ; +#if defined(__clang__) +#pragma clang loop vectorize(assume_safety)Quoted 8 lines
drivers/net/i40e/i40e_rxtx_vec_common.h i40e_tx_free_bufs(struct i40e_tx_queue *txq) · line 105
Quoted 3 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +3
-0+#elif defined(__GNUC__) +#pragma GCC ivdep +#endif IMO, It is better to abstract the compiler features (above compiler feature and
__restrict__) as macros inrte_common.hor so. It will help to support other compilers(ICC or Windows) and enable them to have "changes" in one place.Quoted 1 line
How about defining
RTE_LOOP_AUTO_VECTORIZATIONin the
rte_common.h?
Quoted 8 lines
Other compiler stuff in
rte_common.hare starting with__rtein smallletter(__rte_packed, __rte_unused)etc.
I think, a better name would be__rte_loop_auto_vectorizeor so. No strong opinion for the name though.# Probably it is worth checking and add performance result of x86 testing in git commit as well as it
is common code.
Okay, I will do it.
Quoted 22 lines
Quoted 22 lines
#if
defined(__clang__)
defineRTE_LOOP_AUTO_VECTORIZATION\
#pragma clang loopvectorize(assume_safety)
#elifdefined(__GNUC__)
defineRTE_LOOP_AUTO_VECTORIZATION\
#pragma GCC ivdep
#else
defineRTE_LOOP_AUTO_VECTORIZATION
#endif
If you agree, I will submit a v2. Thanks for your comments! /GavinQuoted 9 lines
Quoted 9 lines
drivers/net/i40e/i40e_rxtx_vec_common.h +0
-1for (i =1 ; i < n; i++) { m =rte_pktmbuf_prefree_seg (txep[i].mbuf);if (likely (m !=NULL )) { --2.17.1
v2 cover i40e NEON vPMD optimization on aarch64
In reply to Gavin HuV2:
- drop the 3/3 auto-vectorization patch from this series, it requires more rework and will submit in a separate patch.
This series is to optimize the i40e NEON vPMD performance on aarch64.
The patches were benchmarked by running the following command on Marvell ThunderX2 and Arm N1SDP and showed positive performance results.
sudo ./build/app/testpmd -l 1,3 -w 0001:01:00.0 -w 0001:01:00.1 --master-lcore 1 -- -i --rxq=4 --txq=4 --nb-cores=1 --nb-ports=2 -a
Gavin Hu (2):
net/i40e: relax barrier in Tx fastpath for NEON vPMD net/i40e: restrict pointer aliasing for NEON vPMD
drivers/net/i40e/i40e_rxtx_vec_neon.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/2 net/i40e: relax barrier in Tx fastpath for NEON vPMD
To keep ordering of mixed accesses, 'DMB OSH' is sufficient. 'DSB' inside the I40E_PCI_REG_WRITE is overkill.[1]
This patch fixes by replacing with just sufficient barriers in the normal PMD and vPMD.
It showed 7% performance uplift on ThunderX2 and 4% on Arm N1SDP. The test case is the RFC2544 zero-loss test running testpmd.
[1] http://inbox.dpdk.org/dev/CALBAE1M-ezVWCjqCZDBw+MMDEC4O9
[email protected]
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +4 -2
@@ -72,8 +72,9 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) rx_id = (uint16_t )((rxq->rxrearm_start ==0 ) ? (rxq->nb_rx_desc -1 ) : (rxq->rxrearm_start -1 )); +rte_cio_wmb ();/* Update the tail pointer on the NIC */ -I40E_PCI_REG_WRITE (rxq->qrx_tail, rx_id); +I40E_PCI_REG_WRITE_RELAXED (rxq->qrx_tail, rx_id); } static inline void
@@ -564,7 +565,8 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -I40E_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail); +rte_cio_wmb (); +I40E_PCI_REG_WRITE_RELAXED (txq->qtx_tail, tx_id);return nb_pkts; }
Signature · 1 line
2.17.1
PATCH v2 2/2 net/i40e: restrict pointer aliasing for NEON vPMD
In reply to Gavin Hurestrict pointer aliasing to optimize the code generated.
The patch showed ~3% performnace uplift on Arm N1SDP platform, and no degradation on ThunderX2. The tet case is RFC2544 zero-loss L2 forwarding running testpmd.
[1] https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Restricted-Pointers.html
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +9 -9
@@ -172,8 +172,8 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],#define I40E_UINT16_BIT (CHAR_BIT * sizeof(uint16_t))static inline void -desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **rx_pkts, -uint32_t *ptype_tbl) +desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **restrict rx_pkts, +uint32_t *restrict ptype_tbl) {int i;uint8_t ptype;
@@ -194,8 +194,8 @@ desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts,* numbers of DD bits */ static inline uint16_t -_recv_raw_pkts_vec (struct i40e_rx_queue *rxq,struct rte_mbuf **rx_pkts, -uint16_t nb_pkts,uint8_t *split_packet) +_recv_raw_pkts_vec (struct i40e_rx_queue *restrict rxq,struct rte_mbuf + **restrict rx_pkts,uint16_t nb_pkts,uint8_t *split_packet) {volatile union i40e_rx_desc *rxdp;struct i40e_rx_entry *sw_ring;
@@ -432,7 +432,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,* numbers of DD bits */ uint16_t -i40e_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts, +i40e_recv_pkts_vec (void *restrict rx_queue,struct rte_mbuf **restrict rx_pkts,uint16_t nb_pkts) {return _recv_raw_pkts_vec (rx_queue, rx_pkts, nb_pkts,NULL );
@@ -494,8 +494,8 @@ vtx1(volatile struct i40e_tx_desc *txdp, }static inline void -vtx (volatile struct i40e_tx_desc *txdp, -struct rte_mbuf **pkt,uint16_t nb_pkts,uint64_t flags) +vtx (volatile struct i40e_tx_desc *txdp,struct rte_mbuf **pkt, +uint16_t nb_pkts,uint64_t flags) {int i;
@@ -504,8 +504,8 @@ vtx(volatile struct i40e_tx_desc *txdp, }uint16_t -i40e_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts, -uint16_t nb_pkts) +i40e_xmit_fixed_burst_vec (void *restrict tx_queue, +struct rte_mbuf **restrict tx_pkts,uint16_t nb_pkts) {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v2 2/2] net/i40e: restrict pointer aliasing for NEON vPMD
Quoted 10 lines · Gavin Hu
restrict pointer aliasing to optimize the code generated.
The patch showed ~3% performnace uplift on Arm N1SDP platform, and no degradation on ThunderX2. The tet case is RFC2544 zero-loss L2 forwarding running testpmd.
[1] https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Restricted-Pointers.html
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>
Hi Gavin,
Raslan supported following build error [1], this is blocking the next-net, can you please check the error, and send a fix for it?
[1]
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:175:65: error: expected ';', ',' or ')' before 'rx_pkts'
desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **restrict rx_pkts, ^
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:197:51: error: expected ';', ',' or ')' before 'rxq'
_recv_raw_pkts_vec(struct i40e_rx_queue *restrict rxq, struct rte_mbuf ^
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:435:35: error: expected ';', ',' or ')' before 'rx_queue'
i40e_recv_pkts_vec(void *restrict rx_queue, struct rte_mbuf **restrict rx_pkts, ^
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c: In function 'i40e_recv_scattered_pkts_vec':
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:456:2: error: implicit declaration of function '_recv_raw_pkts_vec' [-Werror=implicit-function-declaration] uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
^
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:456:2: error: nested extern declaration of '_recv_raw_pkts_vec' [-Werror=nested-externs] /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c: At top level:
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:507:42: error: expected ';', ',' or ')' before 'tx_queue'
i40e_xmit_fixed_burst_vec(void *restrict tx_queue,
gcc version 4.8.5 20150623 (Red Hat 4.8.5-28) (GCC)
seen this with T=arm64-bluefield-linuxapp-gcc and T=arm64-armv8a-linux-gcc using make
Re: [dpdk-dev] [PATCH v2 2/2] net/i40e: restrict pointer aliasing for NEON vPMD
Hi Ferruh,
Quoted 66 lines
-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Monday, April 20, 2020 10:51 PM
To: Gavin Hu <[email protected]>; [email protected]
Cc: nd <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>; Joyce Kong <[email protected]>; Steve Capper <[email protected]>; Raslan Darawsheh <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 2/2] net/i40e: restrict pointer aliasing for NEON vPMDQuoted 3 lines · Gavin Hu
restrict pointer aliasing to optimize the code generated.
The patch showed ~3% performnace uplift on Arm N1SDP platform, and
no
Quoted 7 lines
degradation on ThunderX2. The tet case is RFC2544 zero-loss L2 forwarding running testpmd.
[1] https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Restricted-Pointers.html
Signed-off-by Gavin Hu <[email protected]>Reviewed-by Steve Capper <[email protected]>Hi Gavin,
Raslan supported following build error [1], this is blocking the next-net, can you please check the error, and send a fix for it?
[1] /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:175:65: error:
expected
';', ',' or ')' before 'rx_pkts'
desc_to_ptype_v(uint64x2_t descs[4], structrte_mbuf**restrictrx_pkts, ^ /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:197:51: error:
expected
';', ',' or ')' before 'rxq'
_recv_raw_pkts_vec(structi40e_rx_queue*restrict rxq, structrte_mbuf^ /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:435:35: error:
expected
';', ',' or ')' before 'rx_queue'
i40e_recv_pkts_vec(void *restrictrx_queue, structrte_mbuf**restrictrx_pkts,
^
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c: In function 'i40e_recv_scattered_pkts_vec':
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:456:2: error:
implicit
declaration of function '_recv_raw_pkts_vec' [-Werror=implicit-function- declaration]uint16_tnb_bufs= _recv_raw_pkts_vec(rxq,rx_pkts,nb_pkts, ^ /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:456:2: error:
nested extern
declaration of '_recv_raw_pkts_vec' [-Werror=nested-externs] /download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c: At top level:
/download/dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c:507:42: error:
expected
';', ',' or ')' before 'tx_queue' i40e_xmit_fixed_burst_vec(void *restricttx_queue,
Quoted 4 lines
gcc version 4.8.5 20150623 (Red Hat 4.8.5-28) (GCC)
seen this with T=arm64-bluefield-linuxapp-gcc and T=arm64-armv8a-linux- gcc
using make
Sorry for the leakage, I submitted a patch to fix this: http://patches.dpdk.org/patch/69006/
It was verified with gcc 4.8.5 and gcc-8, as well as meson + clang.
v3 cover i40e NEON vPMD optimization on aarch64
In reply to Gavin HuV3:
- fix the 'Fixes' line warning.
V2:
- drop the 3/3 auto-vectorization patch from this series, it requires more rework and will submit in a separate patch.
This series is to optimize the i40e NEON vPMD performance on aarch64.
The patches were benchmarked by running the following command on Marvell ThunderX2 and Arm N1SDP and showed positive performance results.
sudo ./build/app/testpmd -l 1,3 -w 0001:01:00.0 -w 0001:01:00.1 --master-lcore 1 -- -i --rxq=4 --txq=4 --nb-cores=1 --nb-ports=2 -a
Gavin Hu (2):
net/i40e: relax barrier in Tx fastpath for NEON vPMD net/i40e: restrict pointer aliasing for NEON vPMD
drivers/net/i40e/i40e_rxtx_vec_neon.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-)
Signature · 1 line
2.17.1
PATCH v3 1/2 net/i40e: relax barrier in Tx fastpath for NEON vPMD
To keep ordering of mixed accesses, 'DMB OSH' is sufficient. 'DSB' inside the I40E_PCI_REG_WRITE is overkill.[1]
This patch fixes by replacing with just sufficient barriers in the normal PMD and vPMD.
It showed 7% performance uplift on ThunderX2 and 4% on Arm N1SDP. The test case is the RFC2544 zero-loss test running testpmd.
[1] http://inbox.dpdk.org/dev/CALBAE1M-ezVWCjqCZDBw+MMDEC4O9
[email protected]
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +4 -2
@@ -72,8 +72,9 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) rx_id = (uint16_t )((rxq->rxrearm_start ==0 ) ? (rxq->nb_rx_desc -1 ) : (rxq->rxrearm_start -1 )); +rte_cio_wmb ();/* Update the tail pointer on the NIC */ -I40E_PCI_REG_WRITE (rxq->qrx_tail, rx_id); +I40E_PCI_REG_WRITE_RELAXED (rxq->qrx_tail, rx_id); } static inline void
@@ -564,7 +565,8 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; -I40E_PCI_REG_WRITE (txq->qtx_tail, txq->tx_tail); +rte_cio_wmb (); +I40E_PCI_REG_WRITE_RELAXED (txq->qtx_tail, tx_id);return nb_pkts; }
Signature · 1 line
2.17.1
PATCH v3 2/2 net/i40e: restrict pointer aliasing for NEON vPMD
In reply to Gavin Hurestrict pointer aliasing to optimize the code generated.
The patch showed ~3% performnace uplift on Arm N1SDP platform, and no degradation on ThunderX2. The tet case is RFC2544 zero-loss L2 forwarding running testpmd.
[1] https://gcc.gnu.org/onlinedocs/gcc-4.8.5/gcc/Restricted-Pointers.html
--- drivers/net/i40e/i40e_rxtx_vec_neon.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
drivers/net/i40e/i40e_rxtx_vec_neon.c +9 -9
@@ -172,8 +172,8 @@ desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],#define I40E_UINT16_BIT (CHAR_BIT * sizeof(uint16_t))static inline void -desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **rx_pkts, -uint32_t *ptype_tbl) +desc_to_ptype_v (uint64x2_t descs[4 ],struct rte_mbuf **restrict rx_pkts, +uint32_t *restrict ptype_tbl) {int i;uint8_t ptype;
@@ -194,8 +194,8 @@ desc_to_ptype_v(uint64x2_t descs[4], struct rte_mbuf **rx_pkts,* numbers of DD bits */ static inline uint16_t -_recv_raw_pkts_vec (struct i40e_rx_queue *rxq,struct rte_mbuf **rx_pkts, -uint16_t nb_pkts,uint8_t *split_packet) +_recv_raw_pkts_vec (struct i40e_rx_queue *restrict rxq,struct rte_mbuf + **restrict rx_pkts,uint16_t nb_pkts,uint8_t *split_packet) {volatile union i40e_rx_desc *rxdp;struct i40e_rx_entry *sw_ring;
@@ -432,7 +432,7 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,* numbers of DD bits */ uint16_t -i40e_recv_pkts_vec (void *rx_queue,struct rte_mbuf **rx_pkts, +i40e_recv_pkts_vec (void *restrict rx_queue,struct rte_mbuf **restrict rx_pkts,uint16_t nb_pkts) {return _recv_raw_pkts_vec (rx_queue, rx_pkts, nb_pkts,NULL );
@@ -494,8 +494,8 @@ vtx1(volatile struct i40e_tx_desc *txdp, }static inline void -vtx (volatile struct i40e_tx_desc *txdp, -struct rte_mbuf **pkt,uint16_t nb_pkts,uint64_t flags) +vtx (volatile struct i40e_tx_desc *txdp,struct rte_mbuf **pkt, +uint16_t nb_pkts,uint64_t flags) {int i;
@@ -504,8 +504,8 @@ vtx(volatile struct i40e_tx_desc *txdp, }uint16_t -i40e_xmit_fixed_burst_vec (void *tx_queue,struct rte_mbuf **tx_pkts, -uint16_t nb_pkts) +i40e_xmit_fixed_burst_vec (void *restrict tx_queue, +struct rte_mbuf **restrict tx_pkts,uint16_t nb_pkts) {struct i40e_tx_queue *txq = (struct i40e_tx_queue *)tx_queue;volatile struct i40e_tx_desc *txdp;
Signature · 1 line
2.17.1
Ye Xiaolong
AppliedRe: [dpdk-dev] [PATCH v3 0/2] i40e NEON vPMD optimization on aarch64
In reply to Gavin HuQuoted 14 lines · Gavin Hu
V3:
- fix the 'Fixes' line warning.
V2:
- drop the 3/3 auto-vectorization patch from this series, it requires more rework and will submit in a separate patch.This series is to optimize the i40e NEON vPMD performance on aarch64.
The patches were benchmarked by running the following command on Marvell ThunderX2 and Arm N1SDP and showed positive performance results.
sudo ./build/app/testpmd -l 1,3 -w 0001:01:00.0 -w 0001:01:00.1 --master-lcore 1 -- -i --rxq=4 --txq=4 --nb-cores=1 --nb-ports=2 -a
Quoted 10 lines
Gavin Hu (2):
net/i40e: relax barrier in Tx fastpath for NEON vPMD net/i40e: restrict pointer aliasing for NEON vPMD
drivers/net/i40e/i40e_rxtx_vec_neon.c| 24 +++++++++++++-----------1 file changed, 13 insertions(+), 11 deletions(-)Signature · 1 line
2.17.1
Applied to dpdk-next-net-intel, Thanks.