Skip to contentSkip to conversation

net/iavf: add NEON support for Rx/Tx paths

PATCH Superseded Jay Wang 2 participants 13 messages Apr 17 – May 5

Subsystems (2)
Network drivers Arm/Amlogic Meson SoC support
lore
Share & tools
lore

Jay Wang

author
#0 lore raw

cover net/iavf: add NEON support for Rx/Tx paths

- Add the scattered Rx burst function for 32B legacy descriptor
- Add the NEON-optimised Tx burst function
- Add the NEON-optimised RX burst functions for flexible descriptor

Jay Wang (3):
net/iavf: add Rx scattered function for 32B desc net/iavf: add NEON-optimised Tx burst function net/iavf: add NEON support for Rx flex desc

 drivers/net/intel/iavf/iavf.h               |   4 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  52 +-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 751 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 5 files changed, 795 insertions(+), 16 deletions(-)
Signature · 1 line
2.43.0
author
#1 lore raw

PATCH 1/3 net/iavf: add Rx scattered function for 32B desc

Added the scattered burst function on AArch64 so that we can leverage the NEON-optimised Rx raw burst function to handle scattered packets for the legacy 32B descriptor.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  16 ++-
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 110 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 4 files changed, 122 insertions(+), 7 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index 403c61e2e8..e4936f3566 100644
@@ -334,6 +334,7 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC,
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
+	IAVF_RX_NEON_SCATTERED,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +13 -3
index 4ff6c18dc4..15566a0e18 100644
@@ -3551,16 +3551,26 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 		}
 	},
 #endif
-#elif defined RTE_ARCH_ARM
+#elif defined(RTE_ARCH_ARM64)
 	[IAVF_RX_NEON] = {
 		.pkt_burst = iavf_recv_pkts_vec,
 		.info = "Vector Neon",
 		.features = {
-			.rx_offloads = IAVF_RX_SCALAR_OFFLOADS,
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS,
 			.simd_width = RTE_VECT_SIMD_128,
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_SCATTERED] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec,
+		.info = "Vector Scattered Neon",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
 

@@ -3839,7 +3849,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 	if (adapter->rx_bulk_alloc_allowed) {
 		req_features.bulk_alloc = true;
 		default_path = IAVF_RX_BULK_ALLOC;
-#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM)
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 		if (iavf_rx_vec_dev_check(dev) != -1)
 			req_features.simd_width = iavf_get_max_simd_bitwidth();
 #endif
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +107 -3

145 lines not on this page · raw message

drivers/net/intel/iavf/meson.build +1 -1
index f9576586f6..50630a88c8 100644
@@ -29,7 +29,7 @@ sources = files(
 if arch_subdir == 'x86'
     sources_avx2 += files('iavf_rxtx_vec_avx2.c')
     sources_avx512 += files('iavf_rxtx_vec_avx512.c')
-elif arch_subdir == 'arm'
+elif arch_subdir == 'arm' and dpdk_conf.get('RTE_ARCH_64')
     sources += files('iavf_rxtx_vec_neon.c')
 endif
Signature · 1 line
2.43.0
author
#2 lore raw

PATCH 2/3 net/iavf: add NEON-optimised Tx burst function

In reply to Jay Wang

This patch adds the NEON-optimised Tx burst function for Intel IAVF driver on AArch64.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  15 ++-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 120 ++++++++++++++++++++
 4 files changed, 133 insertions(+), 5 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index e4936f3566..3e71d345a9 100644
@@ -356,6 +356,7 @@ enum iavf_rx_func_type {
 enum iavf_tx_func_type {
 	IAVF_TX_DISABLED,
 	IAVF_TX_DEFAULT,
+	IAVF_TX_NEON,
 	IAVF_TX_AVX2,
 	IAVF_TX_AVX2_OFFLOAD,
 	IAVF_TX_AVX512,
drivers/net/intel/iavf/iavf_rxtx.c +12 -3
index 15566a0e18..645bc5ccf6 100644
@@ -3662,6 +3662,15 @@ static const struct ci_tx_path_info iavf_tx_path_infos[] = {
 		}
 	},
 #endif
+#elif defined(RTE_ARCH_ARM64)
+	[IAVF_TX_NEON] = {
+		.pkt_burst = iavf_xmit_pkts_vec,
+		.info = "Vector Neon",
+		.features = {
+			.tx_offloads = IAVF_TX_VECTOR_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128
+		}
+	},
 #endif
 };
 

@@ -3878,7 +3887,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	int mbuf_check = adapter->devargs.mbuf_check;
 	int no_poll_on_link_down = adapter->devargs.no_poll_on_link_down;
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	struct ci_tx_queue *txq;
 	int i;
 	const struct ci_tx_path_features *selected_features;

@@ -3892,7 +3901,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 	if (dev->data->dev_started)
 		goto out;
 
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	if (iavf_tx_vec_dev_check(dev) != -1)
 		req_features.simd_width = iavf_get_max_simd_bitwidth();
 

@@ -3915,7 +3924,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 						IAVF_TX_DEFAULT);
 
 out:
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	selected_features = &iavf_tx_path_infos[adapter->tx_func_type].features;
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
drivers/net/intel/iavf/iavf_rxtx.h +0 -2
index 80b06518b0..8b8e55e66f 100644
@@ -558,8 +558,6 @@ uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
 					       struct rte_mbuf **rx_pkts,
 					       uint16_t nb_pkts);
-uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
-				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +120 -0

132 lines not on this page · raw message

Signature · 1 line
2.43.0
author
#3 lore raw

PATCH 3/3 net/iavf: add NEON support for Rx flex desc

In reply to Jay Wang

This patch adds the NEON-optimised Rx paths to process receive flex descriptor.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   2 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  21 +
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 521 +++++++++++++++++++-
 3 files changed, 540 insertions(+), 4 deletions(-)
drivers/net/intel/iavf/iavf.h +2 -0
index 3e71d345a9..360d728f3a 100644
@@ -335,6 +335,8 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
 	IAVF_RX_NEON_SCATTERED,
+	IAVF_RX_NEON_FLEX_RXD,
+	IAVF_RX_NEON_SCATTERED_FLEX_RXD,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +21 -0
index 645bc5ccf6..ae005b0648 100644
@@ -3571,6 +3571,27 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_pkts_vec_flex_rxd,
+		.info = "Vector Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
+	[IAVF_RX_NEON_SCATTERED_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec_flex_rxd,
+		.info = "Vector Scattered Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +517 -4

550 lines not on this page · raw message

Signature · 1 line
2.43.0
author
#4 lore raw

v2 cover net/iavf: add NEON support for Rx/Tx paths

In reply to Jay Wang

- Add the scattered Rx burst function for 32B legacy descriptor
- Add the NEON-optimised Tx burst function
- Add the NEON-optimised Rx burst functions for flexible descriptor

---
v2:
- Fixed the AArch32 cross-building issue on AArch64 by restricting the call of iavf_rxq_vec_setup to AArch64 only

Jay Wang (3):
net/iavf: add Rx scattered function for 32B desc net/iavf: add NEON-optimised Tx burst function net/iavf: add NEON support for Rx flex desc

 drivers/net/intel/iavf/iavf.h               |   4 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  54 +-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 751 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 5 files changed, 796 insertions(+), 17 deletions(-)
Signature · 1 line
2.43.0
author
#5 lore raw

PATCH v2 1/3 net/iavf: add Rx scattered function for 32B desc

Added the scattered burst function on AArch64 so that we can leverage the NEON-optimised Rx raw burst function to handle scattered packets for the legacy 32B descriptor.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  16 ++-
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 110 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 4 files changed, 122 insertions(+), 7 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index 403c61e2e8..e4936f3566 100644
@@ -334,6 +334,7 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC,
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
+	IAVF_RX_NEON_SCATTERED,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +13 -3
index 4ff6c18dc4..15566a0e18 100644
@@ -3551,16 +3551,26 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 		}
 	},
 #endif
-#elif defined RTE_ARCH_ARM
+#elif defined(RTE_ARCH_ARM64)
 	[IAVF_RX_NEON] = {
 		.pkt_burst = iavf_recv_pkts_vec,
 		.info = "Vector Neon",
 		.features = {
-			.rx_offloads = IAVF_RX_SCALAR_OFFLOADS,
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS,
 			.simd_width = RTE_VECT_SIMD_128,
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_SCATTERED] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec,
+		.info = "Vector Scattered Neon",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
 

@@ -3839,7 +3849,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 	if (adapter->rx_bulk_alloc_allowed) {
 		req_features.bulk_alloc = true;
 		default_path = IAVF_RX_BULK_ALLOC;
-#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM)
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 		if (iavf_rx_vec_dev_check(dev) != -1)
 			req_features.simd_width = iavf_get_max_simd_bitwidth();
 #endif
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +107 -3

145 lines not on this page · raw message

drivers/net/intel/iavf/meson.build +1 -1
index f9576586f6..50630a88c8 100644
@@ -29,7 +29,7 @@ sources = files(
 if arch_subdir == 'x86'
     sources_avx2 += files('iavf_rxtx_vec_avx2.c')
     sources_avx512 += files('iavf_rxtx_vec_avx512.c')
-elif arch_subdir == 'arm'
+elif arch_subdir == 'arm' and dpdk_conf.get('RTE_ARCH_64')
     sources += files('iavf_rxtx_vec_neon.c')
 endif
Signature · 1 line
2.43.0
author
#6 lore raw

PATCH v2 2/3 net/iavf: add NEON-optimised Tx burst function

In reply to Jay Wang

This patch adds the NEON-optimised Tx burst function for Intel IAVF driver on AArch64.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  15 ++-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 120 ++++++++++++++++++++
 4 files changed, 133 insertions(+), 5 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index e4936f3566..3e71d345a9 100644
@@ -356,6 +356,7 @@ enum iavf_rx_func_type {
 enum iavf_tx_func_type {
 	IAVF_TX_DISABLED,
 	IAVF_TX_DEFAULT,
+	IAVF_TX_NEON,
 	IAVF_TX_AVX2,
 	IAVF_TX_AVX2_OFFLOAD,
 	IAVF_TX_AVX512,
drivers/net/intel/iavf/iavf_rxtx.c +12 -3
index 15566a0e18..645bc5ccf6 100644
@@ -3662,6 +3662,15 @@ static const struct ci_tx_path_info iavf_tx_path_infos[] = {
 		}
 	},
 #endif
+#elif defined(RTE_ARCH_ARM64)
+	[IAVF_TX_NEON] = {
+		.pkt_burst = iavf_xmit_pkts_vec,
+		.info = "Vector Neon",
+		.features = {
+			.tx_offloads = IAVF_TX_VECTOR_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128
+		}
+	},
 #endif
 };
 

@@ -3878,7 +3887,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	int mbuf_check = adapter->devargs.mbuf_check;
 	int no_poll_on_link_down = adapter->devargs.no_poll_on_link_down;
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	struct ci_tx_queue *txq;
 	int i;
 	const struct ci_tx_path_features *selected_features;

@@ -3892,7 +3901,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 	if (dev->data->dev_started)
 		goto out;
 
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	if (iavf_tx_vec_dev_check(dev) != -1)
 		req_features.simd_width = iavf_get_max_simd_bitwidth();
 

@@ -3915,7 +3924,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 						IAVF_TX_DEFAULT);
 
 out:
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	selected_features = &iavf_tx_path_infos[adapter->tx_func_type].features;
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
drivers/net/intel/iavf/iavf_rxtx.h +0 -2
index 80b06518b0..8b8e55e66f 100644
@@ -558,8 +558,6 @@ uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
 					       struct rte_mbuf **rx_pkts,
 					       uint16_t nb_pkts);
-uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
-				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +120 -0

132 lines not on this page · raw message

Signature · 1 line
2.43.0
author
#7 lore raw

PATCH v2 3/3 net/iavf: add NEON support for Rx flex desc

In reply to Jay Wang

This patch adds the NEON-optimised Rx paths to process receive flex descriptor.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   2 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  23 +-
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 521 +++++++++++++++++++-
 3 files changed, 541 insertions(+), 5 deletions(-)
drivers/net/intel/iavf/iavf.h +2 -0
index 3e71d345a9..360d728f3a 100644
@@ -335,6 +335,8 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
 	IAVF_RX_NEON_SCATTERED,
+	IAVF_RX_NEON_FLEX_RXD,
+	IAVF_RX_NEON_SCATTERED_FLEX_RXD,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +22 -1
index 645bc5ccf6..8e711950ff 100644
@@ -713,7 +713,7 @@ iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		ad->rx_bulk_alloc_allowed = false;
 	}
 
-#if defined RTE_ARCH_X86 || defined RTE_ARCH_ARM
+#if defined RTE_ARCH_X86 || defined RTE_ARCH_ARM64
 	/* check vector conflict */
 	if (ci_rxq_vec_capable(rxq->nb_rx_desc, rxq->rx_free_thresh) &&
 			iavf_rxq_vec_setup(rxq)) {

@@ -3571,6 +3571,27 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_pkts_vec_flex_rxd,
+		.info = "Vector Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
+	[IAVF_RX_NEON_SCATTERED_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec_flex_rxd,
+		.info = "Vector Scattered Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +517 -4

550 lines not on this page · raw message

Signature · 1 line
2.43.0
author
#8 lore raw

v3 cover net/iavf: add NEON support for Rx/Tx paths

In reply to Jay Wang

- Add the scattered Rx burst function for 32B legacy descriptor
- Add the NEON-optimised Tx burst function
- Add the NEON-optimised Rx burst functions for flexible descriptor

Single-core benchmark result on a Grace server with an Intel E810:

| buffer sz (B) | prev (MPPS) | optimised (MPPS) | uplift |
| ------------- | ----------- | ---------------- | ------ |
| 64            | 62.241      | 86.010           | 38.2%  |
| 128           | 60.813      | 81.453           | 33.9%  |
| 256           | 23.730      | 24.633           | 3.8%   |
| 512           | 10.268      | 10.380           | 1.1%   |
| 1024          | 4.588       | 4.628            | 0.9%   |
| 1518          | 4.601       | 4.669            | 1.5%   |

---
v3:
- Optimised the branch prediction in counting packets by changing unlikely() to likely()
- Added the single-core performance data from Grace + E810 done by Andrew

v2:
- Fixed the AArch32 cross-building issue on AArch64 by restricting the call of iavf_rxq_vec_setup to AArch64 only

Jay Wang (3):
net/iavf: add Rx scattered function for 32B desc net/iavf: add NEON-optimised Tx burst function net/iavf: add NEON support for Rx flex desc

 drivers/net/intel/iavf/iavf.h               |   4 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  54 +-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 753 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 5 files changed, 797 insertions(+), 18 deletions(-)
Signature · 1 line
2.43.0
author
#9 lore raw

PATCH v3 1/3 net/iavf: add Rx scattered function for 32B desc

Added the scattered burst function on AArch64 so that we can leverage the NEON-optimised Rx raw burst function to handle scattered packets for the legacy 32B descriptor.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  16 ++-
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 110 +++++++++++++++++++-
 drivers/net/intel/iavf/meson.build          |   2 +-
 4 files changed, 122 insertions(+), 7 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index 403c61e2e8..e4936f3566 100644
@@ -334,6 +334,7 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC,
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
+	IAVF_RX_NEON_SCATTERED,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +13 -3
index 4ff6c18dc4..15566a0e18 100644
@@ -3551,16 +3551,26 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 		}
 	},
 #endif
-#elif defined RTE_ARCH_ARM
+#elif defined(RTE_ARCH_ARM64)
 	[IAVF_RX_NEON] = {
 		.pkt_burst = iavf_recv_pkts_vec,
 		.info = "Vector Neon",
 		.features = {
-			.rx_offloads = IAVF_RX_SCALAR_OFFLOADS,
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS,
 			.simd_width = RTE_VECT_SIMD_128,
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_SCATTERED] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec,
+		.info = "Vector Scattered Neon",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
 

@@ -3839,7 +3849,7 @@ iavf_set_rx_function(struct rte_eth_dev *dev)
 	if (adapter->rx_bulk_alloc_allowed) {
 		req_features.bulk_alloc = true;
 		default_path = IAVF_RX_BULK_ALLOC;
-#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM)
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 		if (iavf_rx_vec_dev_check(dev) != -1)
 			req_features.simd_width = iavf_get_max_simd_bitwidth();
 #endif
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +107 -3

145 lines not on this page · raw message

drivers/net/intel/iavf/meson.build +1 -1
index f9576586f6..50630a88c8 100644
@@ -29,7 +29,7 @@ sources = files(
 if arch_subdir == 'x86'
     sources_avx2 += files('iavf_rxtx_vec_avx2.c')
     sources_avx512 += files('iavf_rxtx_vec_avx512.c')
-elif arch_subdir == 'arm'
+elif arch_subdir == 'arm' and dpdk_conf.get('RTE_ARCH_64')
     sources += files('iavf_rxtx_vec_neon.c')
 endif
Signature · 1 line
2.43.0
author
#10 lore raw

PATCH v3 2/3 net/iavf: add NEON-optimised Tx burst function

In reply to Jay Wang

This patch adds the NEON-optimised Tx burst function for Intel IAVF driver on AArch64.

Signed-off-by Jay Wang <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   1 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  15 ++-
 drivers/net/intel/iavf/iavf_rxtx.h          |   2 -
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 120 ++++++++++++++++++++
 4 files changed, 133 insertions(+), 5 deletions(-)
drivers/net/intel/iavf/iavf.h +1 -0
index e4936f3566..3e71d345a9 100644
@@ -356,6 +356,7 @@ enum iavf_rx_func_type {
 enum iavf_tx_func_type {
 	IAVF_TX_DISABLED,
 	IAVF_TX_DEFAULT,
+	IAVF_TX_NEON,
 	IAVF_TX_AVX2,
 	IAVF_TX_AVX2_OFFLOAD,
 	IAVF_TX_AVX512,
drivers/net/intel/iavf/iavf_rxtx.c +12 -3
index 15566a0e18..645bc5ccf6 100644
@@ -3662,6 +3662,15 @@ static const struct ci_tx_path_info iavf_tx_path_infos[] = {
 		}
 	},
 #endif
+#elif defined(RTE_ARCH_ARM64)
+	[IAVF_TX_NEON] = {
+		.pkt_burst = iavf_xmit_pkts_vec,
+		.info = "Vector Neon",
+		.features = {
+			.tx_offloads = IAVF_TX_VECTOR_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128
+		}
+	},
 #endif
 };
 

@@ -3878,7 +3887,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	int mbuf_check = adapter->devargs.mbuf_check;
 	int no_poll_on_link_down = adapter->devargs.no_poll_on_link_down;
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	struct ci_tx_queue *txq;
 	int i;
 	const struct ci_tx_path_features *selected_features;

@@ -3892,7 +3901,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 	if (dev->data->dev_started)
 		goto out;
 
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	if (iavf_tx_vec_dev_check(dev) != -1)
 		req_features.simd_width = iavf_get_max_simd_bitwidth();
 

@@ -3915,7 +3924,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
 						IAVF_TX_DEFAULT);
 
 out:
-#ifdef RTE_ARCH_X86
+#if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
 	selected_features = &iavf_tx_path_infos[adapter->tx_func_type].features;
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
drivers/net/intel/iavf/iavf_rxtx.h +0 -2
index 80b06518b0..8b8e55e66f 100644
@@ -558,8 +558,6 @@ uint16_t iavf_recv_scattered_pkts_vec(void *rx_queue,
 uint16_t iavf_recv_scattered_pkts_vec_flex_rxd(void *rx_queue,
 					       struct rte_mbuf **rx_pkts,
 					       uint16_t nb_pkts);
-uint16_t iavf_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
-				  uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2(void *rx_queue, struct rte_mbuf **rx_pkts,
 				 uint16_t nb_pkts);
 uint16_t iavf_recv_pkts_vec_avx2_offload(void *rx_queue, struct rte_mbuf **rx_pkts,
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +120 -0

132 lines not on this page · raw message

Signature · 1 line
2.43.0
author
#11 lore raw

PATCH v3 3/3 net/iavf: add NEON support for Rx flex desc

In reply to Jay Wang

This patch adds the NEON-optimised Rx paths to process receive flex descriptor.

The single-core testpmd benchmarking was conducted by Andrew Bailey on an Arm Grace system with an Intel E810 100G NIC. The test was run with txq/rxq=1 and 2048 descriptors. The throughput performance uplift is shown in the following table.

| buffer sz (B) | prev (MPPS) | optimised (MPPS) | uplift |
| ------------- | ----------- | ---------------- | ------ |
| 64            | 62.241      | 86.010           | 38.2%  |
| 128           | 60.813      | 81.453           | 33.9%  |
| 256           | 23.730      | 24.633           | 3.8%   |
| 512           | 10.268      | 10.380           | 1.1%   |
| 1024          | 4.588       | 4.628            | 0.9%   |
| 1518          | 4.601       | 4.669            | 1.5%   |
Signed-off-by Jay Wang <[email protected]>
Tested-by Andrew Bailey <[email protected]>
---
 drivers/net/intel/iavf/iavf.h               |   2 +
 drivers/net/intel/iavf/iavf_rxtx.c          |  23 +-
 drivers/net/intel/iavf/iavf_rxtx_vec_neon.c | 523 +++++++++++++++++++-
 3 files changed, 542 insertions(+), 6 deletions(-)
drivers/net/intel/iavf/iavf.h +2 -0
index 3e71d345a9..360d728f3a 100644
@@ -335,6 +335,8 @@ enum iavf_rx_func_type {
 	IAVF_RX_BULK_ALLOC_FLEX_RXD,
 	IAVF_RX_NEON,
 	IAVF_RX_NEON_SCATTERED,
+	IAVF_RX_NEON_FLEX_RXD,
+	IAVF_RX_NEON_SCATTERED_FLEX_RXD,
 	IAVF_RX_AVX2,
 	IAVF_RX_AVX2_SCATTERED,
 	IAVF_RX_AVX2_OFFLOAD,
drivers/net/intel/iavf/iavf_rxtx.c +22 -1
index 645bc5ccf6..8e711950ff 100644
@@ -713,7 +713,7 @@ iavf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 		ad->rx_bulk_alloc_allowed = false;
 	}
 
-#if defined RTE_ARCH_X86 || defined RTE_ARCH_ARM
+#if defined RTE_ARCH_X86 || defined RTE_ARCH_ARM64
 	/* check vector conflict */
 	if (ci_rxq_vec_capable(rxq->nb_rx_desc, rxq->rx_free_thresh) &&
 			iavf_rxq_vec_setup(rxq)) {

@@ -3571,6 +3571,27 @@ static const struct ci_rx_path_info iavf_rx_path_infos[] = {
 			.bulk_alloc = true
 		}
 	},
+	[IAVF_RX_NEON_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_pkts_vec_flex_rxd,
+		.info = "Vector Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS,
+			.simd_width = RTE_VECT_SIMD_128,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
+	[IAVF_RX_NEON_SCATTERED_FLEX_RXD] = {
+		.pkt_burst = iavf_recv_scattered_pkts_vec_flex_rxd,
+		.info = "Vector Scattered Neon Flex",
+		.features = {
+			.rx_offloads = IAVF_RX_VECTOR_FLEX_OFFLOADS | RTE_ETH_RX_OFFLOAD_SCATTER,
+			.simd_width = RTE_VECT_SIMD_128,
+			.scattered = true,
+			.flex_desc = true,
+			.bulk_alloc = true
+		}
+	},
 #endif
 };
drivers/net/intel/iavf/iavf_rxtx_vec_neon.c +518 -5

558 lines not on this page · raw message

Signature · 1 line
2.43.0
#12 lore raw
In reply to Jay Wang
Quoted 6 lines · Jay Wang

- Add the scattered Rx burst function for 32B legacy descriptor
- Add the NEON-optimised Tx burst function
- Add the NEON-optimised Rx burst functions for flexible descriptor

Single-core benchmark result on a Grace server with an Intel E810:

Quoted 10 lines
| buffer sz (B) | prev (MPPS) | optimised (MPPS) | uplift |
| ------------- | ----------- | ---------------- | ------ |
| 64            | 62.241      | 86.010           | 38.2%  |
| 128           | 60.813      | 81.453           | 33.9%  |
| 256           | 23.730      | 24.633           | 3.8%   |
| 512           | 10.268      | 10.380           | 1.1%   |
| 1024          | 4.588       | 4.628            | 0.9%   |
| 1518          | 4.601       | 4.669            | 1.5%   |

Series applied to dpdk-next-net-intel

Thanks,
/Bruce