Skip to contentSkip to conversation

Vectorize packed ring RX path with NEON

RFC In review Joyce Kong 6 participants 26 messages Sep 11, 2020 – Jan 11, 2021

Subsystems (2)
lore
Share & tools
lore

Joyce Kong

author
#0 lore raw

cover Vectorize packed ring RX path with NEON

This patch set introduced vectorized RX path for packed ring with NEON.

Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path

 doc/guides/nics/virtio.rst                   |   4 +-
 drivers/net/virtio/meson.build               |   2 +
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.c      |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h      | 300 +++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 264 +---------------
 drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +
 9 files changed, 566 insertions(+), 271 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
Signature · 1 line
2.28.0
author
#1 lore raw

RFC 1/3 net/virtio: move AVX based Rx and Tx code to separate file

Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.

---
 drivers/net/virtio/meson.build              |   1 +
 drivers/net/virtio/virtio_rxtx_packed.c     |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h     | 284 ++++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +-----------------
 4 files changed, 323 insertions(+), 263 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
drivers/net/virtio/meson.build +1 -0
index 3fd6051f4..e1851b0a6 100644
@@ -5,6 +5,7 @@ sources += files('virtio_ethdev.c',
 	'virtio_pci.c',
 	'virtio_rxtx.c',
 	'virtio_rxtx_simple.c',
+	'virtio_rxtx_packed.c',
 	'virtqueue.c')
 deps += ['kvargs', 'bus_pci']
 
drivers/net/virtio/virtio_rxtx_packed.c new file +37 -0
new file mode 100644
index 000000000..e614e19fc
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_net.h>
+
+#include "virtio_logs.h"
+#include "virtio_ethdev.h"
+#include "virtio_pci.h"
+#include "virtio_rxtx_packed.h"
+#include "virtqueue.h"
+
+/* Stub for linkage when arch specific implementation is not available */
+__rte_weak uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+			struct rte_mbuf **tx_pkts __rte_unused,
+			uint16_t nb_pkts __rte_unused)
+{
+	rte_panic("Wrong weak function linked by linker\n");
+	return 0;
+}
+
+/* Stub for linkage when arch specific implementation is not available */
+__rte_weak uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+			    struct rte_mbuf **rx_pkts __rte_unused,
+			    uint16_t nb_pkts __rte_unused)
+{
+	rte_panic("Wrong weak function linked by linker\n");
+	return 0;
+}
drivers/net/virtio/virtio_rxtx_packed.h new file +284 -0

284 lines not on this page · raw message

drivers/net/virtio/virtio_rxtx_packed_avx.c +1 -263

290 lines not on this page · raw message

Signature · 1 line
2.28.0
author
#2 lore raw

RFC 2/3 net/virtio: add vectorized packed ring Rx NEON path

In reply to Joyce Kong

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/meson.build               |   1 +
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.h      |  16 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++++++++
 4 files changed, 224 insertions(+), 2 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
drivers/net/virtio/meson.build +1 -0
index e1851b0a6..5af633686 100644
@@ -34,6 +34,7 @@ elif arch_subdir == 'ppc'
 	sources += files('virtio_rxtx_simple_altivec.c')
 elif arch_subdir == 'arm' and host_machine.cpu_family().startswith('aarch64')
 	sources += files('virtio_rxtx_simple_neon.c')
+	sources += files('virtio_rxtx_packed_neon.c')
 endif
 
 if is_linux
drivers/net/virtio/virtio_rxtx.c +5 -2
index f915b8a2c..1deb77569 100644
@@ -2020,7 +2020,8 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	return nb_tx;
 }
 
-#ifndef CC_AVX512_SUPPORT
+#if !defined(CC_AVX512_SUPPORT) && !defined(RTE_ARCH_ARM) && \
+	!defined(RTE_ARCH_ARM64)
 uint16_t
 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
 			    struct rte_mbuf **rx_pkts __rte_unused,
@@ -2028,7 +2029,9 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
 {
 	return 0;
 }
+#endif
 
+#if !defined(CC_AVX512_SUPPORT)
 uint16_t
 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
 			    struct rte_mbuf **tx_pkts __rte_unused,
@@ -2036,4 +2039,4 @@ virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
 {
 	return 0;
 }
-#endif /* ifndef CC_AVX512_SUPPORT */
+#endif
drivers/net/virtio/virtio_rxtx_packed.h +16 -0
index b2447843b..fd2d6baa5 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#if defined(AVX512_SUPPORT)
 /* flag bits offset in packed ring desc higher 64bits */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64)
+/* flag bits offset in packed ring desc 32bits */
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,17 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#if defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64)
+/* The cache line size on different aarh64 platforms are
+ * different, so put a four batch size here to match with
+ * the minimum cache line size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
+
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.c new file +202 -0

202 lines not on this page · raw message

Signature · 1 line
2.28.0
author
#3 lore raw

RFC 3/3 net/virtio: add election for packed vector Rx NEON path

In reply to Joyce Kong

Add NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.

Packed ring vectorized neon path need:
NEON is supported by compiler and host VERSION_1 and IN_ORDER features are negotiated mergeable feature is not negotiated LRO offloading is disabled

---
 doc/guides/nics/virtio.rst              |  4 ++--
 drivers/net/virtio/virtio_ethdev.c      | 19 +++++++++++++++----
 drivers/net/virtio/virtio_user_ethdev.c |  2 ++
 3 files changed, 19 insertions(+), 6 deletions(-)
doc/guides/nics/virtio.rst +2 -2
index 0daf25b22..fe9586699 100644
@@ -483,8 +483,8 @@ according to below configuration:
 #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and
    Rx mergeable is not negotiated, this path will be selected.
 #. Packed virtqueue vectorized Rx path: If building and running environment support
-   AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated &&
-   TCP_LRO Rx offloading is disabled && vectorized option enabled,
+   (AVX512 || ARCH_ARM || ARCH_ARM64) && in-order feature is negotiated && Rx mergeable
+   is not negotiated && TCP_LRO Rx offloading is disabled && vectorized option enabled,
    this path will be selected.
 #. Packed virtqueue vectorized Tx path: If building and running environment support
    AVX512 && in-order feature is negotiated && vectorized option enabled,
drivers/net/virtio/virtio_ethdev.c +15 -4
index dc0093bdf..b36ea98cf 100644
@@ -1958,12 +1958,14 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
 		if (!vtpci_packed_queue(hw)) {
 			hw->use_vec_rx = 1;
 		} else {
-#if !defined(CC_AVX512_SUPPORT)
-			PMD_DRV_LOG(INFO,
-				"building environment do not support packed ring vectorized");
-#else
+#if defined(CC_AVX512_SUPPORT)
 			hw->use_vec_rx = 1;
 			hw->use_vec_tx = 1;
+#elif defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64)
+			hw->use_vec_rx = 1;
+#else
+			PMD_DRV_LOG(INFO,
+				"building environment do not support packed ring vectorized");
 #endif
 		}
 	}
@@ -2311,6 +2313,15 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 			hw->use_vec_rx = 0;
 			hw->use_vec_tx = 0;
 		}
+#elif defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64)
+		if (hw->use_vec_rx &&
+		    (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
+		     !vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+		     !vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+			PMD_DRV_LOG(INFO,
+				"disabled packed ring vectorized path for requirements not met");
+			hw->use_vec_rx = 0;
+		}
 #else
 		hw->use_vec_rx = 0;
 		hw->use_vec_tx = 0;
drivers/net/virtio/virtio_user_ethdev.c +2 -0
index 6003f6d50..1cfeb388f 100644
@@ -766,6 +766,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
 #if defined(CC_AVX512_SUPPORT)
 			hw->use_vec_rx = 1;
 			hw->use_vec_tx = 1;
+#elif defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64)
+			hw->use_vec_rx = 1;
 #else
 			PMD_INIT_LOG(INFO,
 				"building environment do not support packed ring vectorized");
Signature · 1 line
2.28.0
#4 lore raw
In reply to Joyce Kong

Hi Joyce,

Quoted 2 lines · Joyce Kong

This patch set introduced vectorized RX path for packed ring with NEON.

Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.

Could you please send a new revision with the build issues reported by Intel CI fixed?

http://mails.dpdk.org/archives/test-report/2020-September/152501.html

Thanks,
Maxime

Quoted 19 lines

Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path

 doc/guides/nics/virtio.rst                   |   4 +-
 drivers/net/virtio/meson.build               |   2 +
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.c      |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h      | 300 +++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 264 +---------------
 drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +
 9 files changed, 566 insertions(+), 271 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
author
#5 lore raw

Hi Maxime,

Quoted 19 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEON

Hi Joyce,

Quoted 2 lines · Joyce Kong

This patch set introduced vectorized RX path for packed ring with NEON.

Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.

Quoted 8 lines

Could you please send a new revision with the build issues reported by Intel CI fixed?

http://mails.dpdk.org/archives/test-report/2020-September/152501.html

Thanks,
Maxime

Thanks for your comments, and I will send the new revision soon.

Quoted 20 lines
Quoted 20 lines

Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path

 doc/guides/nics/virtio.rst                   |   4 +-
 drivers/net/virtio/meson.build               |   2 +
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.c      |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h      | 300 +++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 264 +---------------
drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +
 9 files changed, 566 insertions(+), 271 deletions(-)  create mode
100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
#6 lore raw
In reply to Maxime Coquelin
Quoted 12 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEON

Hi Joyce,

Quoted 7 lines
Quoted 2 lines · Joyce Kong

This patch set introduced vectorized RX path for packed ring with NEON.

Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.

This patch doesn’t target 20.11. New version will be sent out but not in short time. You can skip this one for 20.11 release.

Thanks.

Quoted 29 lines

Could you please send a new revision with the build issues reported by Intel CI fixed?

http://mails.dpdk.org/archives/test-report/2020-September/152501.html

Thanks,
Maxime

Quoted 19 lines

Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path

 doc/guides/nics/virtio.rst                   |   4 +-
 drivers/net/virtio/meson.build               |   2 +
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.c      |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h      | 300 +++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 264 +---------------
drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +
 9 files changed, 566 insertions(+), 271 deletions(-)  create mode
100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
#7 lore raw
Quoted 13 lines · Ruifeng Wang
Quoted 13 lines

-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEON

Hi Joyce,

On 9/11/20 2:09 PM, Joyce Kong wrote:

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines

This patch set introduced vectorized RX path for packed ring with NEON.

Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.

This patch doesn’t target 20.11. New version will be sent out but not in short time. You can skip this one for 20.11 release.

Thanks Ruifeng for the update.
I'll mark the series as postponed then.

Maxime

Quoted 31 lines

Thanks.

Quoted 29 lines

Could you please send a new revision with the build issues reported by Intel CI fixed?

http://mails.dpdk.org/archives/test-report/2020-September/152501.html

Thanks,
Maxime

Quoted 19 lines

Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path

 doc/guides/nics/virtio.rst                   |   4 +-
 drivers/net/virtio/meson.build               |   2 +
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   7 +-
 drivers/net/virtio/virtio_rxtx_packed.c      |  37 +++
 drivers/net/virtio/virtio_rxtx_packed.h      | 300 +++++++++++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 264 +---------------
drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +
 9 files changed, 566 insertions(+), 271 deletions(-)  create mode
100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
author
#8 lore raw

cover Vectorize packed ring RX/TX path with NEON

In reply to Joyce Kong

This patch set introduces vectorized RX/TX path for packed ring with NEON intrinsics.

With this patch set, PVP case has 1.5% perf uplift for the packed vectorized NEON path compared with the non-vector packed path, under 0.001% acceptable loss with 2 cores on vhost side and 1 core on virtio side.

Joyce Kong (4):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add vectorized packed ring Tx NEON path net/virtio: add election for packed vector NEON path

 doc/guides/nics/virtio.rst                   |   6 +-
 drivers/net/virtio/meson.build               |   5 +-
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   6 +-
 drivers/net/virtio/virtio_rxtx_packed.c      | 139 ++++
 drivers/net/virtio/virtio_rxtx_packed.h      | 317 ++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 626 -------------------
 drivers/net/virtio/virtio_rxtx_packed_avx.h  | 239 +++++++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 293 +++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +-
 10 files changed, 1012 insertions(+), 640 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
Signature · 1 line
2.28.0
author
#9 lore raw

PATCH 1/4 net/virtio: move AVX based Rx and Tx code to separate file

Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.

---
 drivers/net/virtio/meson.build              |   4 +-
 drivers/net/virtio/virtio_rxtx.c            |   6 +-
 drivers/net/virtio/virtio_rxtx_packed.c     | 137 +++++
 drivers/net/virtio/virtio_rxtx_packed.h     | 298 ++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 --------------------
 drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 ++++++++
 6 files changed, 678 insertions(+), 632 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h
drivers/net/virtio/meson.build +2 -2
index eaed46373..01b8de6d4 100644
@@ -13,12 +13,12 @@ if arch_subdir == 'x86'
 		if cc.has_argument('-mavx512f') and cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
 			cflags += ['-DCC_AVX512_SUPPORT']
 			virtio_avx512_lib = static_library('virtio_avx512_lib',
-					      'virtio_rxtx_packed_avx.c',
+					      'virtio_rxtx_packed.c',
 					      dependencies: [static_rte_ethdev,
 						static_rte_kvargs, static_rte_bus_pci],
 					      include_directories: includes,
 					      c_args: [cflags, '-mavx512f', '-mavx512bw', '-mavx512vl'])
-			objs += virtio_avx512_lib.extract_objects('virtio_rxtx_packed_avx.c')
+			objs += virtio_avx512_lib.extract_objects('virtio_rxtx_packed.c')
 			if (toolchain == 'gcc' and cc.version().version_compare('>=8.3.0'))
 				cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
 			elif (toolchain == 'clang' and cc.version().version_compare('>=3.7.0'))
drivers/net/virtio/virtio_rxtx.c +2 -4
index 77934e8c5..622d4bf20 100644
@@ -2025,8 +2025,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,
 	return nb_tx;
 }
 
-#ifndef CC_AVX512_SUPPORT
-uint16_t
+__rte_weak uint16_t
 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
 			    struct rte_mbuf **rx_pkts __rte_unused,
 			    uint16_t nb_pkts __rte_unused)
@@ -2034,11 +2033,10 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
 	return 0;
 }
 
-uint16_t
+__rte_weak uint16_t
 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
 			    struct rte_mbuf **tx_pkts __rte_unused,
 			    uint16_t nb_pkts __rte_unused)
 {
 	return 0;
 }
-#endif /* ifndef CC_AVX512_SUPPORT */
drivers/net/virtio/virtio_rxtx_packed.c new file +137 -0

137 lines not on this page · raw message

drivers/net/virtio/virtio_rxtx_packed.h new file +298 -0

298 lines not on this page · raw message

drivers/net/virtio/virtio_rxtx_packed_avx.c deleted +0 -626

626 lines not on this page · raw message

drivers/net/virtio/virtio_rxtx_packed_avx.h new file +239 -0

239 lines not on this page · raw message

Signature · 1 line
2.28.0
Reviewed-by on v1 1/4
#10 lore raw

Re: [dpdk-dev] [PATCH v1 1/4] net/virtio: move AVX based Rx and Tx code to separate file

Quoted 5 lines · Joyce Kong

Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.

Quoted 13 lines
---
 drivers/net/virtio/meson.build              |   4 +-
 drivers/net/virtio/virtio_rxtx.c            |   6 +-
 drivers/net/virtio/virtio_rxtx_packed.c     | 137 +++++
 drivers/net/virtio/virtio_rxtx_packed.h     | 298 ++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 --------------------
 drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 ++++++++
 6 files changed, 678 insertions(+), 632 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h

Thanks,
Maxime

author
#11 lore raw

PATCH 2/4 net/virtio: add vectorized packed ring Rx NEON path

In reply to Joyce Kong

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |  15 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +15 -0
index b0b1d63ec..8f5198ad7 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#ifdef CC_AVX512_SUPPORT
 /* flag bits offset in packed ring desc higher 64bits */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM)
+/* flag bits offset in packed ring desc from ID */
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#ifdef RTE_ARCH_ARM
+/* The cache line size on different Arm platforms are different, so
+ * put a four batch size here to match with the minimum cache line
+ * size and accommodate NEON register size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +150 -0

150 lines not on this page · raw message

Signature · 1 line
2.28.0
#12 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

Quoted 78 lines · Joyce Kong

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |  15 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +15 -0
index b0b1d63ec..8f5198ad7 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#ifdef CC_AVX512_SUPPORT
 /* flag bits offset in packed ring desc higher 64bits */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM)
+/* flag bits offset in packed ring desc from ID */
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#ifdef RTE_ARCH_ARM
+/* The cache line size on different Arm platforms are different, so
+ * put a four batch size here to match with the minimum cache line
+ * size and accommodate NEON register size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24 -0
new file mode 100644
index 000000000..fb1e49909
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Arm Corporation
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_net.h>
+#include <rte_vect.h>
+
+#include "virtio_ethdev.h"
+#include "virtio_pci.h"
+#include "virtio_rxtx_packed.h"
+#include "virtqueue.h"
+
+static inline uint16_t
+virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
+				   struct rte_mbuf **rx_pkts)
+{
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;

drivers/net/virtio/virtio_rxtx_packed_neon.h line 34

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10 -0
+	uint16_t head_size = hw->vtnet_hdr_size;
+	uint16_t id = vq->vq_used_cons_idx;
+	struct vring_packed_desc *p_desc;
+	uint16_t i;
+
+	if (id & PACKED_BATCH_MASK)
+		return -1;
+
+	if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
+		return -1;

This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.

Quoted 44 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +44 -0
+	/* Map packed descriptor to mbuf fields. */
+	uint8x16_t shuf_msk1 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		0, 1,			/* octet 1~0, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		0, 1,			/* octet 1~0, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	uint8x16_t shuf_msk2 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		8, 9,			/* octet 9~8, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		8, 9,			/* octet 9~8, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	/* Subtract the header length. */
+	uint16x8_t len_adjust = {
+		0, 0,		/* ignore pkt_type field */
+		head_size,	/* sub head_size on pkt_len */
+		0,		/* ignore high 16 bits of pkt_len */
+		head_size,	/* sub head_size on data_len */
+		0, 0, 0		/* ignore non-length fields */
+	};
+
+	uint64x2_t desc[PACKED_BATCH_SIZE / 2];
+	uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2];
+	uint64x2_t pkt_mb[PACKED_BATCH_SIZE];
+
+	p_desc = &vq->vq_packed.ring.desc[id];
+	/* Load high 64 bits of packed descriptor 0,1. */
+	desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1];
+	/* Load high 64 bits of packed descriptor 2,3. */
+	desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1];
+
+	/* Only care avail/used bits. */
+	uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK);
+	/* Extract high 32 bits of packed descriptor (id, flags). */
+	uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]),
+				vreinterpretq_u32_u64(desc[1]));
+	uint32x4_t v_flag = vandq_u32(v_desc, v_mask);

drivers/net/virtio/virtio_rxtx_packed_neon.h line 89

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10 -0
+
+	uint32x4_t v_used_flag = vdupq_n_u32(0);
+	if (vq->vq_packed.used_wrap_counter)
+		v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK);
+
+	poly128_t desc_stats = vreinterpretq_p128_u32(~vceqq_u32(v_flag, v_used_flag));
+
+	/* Check all descs are used. */
+	if (desc_stats)
+		return -1;

Same here. You should return 0 here as the queue is full.

Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61 -0
+
+	/* Load 2 mbuf pointers per time. */
+	mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]);
+	vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]);
+
+	mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]);
+	vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]);
+
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
+	pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk2));
+
+	pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
+	pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
+	pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
+	pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
+
+	vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]);
+	vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]);
+	vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]);
+	vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]);
+
+	if (hw->has_rx_offload) {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			char *addr = (char *)rx_pkts[i]->buf_addr +
+				RTE_PKTMBUF_HEADROOM - head_size;
+			virtio_vec_rx_offload(rx_pkts[i],
+					(struct virtio_net_hdr *)addr);
+		}
+	}
+
+	virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
+			rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
+			rx_pkts[3]->pkt_len);
+
+	vq->vq_free_cnt += PACKED_BATCH_SIZE;
+
+	vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
+	if (vq->vq_used_cons_idx >= vq->vq_nentries) {
+		vq->vq_used_cons_idx -= vq->vq_nentries;
+		vq->vq_packed.used_wrap_counter ^= 1;
+	}
+
+	return 0;
+}
#13 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

Quoted 143 lines · Maxime Coquelin
Quoted 78 lines · Joyce Kong

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |  15 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +15 -0
index b0b1d63ec..8f5198ad7 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#ifdef CC_AVX512_SUPPORT
 /* flag bits offset in packed ring desc higher 64bits */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM)
+/* flag bits offset in packed ring desc from ID */
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#ifdef RTE_ARCH_ARM
+/* The cache line size on different Arm platforms are different, so
+ * put a four batch size here to match with the minimum cache line
+ * size and accommodate NEON register size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24 -0
new file mode 100644
index 000000000..fb1e49909
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Arm Corporation
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_net.h>
+#include <rte_vect.h>
+
+#include "virtio_ethdev.h"
+#include "virtio_pci.h"
+#include "virtio_rxtx_packed.h"
+#include "virtqueue.h"
+
+static inline uint16_t
+virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
+				   struct rte_mbuf **rx_pkts)
+{
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;

drivers/net/virtio/virtio_rxtx_packed_neon.h line 34

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10 -0
+	uint16_t head_size = hw->vtnet_hdr_size;
+	uint16_t id = vq->vq_used_cons_idx;
+	struct vring_packed_desc *p_desc;
+	uint16_t i;
+
+	if (id & PACKED_BATCH_MASK)
+		return -1;
+
+	if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
+		return -1;

This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.

Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +48 -0
+	/* Map packed descriptor to mbuf fields. */
+	uint8x16_t shuf_msk1 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		0, 1,			/* octet 1~0, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		0, 1,			/* octet 1~0, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	uint8x16_t shuf_msk2 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		8, 9,			/* octet 9~8, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		8, 9,			/* octet 9~8, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	/* Subtract the header length. */
+	uint16x8_t len_adjust = {
+		0, 0,		/* ignore pkt_type field */
+		head_size,	/* sub head_size on pkt_len */
+		0,		/* ignore high 16 bits of pkt_len */
+		head_size,	/* sub head_size on data_len */
+		0, 0, 0		/* ignore non-length fields */
+	};
+
+	uint64x2_t desc[PACKED_BATCH_SIZE / 2];
+	uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2];
+	uint64x2_t pkt_mb[PACKED_BATCH_SIZE];
+
+	p_desc = &vq->vq_packed.ring.desc[id];
+	/* Load high 64 bits of packed descriptor 0,1. */
+	desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1];
+	/* Load high 64 bits of packed descriptor 2,3. */
+	desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1];
+
+	/* Only care avail/used bits. */
+	uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK);
+	/* Extract high 32 bits of packed descriptor (id, flags). */
+	uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]),
+				vreinterpretq_u32_u64(desc[1]));
+	uint32x4_t v_flag = vandq_u32(v_desc, v_mask);
+
+	uint32x4_t v_used_flag = vdupq_n_u32(0);
+	if (vq->vq_packed.used_wrap_counter)
+		v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK);
Quoted 8 lines

drivers/net/virtio/virtio_rxtx_packed_neon.h line 89

Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6 -0
+
+	poly128_t desc_stats = vreinterpretq_p128_u32(~vceqq_u32(v_flag, v_used_flag));
+
+	/* Check all descs are used. */
+	if (desc_stats)
+		return -1;

Same here. You should return 0 here as the queue is full.

Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.

Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure

Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61 -0
+
+	/* Load 2 mbuf pointers per time. */
+	mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]);
+	vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]);
+
+	mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]);
+	vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]);
+
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
+	pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk2));
+
+	pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
+	pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
+	pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
+	pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
+
+	vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]);
+	vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]);
+	vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]);
+	vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]);
+
+	if (hw->has_rx_offload) {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			char *addr = (char *)rx_pkts[i]->buf_addr +
+				RTE_PKTMBUF_HEADROOM - head_size;
+			virtio_vec_rx_offload(rx_pkts[i],
+					(struct virtio_net_hdr *)addr);
+		}
+	}
+
+	virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
+			rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
+			rx_pkts[3]->pkt_len);
+
+	vq->vq_free_cnt += PACKED_BATCH_SIZE;
+
+	vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
+	if (vq->vq_used_cons_idx >= vq->vq_nentries) {
+		vq->vq_used_cons_idx -= vq->vq_nentries;
+		vq->vq_packed.used_wrap_counter ^= 1;
+	}
+
+	return 0;
+}
#14 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

Quoted 155 lines · Maxime Coquelin
Quoted 143 lines · Maxime Coquelin
Quoted 78 lines · Joyce Kong

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |  15 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +15 -0
index b0b1d63ec..8f5198ad7 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#ifdef CC_AVX512_SUPPORT
 /* flag bits offset in packed ring desc higher 64bits */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM)
+/* flag bits offset in packed ring desc from ID */
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#ifdef RTE_ARCH_ARM
+/* The cache line size on different Arm platforms are different, so
+ * put a four batch size here to match with the minimum cache line
+ * size and accommodate NEON register size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24 -0
new file mode 100644
index 000000000..fb1e49909
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Arm Corporation
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_net.h>
+#include <rte_vect.h>
+
+#include "virtio_ethdev.h"
+#include "virtio_pci.h"
+#include "virtio_rxtx_packed.h"
+#include "virtqueue.h"
+
+static inline uint16_t
+virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
+				   struct rte_mbuf **rx_pkts)
+{
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;

drivers/net/virtio/virtio_rxtx_packed_neon.h line 34

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10 -0
+	uint16_t head_size = hw->vtnet_hdr_size;
+	uint16_t id = vq->vq_used_cons_idx;
+	struct vring_packed_desc *p_desc;
+	uint16_t i;
+
+	if (id & PACKED_BATCH_MASK)
+		return -1;
+
+	if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
+		return -1;

This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.

Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +48 -0
+	/* Map packed descriptor to mbuf fields. */
+	uint8x16_t shuf_msk1 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		0, 1,			/* octet 1~0, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		0, 1,			/* octet 1~0, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	uint8x16_t shuf_msk2 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		8, 9,			/* octet 9~8, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out */
+		8, 9,			/* octet 9~8, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	/* Subtract the header length. */
+	uint16x8_t len_adjust = {
+		0, 0,		/* ignore pkt_type field */
+		head_size,	/* sub head_size on pkt_len */
+		0,		/* ignore high 16 bits of pkt_len */
+		head_size,	/* sub head_size on data_len */
+		0, 0, 0		/* ignore non-length fields */
+	};
+
+	uint64x2_t desc[PACKED_BATCH_SIZE / 2];
+	uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2];
+	uint64x2_t pkt_mb[PACKED_BATCH_SIZE];
+
+	p_desc = &vq->vq_packed.ring.desc[id];
+	/* Load high 64 bits of packed descriptor 0,1. */
+	desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1];
+	/* Load high 64 bits of packed descriptor 2,3. */
+	desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1];
+
+	/* Only care avail/used bits. */
+	uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK);
+	/* Extract high 32 bits of packed descriptor (id, flags). */
+	uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]),
+				vreinterpretq_u32_u64(desc[1]));
+	uint32x4_t v_flag = vandq_u32(v_desc, v_mask);
+
+	uint32x4_t v_used_flag = vdupq_n_u32(0);
+	if (vq->vq_packed.used_wrap_counter)
+		v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK);
Quoted 8 lines

drivers/net/virtio/virtio_rxtx_packed_neon.h line 89

Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6 -0
+
+	poly128_t desc_stats = vreinterpretq_p128_u32(~vceqq_u32(v_flag, v_used_flag));
+
+	/* Check all descs are used. */
+	if (desc_stats)
+		return -1;

Same here. You should return 0 here as the queue is full.

Quoted 6 lines

Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.

Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure

Joyce, are you fine if I do the cange while applying?
I have a big series that will conflicts with your patch set, so I'd like to have yours merged ASAP so I can start the rebase.

Thanks,
Maxime

Quoted 63 lines
Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61 -0
+
+	/* Load 2 mbuf pointers per time. */
+	mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]);
+	vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]);
+
+	mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]);
+	vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]);
+
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
+	pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk2));
+
+	pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
+	pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
+	pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
+	pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
+
+	vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]);
+	vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]);
+	vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]);
+	vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]);
+
+	if (hw->has_rx_offload) {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			char *addr = (char *)rx_pkts[i]->buf_addr +
+				RTE_PKTMBUF_HEADROOM - head_size;
+			virtio_vec_rx_offload(rx_pkts[i],
+					(struct virtio_net_hdr *)addr);
+		}
+	}
+
+	virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
+			rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
+			rx_pkts[3]->pkt_len);
+
+	vq->vq_free_cnt += PACKED_BATCH_SIZE;
+
+	vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
+	if (vq->vq_used_cons_idx >= vq->vq_nentries) {
+		vq->vq_used_cons_idx -= vq->vq_nentries;
+		vq->vq_packed.used_wrap_counter ^= 1;
+	}
+
+	return 0;
+}
author
#15 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

Quoted 169 lines · Maxime Coquelin
Quoted 36 lines · Maxime Coquelin
Quoted 33 lines · Joyce Kong
Quoted 30 lines

Optimize packed ring Rx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |  15 ++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 150
+++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +6 -0
index b0b1d63ec..8f5198ad7 100644
@@ -19,9 +19,16 @@
 #include "virtqueue.h"
 
 #define BYTE_SIZE 8
+
+#ifdef CC_AVX512_SUPPORT
 /* flag bits offset in packed ring desc higher 64bits */  #define
 FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+#elif defined(RTE_ARCH_ARM)
+/* flag bits offset in packed ring desc from ID */ #define
+FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) #endif
 
Quoted 1 line
Quoted 1 line

drivers/net/virtio/virtio_rxtx_packed.h line 32

Quoted 1 line
drivers/net/virtio/virtio_rxtx_packed.h +0 -0
 #define PACKED_FLAGS_MASK ((0ULL |

VRING_PACKED_DESC_F_AVAIL_USED) << \

Quoted 62 lines
Quoted 62 lines
Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed.h +8 -0
 	FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@
 /* net hdr short size mask */
 #define NET_HDR_MASK 0x3F
 
+#ifdef RTE_ARCH_ARM
+/* The cache line size on different Arm platforms are different, so
+ * put a four batch size here to match with the minimum cache line
+ * size and accommodate NEON register size.
+ */
+#define PACKED_BATCH_SIZE 4
+#else
 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
 	sizeof(struct vring_packed_desc))
+#endif
 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
 
 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +23 -0
new file mode 100644
index 000000000..fb1e49909
@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Arm Corporation  */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <rte_net.h>
+#include <rte_vect.h>
+
+#include "virtio_ethdev.h"
+#include "virtio_pci.h"
+#include "virtio_rxtx_packed.h"
+#include "virtqueue.h"
+
+static inline uint16_t
+virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
+				   struct rte_mbuf **rx_pkts)
+{
+	struct virtqueue *vq = rxvq->vq;
+	struct virtio_hw *hw = vq->hw;

drivers/net/virtio/virtio_rxtx_packed_neon.h line 33

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10 -0
+	uint16_t head_size = hw->vtnet_hdr_size;
+	uint16_t id = vq->vq_used_cons_idx;
+	struct vring_packed_desc *p_desc;
+	uint16_t i;
+
+	if (id & PACKED_BATCH_MASK)
+		return -1;
+
+	if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
+		return -1;

This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.

Quoted 5 lines
Quoted 5 lines

drivers/net/virtio/virtio_rxtx_packed_neon.h

Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5 -0
+	/* Map packed descriptor to mbuf fields. */
+	uint8x16_t shuf_msk1 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		0, 1,			/* octet 1~0, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out

*/

Quoted 9 lines
Quoted 9 lines

drivers/net/virtio/virtio_rxtx_packed_neon.h

Quoted 9 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +9 -0
+		0, 1,			/* octet 1~0, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	uint8x16_t shuf_msk2 = {
+		0xFF, 0xFF, 0xFF, 0xFF, /* pkt_type set as unknown */
+		8, 9,			/* octet 9~8, low 16 bits pkt_len */
+		0xFF, 0xFF,		/* skip high 16 bits of pkt_len, zero out

*/

Quoted 44 lines
Quoted 35 lines
Quoted 35 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +35 -0
+		8, 9,			/* octet 9~8, 16 bits data_len */
+		0xFF, 0xFF,		/* vlan tci set as unknown */
+		0xFF, 0xFF, 0xFF, 0xFF
+	};
+
+	/* Subtract the header length. */
+	uint16x8_t len_adjust = {
+		0, 0,		/* ignore pkt_type field */
+		head_size,	/* sub head_size on pkt_len */
+		0,		/* ignore high 16 bits of pkt_len */
+		head_size,	/* sub head_size on data_len */
+		0, 0, 0		/* ignore non-length fields */
+	};
+
+	uint64x2_t desc[PACKED_BATCH_SIZE / 2];
+	uint64x2x2_t mbp[PACKED_BATCH_SIZE / 2];
+	uint64x2_t pkt_mb[PACKED_BATCH_SIZE];
+
+	p_desc = &vq->vq_packed.ring.desc[id];
+	/* Load high 64 bits of packed descriptor 0,1. */
+	desc[0] = vld2q_u64((uint64_t *)(p_desc)).val[1];
+	/* Load high 64 bits of packed descriptor 2,3. */
+	desc[1] = vld2q_u64((uint64_t *)(p_desc + 2)).val[1];
+
+	/* Only care avail/used bits. */
+	uint32x4_t v_mask = vdupq_n_u32(PACKED_FLAGS_MASK);
+	/* Extract high 32 bits of packed descriptor (id, flags). */
+	uint32x4_t v_desc = vuzp2q_u32(vreinterpretq_u32_u64(desc[0]),
+				vreinterpretq_u32_u64(desc[1]));
+	uint32x4_t v_flag = vandq_u32(v_desc, v_mask);
+
+	uint32x4_t v_used_flag = vdupq_n_u32(0);
+	if (vq->vq_packed.used_wrap_counter)
+		v_used_flag = vdupq_n_u32(PACKED_FLAGS_MASK);
+
Quoted 8 lines

drivers/net/virtio/virtio_rxtx_packed_neon.h line 89

Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6 -0
+	poly128_t desc_stats = vreinterpretq_p128_u32(~vceqq_u32(v_flag,
+v_used_flag));
+
+	/* Check all descs are used. */
+	if (desc_stats)
+		return -1;

Same here. You should return 0 here as the queue is full.

Quoted 6 lines

Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.

Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure

Quoted 7 lines

Joyce, are you fine if I do the cange while applying?
I have a big series that will conflicts with your patch set, so I'd like to have yours merged ASAP so I can start the rebase.

Thanks,
Maxime

Maxime, It's ok if you would do the change while applying.

Thanks,
Joyce

Quoted 63 lines
Quoted 63 lines
Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61 -0
+
+	/* Load 2 mbuf pointers per time. */
+	mbp[0] = vld2q_u64((uint64_t *)&vq->vq_descx[id]);
+	vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0].val[0]);
+
+	mbp[1] = vld2q_u64((uint64_t *)&vq->vq_descx[id + 2]);
+	vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1].val[0]);
+
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
+	pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk2));
+
+	pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
+	pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
+	pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
+	pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
+			vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
+
+	vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1, pkt_mb[0]);
+	vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1, pkt_mb[1]);
+	vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1, pkt_mb[2]);
+	vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1, pkt_mb[3]);
+
+	if (hw->has_rx_offload) {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			char *addr = (char *)rx_pkts[i]->buf_addr +
+				RTE_PKTMBUF_HEADROOM - head_size;
+			virtio_vec_rx_offload(rx_pkts[i],
+					(struct virtio_net_hdr *)addr);
+		}
+	}
+
+	virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
+			rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
+			rx_pkts[3]->pkt_len);
+
+	vq->vq_free_cnt += PACKED_BATCH_SIZE;
+
+	vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
+	if (vq->vq_used_cons_idx >= vq->vq_nentries) {
+		vq->vq_used_cons_idx -= vq->vq_nentries;
+		vq->vq_packed.used_wrap_counter ^= 1;
+	}
+
+	return 0;
+}
#16 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

In reply to Joyce Kong

drivers/net/virtio/virtio_rxtx_packed_neon.h line 110

Quoted 13 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +13 -0
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'

s\'\;

I will fix in next-net but my concern is why this has been not caught by any of our automated builds?

In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/

#17 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

<snip>

Quoted 19 lines · Joyce Kong

drivers/net/virtio/virtio_rxtx_packed_neon.h

Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5 -0
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits

vlan_tci|

drivers/net/virtio/virtio_rxtx_packed_neon.h line 110

Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +8 -0
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'

s\'\;

Quoted 5 lines

I will fix in next-net but my concern is why this has been not caught by any of our automated builds?

In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/

Looking at [1], Travis CI has not run and the UNH CI did not have Arm builds enabled at the time this patch was submitted.

[1] https://patches.dpdk.org/patch/84262/

#18 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

Quoted 21 lines · Honnappa Nagarahalli

<snip>

Quoted 19 lines · Joyce Kong

drivers/net/virtio/virtio_rxtx_packed_neon.h

Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5 -0
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits

vlan_tci|

drivers/net/virtio/virtio_rxtx_packed_neon.h line 110

Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +8 -0
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'

s\'\;

Quoted 8 lines
Quoted 5 lines

I will fix in next-net but my concern is why this has been not caught by any of our automated builds?

In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/

Looking at [1], Travis CI has not run and the UNH CI did not have Arm builds enabled at the time this patch was submitted.

Seem my other email. Travis CI doesn't run 'patch-at-a-time' for execution time reasons.

Quoted 1 line

[1] https://patches.dpdk.org/patch/84262/

#19 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

In reply to Ferruh Yigit
Quoted 18 lines · Ferruh Yigit
Quoted 8 lines · Joyce Kong

+    /**
+     *  Update data length and packet length for descriptor. +     *  structure of pkt_mb:
+     *
--------------------------------------------------------------------
+     *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+     *

Quoted 8 lines

--------------------------------------------------------------------
+     */
+    pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8( +            vreinterpretq_u8_u64(desc[0]), shuf_msk1)); +    pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8( +            vreinterpretq_u8_u64(desc[0]), shuf_msk2)); +    pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8( +            vreinterpretq_u8_u64(desc[1]), shuf_msk1))'

Quoted 8 lines

s\'\;

I will fix in next-net but my concern is why this has been not caught by any of our automated builds?

In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/

Thanks Ferruh for spotting and fixing it.

I think the CI was broken at the time it was submitted, it would be great to have a way to manually trigger the CI again!

That plus me changing laptop recently and not having a full multi-arch build system up and running again made this build issue pass through...

Regards,
Maxime

#20 lore raw

Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path

In reply to Ferruh Yigit
Quoted 17 lines · Ferruh Yigit

drivers/net/virtio/virtio_rxtx_packed_neon.h line 110

Quoted 13 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +13 -0
+	/**
+	 *  Update data length and packet length for descriptor.
+	 *  structure of pkt_mb:
+	 *  --------------------------------------------------------------------
+	 *  |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+	 *  --------------------------------------------------------------------
+	 */
+	pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk1));
+	pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[0]), shuf_msk2));
+	pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
+			vreinterpretq_u8_u64(desc[1]), shuf_msk1))'

s\'\;

Quoted 2 lines

I will fix in next-net but my concern is why this has been not caught by any of our automated builds?

That series was flagged for error with Travis:

http://mails.dpdk.org/archives/test-report/2020-November/167602.html

Unfortunately, the build seems to have been purged (since it's from November). But Travis did flag the build as failing. With github actions we hope to pull the full logs into the email.

Quoted 2 lines

In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/

At least the 0-day robot does not submit each patch for separate build. We did that at first, and the robot's queue reached a week of backlog because the build takes a while. Especially when we get 20+ patch series followed by v2-v4 fixing build errors or compile errors.

author
#21 lore raw

PATCH 3/4 net/virtio: add vectorized packed ring Tx NEON path

In reply to Joyce Kong

Optimize packed ring Tx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |   6 +-
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 143 +++++++++++++++++++
 2 files changed, 148 insertions(+), 1 deletion(-)
drivers/net/virtio/virtio_rxtx_packed.h +5 -1
index 8f5198ad7..016b6fb24 100644
@@ -28,6 +28,8 @@
 /* flag bits offset in packed ring desc from ID */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#define FLAGS_LEN_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
 #endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
@@ -36,13 +38,15 @@
 /* reference count offset in mbuf rearm data */
 #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
 	offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+#ifdef CC_AVX512_SUPPORT
 /* segment number offset in mbuf rearm data */
 #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
 	offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
-
 /* default rearm data */
 #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
 	1ULL << REFCNT_BITS_OFFSET)
+#endif
 
 /* id bits offset in packed ring desc higher 64bits */
 #define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
drivers/net/virtio/virtio_rxtx_packed_neon.h +143 -0

149 lines not on this page · raw message

Signature · 1 line
2.28.0
Reviewed-by on v1 3/4
#22 lore raw

Re: [dpdk-dev] [PATCH v1 3/4] net/virtio: add vectorized packed ring Tx NEON path

Quoted 184 lines · Joyce Kong

Optimize packed ring Tx batch path with NEON instructions.

---
 drivers/net/virtio/virtio_rxtx_packed.h      |   6 +-
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 143 +++++++++++++++++++
 2 files changed, 148 insertions(+), 1 deletion(-)
drivers/net/virtio/virtio_rxtx_packed.h +5 -1
index 8f5198ad7..016b6fb24 100644
@@ -28,6 +28,8 @@
 /* flag bits offset in packed ring desc from ID */
 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
 	offsetof(struct vring_packed_desc, id)) * BYTE_SIZE)
+#define FLAGS_LEN_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+	offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
 #endif
 
 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
@@ -36,13 +38,15 @@
 /* reference count offset in mbuf rearm data */
 #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
 	offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+#ifdef CC_AVX512_SUPPORT
 /* segment number offset in mbuf rearm data */
 #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
 	offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
-
 /* default rearm data */
 #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
 	1ULL << REFCNT_BITS_OFFSET)
+#endif
 
 /* id bits offset in packed ring desc higher 64bits */
 #define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
drivers/net/virtio/virtio_rxtx_packed_neon.h +136 -0
index fb1e49909..041f771ea 100644
@@ -16,6 +16,149 @@
 #include "virtio_rxtx_packed.h"
 #include "virtqueue.h"
 
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+				   struct rte_mbuf **tx_pkts)
+{
+	struct virtqueue *vq = txvq->vq;
+	uint16_t head_size = vq->hw->vtnet_hdr_size;
+	uint16_t idx = vq->vq_avail_idx;
+	struct virtio_net_hdr *hdr;
+	struct vq_desc_extra *dxp;
+	struct vring_packed_desc *p_desc;
+	uint16_t i;
+
+	if (idx & PACKED_BATCH_MASK)
+		return -1;
+
+	if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+		return -1;
+
+	/* Map four refcnt and nb_segs from mbufs to one NEON register. */
+	uint8x16_t ref_seg_msk = {
+		2, 3, 4, 5,
+		10, 11, 12, 13,
+		18, 19, 20, 21,
+		26, 27, 28, 29
+	};
+
+	/* Map four data_off from mbufs to one NEON register. */
+	uint8x8_t data_msk = {
+		0, 1,
+		8, 9,
+		16, 17,
+		24, 25
+	};
+
+	uint16x8_t net_hdr_msk = {
+		0xFFFF, 0xFFFF,
+		0, 0, 0, 0
+	};
+
+	uint16x4_t pkts[PACKED_BATCH_SIZE];
+	uint8x16x2_t mbuf;
+	/* Load four mbufs rearm data. */
+	RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+	pkts[0] = vld1_u16((uint16_t *)&tx_pkts[0]->rearm_data);
+	pkts[1] = vld1_u16((uint16_t *)&tx_pkts[1]->rearm_data);
+	pkts[2] = vld1_u16((uint16_t *)&tx_pkts[2]->rearm_data);
+	pkts[3] = vld1_u16((uint16_t *)&tx_pkts[3]->rearm_data);
+
+	mbuf.val[0] = vreinterpretq_u8_u16(vcombine_u16(pkts[0], pkts[1]));
+	mbuf.val[1] = vreinterpretq_u8_u16(vcombine_u16(pkts[2], pkts[3]));
+
+	/* refcnt = 1 and nb_segs = 1 */
+	uint32x4_t def_ref_seg = vdupq_n_u32(0x10001);
+	/* Check refcnt and nb_segs. */
+	uint32x4_t ref_seg = vreinterpretq_u32_u8(vqtbl2q_u8(mbuf, ref_seg_msk));
+	poly128_t cmp1 = vreinterpretq_p128_u32(~vceqq_u32(ref_seg, def_ref_seg));
+	if (unlikely(cmp1))
+		return -1;
+
+	/* Check headroom is enough. */
+	uint16x4_t head_rooms = vdup_n_u16(head_size);
+	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
+			 offsetof(struct rte_mbuf, rearm_data));
+	uint16x4_t data_offset = vreinterpret_u16_u8(vqtbl2_u8(mbuf, data_msk));
+	uint64x1_t cmp2 = vreinterpret_u64_u16(vclt_u16(data_offset, head_rooms));
+	if (unlikely(vget_lane_u64(cmp2, 0)))
+		return -1;
+
+	virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+		dxp = &vq->vq_descx[idx + i];
+		dxp->ndescs = 1;
+		dxp->cookie = tx_pkts[i];
+	}
+
+	virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+		tx_pkts[i]->data_off -= head_size;
+		tx_pkts[i]->data_len += head_size;
+	}
+
+	uint64x2x2_t desc[PACKED_BATCH_SIZE / 2];
+	uint64x2_t base_addr0 = {
+		VIRTIO_MBUF_ADDR(tx_pkts[0], vq) + tx_pkts[0]->data_off,
+		VIRTIO_MBUF_ADDR(tx_pkts[1], vq) + tx_pkts[1]->data_off
+	};
+	uint64x2_t base_addr1 = {
+		VIRTIO_MBUF_ADDR(tx_pkts[2], vq) + tx_pkts[2]->data_off,
+		VIRTIO_MBUF_ADDR(tx_pkts[3], vq) + tx_pkts[3]->data_off
+	};
+
+	desc[0].val[0] = base_addr0;
+	desc[1].val[0] = base_addr1;
+
+	uint64_t flags = (uint64_t)vq->vq_packed.cached_flags << FLAGS_LEN_BITS_OFFSET;
+	uint64x2_t tx_desc0 = {
+		flags | (uint64_t)idx << ID_BITS_OFFSET | tx_pkts[0]->data_len,
+		flags | (uint64_t)(idx + 1) << ID_BITS_OFFSET | tx_pkts[1]->data_len
+	};
+
+	uint64x2_t tx_desc1 = {
+		flags | (uint64_t)(idx + 2) << ID_BITS_OFFSET | tx_pkts[2]->data_len,
+		flags | (uint64_t)(idx + 3) << ID_BITS_OFFSET | tx_pkts[3]->data_len
+	};
+
+	desc[0].val[1] = tx_desc0;
+	desc[1].val[1] = tx_desc1;
+
+	if (!vq->hw->has_tx_offload) {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
+					struct virtio_net_hdr *, -head_size);
+			/* Clear net hdr. */
+			uint16x8_t v_hdr = vld1q_u16((void *)hdr);
+			vst1q_u16((void *)hdr, vandq_u16(v_hdr, net_hdr_msk));
+		}
+	} else {
+		virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+			hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
+					struct virtio_net_hdr *, -head_size);
+			virtqueue_xmit_offload(hdr, tx_pkts[i], true);
+		}
+	}
+
+	/* Enqueue packet buffers. */
+	p_desc = &vq->vq_packed.ring.desc[idx];
+	vst2q_u64((uint64_t *)p_desc, desc[0]);
+	vst2q_u64((uint64_t *)(p_desc + 2), desc[1]);
+
+	virtio_update_batch_stats(&txvq->stats, tx_pkts[0]->pkt_len,
+			tx_pkts[1]->pkt_len, tx_pkts[2]->pkt_len,
+			tx_pkts[3]->pkt_len);
+
+	vq->vq_avail_idx += PACKED_BATCH_SIZE;
+	vq->vq_free_cnt -= PACKED_BATCH_SIZE;
+
+	if (vq->vq_avail_idx >= vq->vq_nentries) {
+		vq->vq_avail_idx -= vq->vq_nentries;

drivers/net/virtio/virtio_rxtx_packed_neon.h line 164

Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h +7 -0
+		vq->vq_packed.cached_flags ^=
+			VRING_PACKED_DESC_F_AVAIL_USED;
+	}
+
+	return 0;
+}
+
 static inline uint16_t
 virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
 				   struct rte_mbuf **rx_pkts)

Thanks,
Maxime

author
#23 lore raw

PATCH 4/4 net/virtio: add election for packed vector NEON path

In reply to Joyce Kong

Add NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.

Packed ring vectorized neon path need:
NEON is supported by compiler and host VERSION_1 and IN_ORDER features are negotiated mergeable feature is not negotiated LRO offloading is disabled

---
 doc/guides/nics/virtio.rst              |  6 +++---
 drivers/net/virtio/meson.build          |  1 +
 drivers/net/virtio/virtio_ethdev.c      | 19 +++++++++++++++----
 drivers/net/virtio/virtio_rxtx_packed.c |  2 ++
 drivers/net/virtio/virtio_user_ethdev.c |  2 +-
 5 files changed, 22 insertions(+), 8 deletions(-)
doc/guides/nics/virtio.rst +3 -3
index c03c2d0fe..b7be3aca1 100644
@@ -483,11 +483,11 @@ according to below configuration:
 #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and
    Rx mergeable is not negotiated, this path will be selected.
 #. Packed virtqueue vectorized Rx path: If building and running environment support
-   AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated &&
-   TCP_LRO Rx offloading is disabled && vectorized option enabled,
+   (AVX512 || NEON) && in-order feature is negotiated && Rx mergeable
+   is not negotiated && TCP_LRO Rx offloading is disabled && vectorized option enabled,
    this path will be selected.
 #. Packed virtqueue vectorized Tx path: If building and running environment support
-   AVX512 && in-order feature is negotiated && vectorized option enabled,
+   (AVX512 || NEON)  && in-order feature is negotiated && vectorized option enabled,
    this path will be selected.
 
 Rx/Tx callbacks of each Virtio path
drivers/net/virtio/meson.build +1 -0
index 01b8de6d4..738d66746 100644
@@ -32,6 +32,7 @@ if arch_subdir == 'x86'
 elif arch_subdir == 'ppc'
 	sources += files('virtio_rxtx_simple_altivec.c')
 elif arch_subdir == 'arm' and host_machine.cpu_family().startswith('aarch64')
+	sources += files('virtio_rxtx_packed.c')
 	sources += files('virtio_rxtx_simple_neon.c')
 endif
 
drivers/net/virtio/virtio_ethdev.c +15 -4
index 6c233b75b..54a6d6ca9 100644
@@ -1967,12 +1967,12 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
 		if (!vtpci_packed_queue(hw)) {
 			hw->use_vec_rx = 1;
 		} else {
-#if !defined(CC_AVX512_SUPPORT)
-			PMD_DRV_LOG(INFO,
-				"building environment do not support packed ring vectorized");
-#else
+#if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
 			hw->use_vec_rx = 1;
 			hw->use_vec_tx = 1;
+#else
+			PMD_DRV_LOG(INFO,
+				"building environment do not support packed ring vectorized");
 #endif
 		}
 	}
@@ -2320,6 +2320,17 @@ virtio_dev_configure(struct rte_eth_dev *dev)
 			hw->use_vec_rx = 0;
 			hw->use_vec_tx = 0;
 		}
+#elif defined(RTE_ARCH_ARM)
+		if ((hw->use_vec_rx || hw->use_vec_tx) &&
+		    (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) ||
+		     !vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+		     !vtpci_with_feature(hw, VIRTIO_F_VERSION_1) ||
+		     rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) {
+			PMD_DRV_LOG(INFO,
+				"disabled packed ring vectorized path for requirements not met");
+			hw->use_vec_rx = 0;
+			hw->use_vec_tx = 0;
+		}
 #else
 		hw->use_vec_rx = 0;
 		hw->use_vec_tx = 0;
drivers/net/virtio/virtio_rxtx_packed.c +2 -0
index 99d9a5a99..882dca36e 100644
@@ -18,6 +18,8 @@
 
 #ifdef CC_AVX512_SUPPORT
 #include "virtio_rxtx_packed_avx.h"
+#elif defined(RTE_ARCH_ARM)
+#include "virtio_rxtx_packed_neon.h"
 #endif
 
 uint16_t
drivers/net/virtio/virtio_user_ethdev.c +1 -1
index 40345193e..241808cd8 100644
@@ -856,7 +856,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
 
 	if (vectorized) {
 		if (packed_vq) {
-#if defined(CC_AVX512_SUPPORT)
+#if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
 			hw->use_vec_rx = 1;
 			hw->use_vec_tx = 1;
 #else
Signature · 1 line
2.28.0
Reviewed-by on v1 4/4
#24 lore raw

Re: [dpdk-dev] [PATCH v1 4/4] net/virtio: add election for packed vector NEON path

Quoted 11 lines · Joyce Kong

Add NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.

Packed ring vectorized neon path need:
NEON is supported by compiler and host VERSION_1 and IN_ORDER features are negotiated mergeable feature is not negotiated LRO offloading is disabled

Quoted 8 lines
---
 doc/guides/nics/virtio.rst              |  6 +++---
 drivers/net/virtio/meson.build          |  1 +
 drivers/net/virtio/virtio_ethdev.c      | 19 +++++++++++++++----
 drivers/net/virtio/virtio_rxtx_packed.c |  2 ++
 drivers/net/virtio/virtio_user_ethdev.c |  2 +-
 5 files changed, 22 insertions(+), 8 deletions(-)

Thanks,
Maxime

#25 lore raw

Re: [dpdk-dev] [PATCH v1 0/4] Vectorize packed ring RX/TX path with NEON

In reply to Joyce Kong
Quoted 13 lines · Joyce Kong

This patch set introduces vectorized RX/TX path for packed ring with NEON intrinsics.

With this patch set, PVP case has 1.5% perf uplift for the packed vectorized NEON path compared with the non-vector packed path, under 0.001% acceptable loss with 2 cores on vhost side and 1 core on virtio side.

Joyce Kong (4):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add vectorized packed ring Tx NEON path net/virtio: add election for packed vector NEON path

Quoted 17 lines
 doc/guides/nics/virtio.rst                   |   6 +-
 drivers/net/virtio/meson.build               |   5 +-
 drivers/net/virtio/virtio_ethdev.c           |  19 +-
 drivers/net/virtio/virtio_rxtx.c             |   6 +-
 drivers/net/virtio/virtio_rxtx_packed.c      | 139 ++++
 drivers/net/virtio/virtio_rxtx_packed.h      | 317 ++++++++++
 drivers/net/virtio/virtio_rxtx_packed_avx.c  | 626 -------------------
 drivers/net/virtio/virtio_rxtx_packed_avx.h  | 239 +++++++
 drivers/net/virtio/virtio_rxtx_packed_neon.h | 293 +++++++++
 drivers/net/virtio/virtio_user_ethdev.c      |   2 +-
 10 files changed, 1012 insertions(+), 640 deletions(-)
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
 delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h
 create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h

Series applied to dpdk-next-virtio/main.

Thanks,
Maxime