Vectorize packed ring RX path with NEON
Subsystems (2)
cover Vectorize packed ring RX path with NEON
This patch set introduced vectorized RX path for packed ring with NEON.
Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON path
doc/guides/nics/virtio.rst | 4 +- drivers/net/virtio/meson.build | 2 + drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 300 +++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +--------------- drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 + 9 files changed, 566 insertions(+), 271 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
Signature · 1 line
2.28.0
RFC 1/3 net/virtio: move AVX based Rx and Tx code to separate file
Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.
--- drivers/net/virtio/meson.build | 1 + drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 284 ++++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +----------------- 4 files changed, 323 insertions(+), 263 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h
drivers/net/virtio/meson.build +1 -0
@@ -5,6 +5,7 @@ sources += files('virtio_ethdev.c', 'virtio_pci.c', 'virtio_rxtx.c', 'virtio_rxtx_simple.c', + 'virtio_rxtx_packed.c', 'virtqueue.c') deps += ['kvargs', 'bus_pci']
drivers/net/virtio/virtio_rxtx_packed.c new file +37 -0
@@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2010-2020 Intel Corporation +*/ + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> + +#include <rte_net.h> + +#include "virtio_logs.h" +#include "virtio_ethdev.h" +#include "virtio_pci.h" +#include "virtio_rxtx_packed.h" +#include "virtqueue.h" + +/* Stub for linkage when arch specific implementation is not available */ +__rte_weak uint16_t +virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, + struct rte_mbuf **tx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ +rte_panic ("Wrong weak function linked by linker\n" ); +return 0 ; +} + +/* Stub for linkage when arch specific implementation is not available */ +__rte_weak uint16_t +virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, + struct rte_mbuf **rx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ +rte_panic ("Wrong weak function linked by linker\n" ); +return 0 ; +}
drivers/net/virtio/virtio_rxtx_packed.h new file +284 -0
284 lines not on this page · raw message
drivers/net/virtio/virtio_rxtx_packed_avx.c +1 -263
290 lines not on this page · raw message
Signature · 1 line
2.28.0
RFC 2/3 net/virtio: add vectorized packed ring Rx NEON path
In reply to Joyce KongOptimize packed ring Rx batch path with NEON instructions.
--- drivers/net/virtio/meson.build | 1 + drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.h | 16 ++ drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++++++++ 4 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
drivers/net/virtio/meson.build +1 -0
@@ -34,6 +34,7 @@ elif arch_subdir == 'ppc' sources += files('virtio_rxtx_simple_altivec.c') elif arch_subdir == 'arm' and host_machine.cpu_family().startswith('aarch64') sources += files('virtio_rxtx_simple_neon.c') + sources += files('virtio_rxtx_packed_neon.c') endif if is_linux
drivers/net/virtio/virtio_rxtx.c +5 -2
@@ -2020,7 +2020,8 @@ virtio_xmit_pkts_inorder(void *tx_queue,return nb_tx; } -#ifndef CC_AVX512_SUPPORT +#if !defined(CC_AVX512_SUPPORT ) && !defined(RTE_ARCH_ARM ) && \ + !defined(RTE_ARCH_ARM64 ) uint16_t virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, struct rte_mbuf **rx_pkts __rte_unused,
@@ -2028,7 +2029,9 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, {return 0 ; } +#endif +#if !defined(CC_AVX512_SUPPORT ) uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts __rte_unused,
@@ -2036,4 +2039,4 @@ virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, {return 0 ; } -#endif /* ifndef CC_AVX512_SUPPORT */ +#endif
drivers/net/virtio/virtio_rxtx_packed.h +16 -0
@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#if defined(AVX512_SUPPORT )/* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) || defined(RTE_ARCH_ARM64 ) +/* flag bits offset in packed ring desc 32bits */ +#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \ FLAGS_BITS_OFFSET)
@@ -44,8 +51,17 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#if defined(RTE_ARCH_ARM ) || defined(RTE_ARCH_ARM64 ) +/* The cache line size on different aarh64 platforms are +* different, so put a four batch size here to match with +* the minimum cache line size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif +#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.c new file +202 -0
202 lines not on this page · raw message
Signature · 1 line
2.28.0
RFC 3/3 net/virtio: add election for packed vector Rx NEON path
In reply to Joyce KongAdd NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.
Packed ring vectorized neon path need:
NEON is supported by compiler and host VERSION_1 and IN_ORDER features are negotiated mergeable feature is not negotiated LRO offloading is disabled
--- doc/guides/nics/virtio.rst | 4 ++-- drivers/net/virtio/virtio_ethdev.c | 19 +++++++++++++++---- drivers/net/virtio/virtio_user_ethdev.c | 2 ++ 3 files changed, 19 insertions(+), 6 deletions(-)
doc/guides/nics/virtio.rst +2 -2
@@ -483,8 +483,8 @@ according to below configuration: #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. #. Packed virtqueue vectorized Rx path: If building and running environment support - AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && - TCP_LRO Rx offloading is disabled && vectorized option enabled, + (AVX512 || ARCH_ARM || ARCH_ARM64) && in-order feature is negotiated && Rx mergeable + is not negotiated && TCP_LRO Rx offloading is disabled && vectorized option enabled, this path will be selected. #. Packed virtqueue vectorized Tx path: If building and running environment support AVX512 && in-order feature is negotiated && vectorized option enabled,
drivers/net/virtio/virtio_ethdev.c +15 -4
@@ -1958,12 +1958,14 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)if (!vtpci_packed_queue (hw)) { hw->use_vec_rx =1 ; }else { -#if !defined(CC_AVX512_SUPPORT ) -PMD_DRV_LOG (INFO , -"building environment do not support packed ring vectorized" ); -#else +#if defined(CC_AVX512_SUPPORT ) hw->use_vec_rx =1 ; hw->use_vec_tx =1 ; +#elif defined(RTE_ARCH_ARM ) || defined(RTE_ARCH_ARM64 ) + hw->use_vec_rx =1 ; +#else +PMD_DRV_LOG (INFO , +"building environment do not support packed ring vectorized" );#endif } }
@@ -2311,6 +2313,15 @@ virtio_dev_configure(struct rte_eth_dev *dev) hw->use_vec_rx =0 ; hw->use_vec_tx = 0; } +#elif defined(RTE_ARCH_ARM) || defined(RTE_ARCH_ARM64) +if (hw->use_vec_rx && + (!rte_cpu_get_flag_enabled (RTE_CPUFLAG_NEON ) || + !vtpci_with_feature (hw,VIRTIO_F_IN_ORDER ) || + !vtpci_with_feature (hw,VIRTIO_F_VERSION_1 ))) { +PMD_DRV_LOG (INFO , +"disabled packed ring vectorized path for requirements not met" ); + hw->use_vec_rx =0 ; + }#else hw->use_vec_rx =0 ; hw->use_vec_tx =0 ;
drivers/net/virtio/virtio_user_ethdev.c +2 -0
@@ -766,6 +766,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)#if defined(CC_AVX512_SUPPORT ) hw->use_vec_rx =1 ; hw->use_vec_tx =1 ; +#elif defined(RTE_ARCH_ARM ) || defined(RTE_ARCH_ARM64 ) + hw->use_vec_rx =1 ;#else PMD_INIT_LOG (INFO ,"building environment do not support packed ring vectorized" );
Signature · 1 line
2.28.0
Hi Joyce,
Quoted 2 lines · Joyce Kong
This patch set introduced vectorized RX path for packed ring with NEON.
Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.
Could you please send a new revision with the build issues reported by Intel CI fixed?
http://mails.dpdk.org/archives/test-report/2020-September/152501.html
Thanks,
Maxime
Quoted 19 lines
Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON pathdoc/guides/nics/virtio.rst | 4 +- drivers/net/virtio/meson.build | 2 + drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 300 +++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +--------------- drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 + 9 files changed, 566 insertions(+), 271 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
Hi Maxime,
Quoted 19 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEONHi Joyce,
Quoted 2 lines · Joyce Kong
This patch set introduced vectorized RX path for packed ring with NEON.
Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.
Quoted 8 lines
Could you please send a new revision with the build issues reported by Intel CI fixed?
http://mails.dpdk.org/archives/test-report/2020-September/152501.html
Thanks,
Maxime
Thanks for your comments, and I will send the new revision soon.
Quoted 20 lines
Quoted 20 lines
Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON pathdoc/guides/nics/virtio.rst | 4 +- drivers/net/virtio/meson.build | 2 + drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 300 +++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +--------------- drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 + 9 files changed, 566 insertions(+), 271 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
Quoted 12 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEONHi Joyce,
Quoted 7 lines
Quoted 2 lines · Joyce Kong
This patch set introduced vectorized RX path for packed ring with NEON.
Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.
This patch doesn’t target 20.11. New version will be sent out but not in short time. You can skip this one for 20.11 release.
Thanks.
Quoted 29 lines
Could you please send a new revision with the build issues reported by Intel CI fixed?
http://mails.dpdk.org/archives/test-report/2020-September/152501.html
Thanks,
MaximeQuoted 19 lines
Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON pathdoc/guides/nics/virtio.rst | 4 +- drivers/net/virtio/meson.build | 2 + drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 300 +++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +--------------- drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 + 9 files changed, 566 insertions(+), 271 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
Quoted 13 lines · Ruifeng Wang
Quoted 13 lines
-----Original Message-----
From: Maxime Coquelin <[email protected]>
Sent: Monday, October 5, 2020 3:34 PM
To: Joyce Kong <[email protected]>
Cc: [email protected]; [email protected]; nd <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>; Phil Yang <[email protected]>
Subject: Re: [RFC 0/3] Vectorize packed ring RX path with NEONHi Joyce,
On 9/11/20 2:09 PM, Joyce Kong wrote:
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
This patch set introduced vectorized RX path for packed ring with NEON.
Overall, the series looks good. I will have to trust you on the NEON implementation. Ideally, having a detailed review from someone from ARM would be good.
This patch doesn’t target 20.11. New version will be sent out but not in short time. You can skip this one for 20.11 release.
Thanks Ruifeng for the update.
I'll mark the series as postponed then.
Maxime
Quoted 31 lines
Thanks.
Quoted 29 lines
Could you please send a new revision with the build issues reported by Intel CI fixed?
http://mails.dpdk.org/archives/test-report/2020-September/152501.html
Thanks,
MaximeQuoted 19 lines
Joyce Kong (3):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add election for packed vector Rx NEON pathdoc/guides/nics/virtio.rst | 4 +- drivers/net/virtio/meson.build | 2 + drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 7 +- drivers/net/virtio/virtio_rxtx_packed.c | 37 +++ drivers/net/virtio/virtio_rxtx_packed.h | 300 +++++++++++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 264 +--------------- drivers/net/virtio/virtio_rxtx_packed_neon.c | 202 +++++++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 + 9 files changed, 566 insertions(+), 271 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.c
cover Vectorize packed ring RX/TX path with NEON
In reply to Joyce KongThis patch set introduces vectorized RX/TX path for packed ring with NEON intrinsics.
With this patch set, PVP case has 1.5% perf uplift for the packed vectorized NEON path compared with the non-vector packed path, under 0.001% acceptable loss with 2 cores on vhost side and 1 core on virtio side.
Joyce Kong (4):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add vectorized packed ring Tx NEON path net/virtio: add election for packed vector NEON path
doc/guides/nics/virtio.rst | 6 +- drivers/net/virtio/meson.build | 5 +- drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 6 +- drivers/net/virtio/virtio_rxtx_packed.c | 139 ++++ drivers/net/virtio/virtio_rxtx_packed.h | 317 ++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 ------------------- drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 +++++++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 293 +++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 +- 10 files changed, 1012 insertions(+), 640 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
Signature · 1 line
2.28.0
PATCH 1/4 net/virtio: move AVX based Rx and Tx code to separate file
Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.
--- drivers/net/virtio/meson.build | 4 +- drivers/net/virtio/virtio_rxtx.c | 6 +- drivers/net/virtio/virtio_rxtx_packed.c | 137 +++++ drivers/net/virtio/virtio_rxtx_packed.h | 298 ++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 -------------------- drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 ++++++++ 6 files changed, 678 insertions(+), 632 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h
drivers/net/virtio/meson.build +2 -2
@@ -13,12 +13,12 @@ if arch_subdir == 'x86' if cc.has_argument('-mavx512f') and cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw') cflags += ['-DCC_AVX512_SUPPORT'] virtio_avx512_lib = static_library('virtio_avx512_lib', - 'virtio_rxtx_packed_avx.c', + 'virtio_rxtx_packed.c', dependencies: [static_rte_ethdev, static_rte_kvargs, static_rte_bus_pci], include_directories: includes, c_args: [cflags, '-mavx512f', '-mavx512bw', '-mavx512vl']) - objs += virtio_avx512_lib.extract_objects('virtio_rxtx_packed_avx.c') + objs += virtio_avx512_lib.extract_objects('virtio_rxtx_packed.c') if (toolchain == 'gcc' and cc.version().version_compare('>=8.3.0')) cflags += '-DVHOST_GCC_UNROLL_PRAGMA' elif (toolchain == 'clang' and cc.version().version_compare('>=3.7.0'))
drivers/net/virtio/virtio_rxtx.c +2 -4
@@ -2025,8 +2025,7 @@ virtio_xmit_pkts_inorder(void *tx_queue,return nb_tx; } -#ifndef CC_AVX512_SUPPORT -uint16_t +__rte_weak uint16_t virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused, struct rte_mbuf **rx_pkts __rte_unused, uint16_t nb_pkts __rte_unused)
@@ -2034,11 +2033,10 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,return 0 ; } -uint16_t +__rte_weak uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts __rte_unused, uint16_t nb_pkts __rte_unused) {return 0 ; } -#endif /* ifndef CC_AVX512_SUPPORT */
drivers/net/virtio/virtio_rxtx_packed.c new file +137 -0
137 lines not on this page · raw message
drivers/net/virtio/virtio_rxtx_packed.h new file +298 -0
298 lines not on this page · raw message
drivers/net/virtio/virtio_rxtx_packed_avx.c deleted +0 -626
626 lines not on this page · raw message
drivers/net/virtio/virtio_rxtx_packed_avx.h new file +239 -0
239 lines not on this page · raw message
Signature · 1 line
2.28.0
Re: [dpdk-dev] [PATCH v1 1/4] net/virtio: move AVX based Rx and Tx code to separate file
Quoted 5 lines · Joyce Kong
Split out AVX instruction based virtio packed ring Rx and Tx implementation to a separate file.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>
Quoted 13 lines
--- drivers/net/virtio/meson.build | 4 +- drivers/net/virtio/virtio_rxtx.c | 6 +- drivers/net/virtio/virtio_rxtx_packed.c | 137 +++++ drivers/net/virtio/virtio_rxtx_packed.h | 298 ++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 -------------------- drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 ++++++++ 6 files changed, 678 insertions(+), 632 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h
Thanks,
Maxime
PATCH 2/4 net/virtio: add vectorized packed ring Rx NEON path
In reply to Joyce KongOptimize packed ring Rx batch path with NEON instructions.
--- drivers/net/virtio/virtio_rxtx_packed.h | 15 ++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
drivers/net/virtio/virtio_rxtx_packed.h +15 -0
@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#ifdef CC_AVX512_SUPPORT /* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) +/* flag bits offset in packed ring desc from ID */ +#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \ FLAGS_BITS_OFFSET)
@@ -44,8 +51,16 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#ifdef RTE_ARCH_ARM +/* The cache line size on different Arm platforms are different, so +* put a four batch size here to match with the minimum cache line +* size and accommodate NEON register size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +150 -0
150 lines not on this page · raw message
Signature · 1 line
2.28.0
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
Quoted 78 lines · Joyce Kong
Optimize packed ring Rx batch path with NEON instructions.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- drivers/net/virtio/virtio_rxtx_packed.h | 15 ++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.hdrivers/net/virtio/virtio_rxtx_packed.h +15
-0@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#ifdef CC_AVX512_SUPPORT /* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) +/* flag bits offset in packed ring desc from ID */ +#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \ FLAGS_BITS_OFFSET)@@ -44,8 +51,16 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#ifdef RTE_ARCH_ARM +/* The cache line size on different Arm platforms are different, so +* put a four batch size here to match with the minimum cache line +* size and accommodate NEON register size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24
-0@@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Arm Corporation +*/ + +#include <stdlib.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <rte_net.h> +#include <rte_vect.h> + +#include "virtio_ethdev.h" +#include "virtio_pci.h" +#include "virtio_rxtx_packed.h" +#include "virtqueue.h" + +static inline uint16_t +virtqueue_dequeue_batch_packed_vec (struct virtnet_rx *rxvq, +struct rte_mbuf **rx_pkts) +{ +struct virtqueue *vq = rxvq->vq; +struct virtio_hw *hw = vq->hw;
drivers/net/virtio/virtio_rxtx_packed_neon.h line 34
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10
-0+uint16_t head_size = hw->vtnet_hdr_size; +uint16_t id = vq->vq_used_cons_idx; +struct vring_packed_desc *p_desc; +uint16_t i; + +if (id &PACKED_BATCH_MASK ) +return -1 ; + +if (unlikely ((id +PACKED_BATCH_SIZE ) > vq->vq_nentries)) +return -1 ;
This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.
Quoted 44 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +44
-0+/* Map packed descriptor to mbuf fields. */ +uint8x16_t shuf_msk1 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +0 ,1 ,/* octet 1~0, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +0 ,1 ,/* octet 1~0, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +uint8x16_t shuf_msk2 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +8 ,9 ,/* octet 9~8, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +8 ,9 ,/* octet 9~8, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +/* Subtract the header length. */ +uint16x8_t len_adjust = { +0 ,0 ,/* ignore pkt_type field */ + head_size,/* sub head_size on pkt_len */ +0 ,/* ignore high 16 bits of pkt_len */ + head_size,/* sub head_size on data_len */ +0 ,0 ,0 /* ignore non-length fields */ + }; + +uint64x2_t desc[PACKED_BATCH_SIZE /2 ]; +uint64x2x2_t mbp[PACKED_BATCH_SIZE /2 ]; +uint64x2_t pkt_mb[PACKED_BATCH_SIZE ]; + + p_desc = &vq->vq_packed.ring.desc[id]; +/* Load high 64 bits of packed descriptor 0,1. */ + desc[0 ] =vld2q_u64 ((uint64_t *)(p_desc)).val[1 ]; +/* Load high 64 bits of packed descriptor 2,3. */ + desc[1 ] =vld2q_u64 ((uint64_t *)(p_desc +2 )).val[1 ]; + +/* Only care avail/used bits. */ +uint32x4_t v_mask =vdupq_n_u32 (PACKED_FLAGS_MASK ); +/* Extract high 32 bits of packed descriptor (id, flags). */ +uint32x4_t v_desc =vuzp2q_u32 (vreinterpretq_u32_u64 (desc[0 ]), +vreinterpretq_u32_u64 (desc[1 ])); +uint32x4_t v_flag =vandq_u32 (v_desc, v_mask);
drivers/net/virtio/virtio_rxtx_packed_neon.h line 89
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10
-0+ +uint32x4_t v_used_flag =vdupq_n_u32 (0 ); +if (vq->vq_packed.used_wrap_counter) + v_used_flag =vdupq_n_u32 (PACKED_FLAGS_MASK ); + +poly128_t desc_stats =vreinterpretq_p128_u32 (~vceqq_u32 (v_flag, v_used_flag)); + +/* Check all descs are used. */ +if (desc_stats) +return -1 ;
Same here. You should return 0 here as the queue is full.
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61
-0+ +/* Load 2 mbuf pointers per time. */ + mbp[0 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id]); +vst1q_u64 ((uint64_t *)&rx_pkts[0 ], mbp[0 ].val[0 ]); + + mbp[1 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id +2 ]); +vst1q_u64 ((uint64_t *)&rx_pkts[2 ], mbp[1 ].val[0 ]); + +/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))' + pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( +vreinterpretq_u8_u64 (desc[1 ]), shuf_msk2)); + + pkt_mb[0 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[0 ]), len_adjust)); + pkt_mb[1 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[1 ]), len_adjust)); + pkt_mb[2 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[2 ]), len_adjust)); + pkt_mb[3 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[3 ]), len_adjust)); + +vst1q_u64 ((void *)&rx_pkts[0 ]->rx_descriptor_fields1, pkt_mb[0 ]); +vst1q_u64 ((void *)&rx_pkts[1 ]->rx_descriptor_fields1, pkt_mb[1 ]); +vst1q_u64 ((void *)&rx_pkts[2 ]->rx_descriptor_fields1, pkt_mb[2 ]); +vst1q_u64 ((void *)&rx_pkts[3 ]->rx_descriptor_fields1, pkt_mb[3 ]); + +if (hw->has_rx_offload) { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { +char *addr = (char *)rx_pkts[i]->buf_addr + +RTE_PKTMBUF_HEADROOM - head_size; +virtio_vec_rx_offload (rx_pkts[i], + (struct virtio_net_hdr *)addr); + } + } + +virtio_update_batch_stats (&rxvq->stats, rx_pkts[0 ]->pkt_len, + rx_pkts[1 ]->pkt_len, rx_pkts[2 ]->pkt_len, + rx_pkts[3 ]->pkt_len); + + vq->vq_free_cnt +=PACKED_BATCH_SIZE ; + + vq->vq_used_cons_idx +=PACKED_BATCH_SIZE ; +if (vq->vq_used_cons_idx >= vq->vq_nentries) { + vq->vq_used_cons_idx -= vq->vq_nentries; + vq->vq_packed.used_wrap_counter ^=1 ; + } + +return 0 ; +}
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
Quoted 143 lines · Maxime Coquelin
Quoted 78 lines · Joyce Kong
Optimize packed ring Rx batch path with NEON instructions.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- drivers/net/virtio/virtio_rxtx_packed.h | 15 ++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.hdrivers/net/virtio/virtio_rxtx_packed.h +15
-0@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#ifdef CC_AVX512_SUPPORT /* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) +/* flag bits offset in packed ring desc from ID */ +#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \ FLAGS_BITS_OFFSET)@@ -44,8 +51,16 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#ifdef RTE_ARCH_ARM +/* The cache line size on different Arm platforms are different, so +* put a four batch size here to match with the minimum cache line +* size and accommodate NEON register size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24
-0@@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Arm Corporation +*/ + +#include <stdlib.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <rte_net.h> +#include <rte_vect.h> + +#include "virtio_ethdev.h" +#include "virtio_pci.h" +#include "virtio_rxtx_packed.h" +#include "virtqueue.h" + +static inline uint16_t +virtqueue_dequeue_batch_packed_vec (struct virtnet_rx *rxvq, +struct rte_mbuf **rx_pkts) +{ +struct virtqueue *vq = rxvq->vq; +struct virtio_hw *hw = vq->hw;drivers/net/virtio/virtio_rxtx_packed_neon.h line 34
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10
-0+uint16_t head_size = hw->vtnet_hdr_size; +uint16_t id = vq->vq_used_cons_idx; +struct vring_packed_desc *p_desc; +uint16_t i; + +if (id &PACKED_BATCH_MASK ) +return -1 ; + +if (unlikely ((id +PACKED_BATCH_SIZE ) > vq->vq_nentries)) +return -1 ;This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.
Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +48
-0+/* Map packed descriptor to mbuf fields. */ +uint8x16_t shuf_msk1 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +0 ,1 ,/* octet 1~0, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +0 ,1 ,/* octet 1~0, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +uint8x16_t shuf_msk2 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +8 ,9 ,/* octet 9~8, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +8 ,9 ,/* octet 9~8, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +/* Subtract the header length. */ +uint16x8_t len_adjust = { +0 ,0 ,/* ignore pkt_type field */ + head_size,/* sub head_size on pkt_len */ +0 ,/* ignore high 16 bits of pkt_len */ + head_size,/* sub head_size on data_len */ +0 ,0 ,0 /* ignore non-length fields */ + }; + +uint64x2_t desc[PACKED_BATCH_SIZE /2 ]; +uint64x2x2_t mbp[PACKED_BATCH_SIZE /2 ]; +uint64x2_t pkt_mb[PACKED_BATCH_SIZE ]; + + p_desc = &vq->vq_packed.ring.desc[id]; +/* Load high 64 bits of packed descriptor 0,1. */ + desc[0 ] =vld2q_u64 ((uint64_t *)(p_desc)).val[1 ]; +/* Load high 64 bits of packed descriptor 2,3. */ + desc[1 ] =vld2q_u64 ((uint64_t *)(p_desc +2 )).val[1 ]; + +/* Only care avail/used bits. */ +uint32x4_t v_mask =vdupq_n_u32 (PACKED_FLAGS_MASK ); +/* Extract high 32 bits of packed descriptor (id, flags). */ +uint32x4_t v_desc =vuzp2q_u32 (vreinterpretq_u32_u64 (desc[0 ]), +vreinterpretq_u32_u64 (desc[1 ])); +uint32x4_t v_flag =vandq_u32 (v_desc, v_mask); + +uint32x4_t v_used_flag =vdupq_n_u32 (0 ); +if (vq->vq_packed.used_wrap_counter) + v_used_flag =vdupq_n_u32 (PACKED_FLAGS_MASK );
Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h line 89
Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6
-0+ +poly128_t desc_stats =vreinterpretq_p128_u32 (~vceqq_u32 (v_flag, v_used_flag)); + +/* Check all descs are used. */ +if (desc_stats) +return -1 ;Same here. You should return 0 here as the queue is full.
Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.
Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure
Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61
-0+ +/* Load 2 mbuf pointers per time. */ + mbp[0 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id]); +vst1q_u64 ((uint64_t *)&rx_pkts[0 ], mbp[0 ].val[0 ]); + + mbp[1 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id +2 ]); +vst1q_u64 ((uint64_t *)&rx_pkts[2 ], mbp[1 ].val[0 ]); + +/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))' + pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( +vreinterpretq_u8_u64 (desc[1 ]), shuf_msk2)); + + pkt_mb[0 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[0 ]), len_adjust)); + pkt_mb[1 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[1 ]), len_adjust)); + pkt_mb[2 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[2 ]), len_adjust)); + pkt_mb[3 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[3 ]), len_adjust)); + +vst1q_u64 ((void *)&rx_pkts[0 ]->rx_descriptor_fields1, pkt_mb[0 ]); +vst1q_u64 ((void *)&rx_pkts[1 ]->rx_descriptor_fields1, pkt_mb[1 ]); +vst1q_u64 ((void *)&rx_pkts[2 ]->rx_descriptor_fields1, pkt_mb[2 ]); +vst1q_u64 ((void *)&rx_pkts[3 ]->rx_descriptor_fields1, pkt_mb[3 ]); + +if (hw->has_rx_offload) { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { +char *addr = (char *)rx_pkts[i]->buf_addr + +RTE_PKTMBUF_HEADROOM - head_size; +virtio_vec_rx_offload (rx_pkts[i], + (struct virtio_net_hdr *)addr); + } + } + +virtio_update_batch_stats (&rxvq->stats, rx_pkts[0 ]->pkt_len, + rx_pkts[1 ]->pkt_len, rx_pkts[2 ]->pkt_len, + rx_pkts[3 ]->pkt_len); + + vq->vq_free_cnt +=PACKED_BATCH_SIZE ; + + vq->vq_used_cons_idx +=PACKED_BATCH_SIZE ; +if (vq->vq_used_cons_idx >= vq->vq_nentries) { + vq->vq_used_cons_idx -= vq->vq_nentries; + vq->vq_packed.used_wrap_counter ^=1 ; + } + +return 0 ; +}
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
Quoted 155 lines · Maxime Coquelin
Quoted 143 lines · Maxime Coquelin
Quoted 78 lines · Joyce Kong
Optimize packed ring Rx batch path with NEON instructions.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- drivers/net/virtio/virtio_rxtx_packed.h | 15 ++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.hdrivers/net/virtio/virtio_rxtx_packed.h +15
-0@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#ifdef CC_AVX512_SUPPORT /* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) +/* flag bits offset in packed ring desc from ID */ +#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \ FLAGS_BITS_OFFSET)@@ -44,8 +51,16 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#ifdef RTE_ARCH_ARM +/* The cache line size on different Arm platforms are different, so +* put a four batch size here to match with the minimum cache line +* size and accommodate NEON register size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA drivers/net/virtio/virtio_rxtx_packed_neon.h new file +24
-0@@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Arm Corporation +*/ + +#include <stdlib.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <rte_net.h> +#include <rte_vect.h> + +#include "virtio_ethdev.h" +#include "virtio_pci.h" +#include "virtio_rxtx_packed.h" +#include "virtqueue.h" + +static inline uint16_t +virtqueue_dequeue_batch_packed_vec (struct virtnet_rx *rxvq, +struct rte_mbuf **rx_pkts) +{ +struct virtqueue *vq = rxvq->vq; +struct virtio_hw *hw = vq->hw;drivers/net/virtio/virtio_rxtx_packed_neon.h line 34
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10
-0+uint16_t head_size = hw->vtnet_hdr_size; +uint16_t id = vq->vq_used_cons_idx; +struct vring_packed_desc *p_desc; +uint16_t i; + +if (id &PACKED_BATCH_MASK ) +return -1 ; + +if (unlikely ((id +PACKED_BATCH_SIZE ) > vq->vq_nentries)) +return -1 ;This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.
Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +48
-0+/* Map packed descriptor to mbuf fields. */ +uint8x16_t shuf_msk1 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +0 ,1 ,/* octet 1~0, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +0 ,1 ,/* octet 1~0, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +uint8x16_t shuf_msk2 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +8 ,9 ,/* octet 9~8, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */ +8 ,9 ,/* octet 9~8, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +/* Subtract the header length. */ +uint16x8_t len_adjust = { +0 ,0 ,/* ignore pkt_type field */ + head_size,/* sub head_size on pkt_len */ +0 ,/* ignore high 16 bits of pkt_len */ + head_size,/* sub head_size on data_len */ +0 ,0 ,0 /* ignore non-length fields */ + }; + +uint64x2_t desc[PACKED_BATCH_SIZE /2 ]; +uint64x2x2_t mbp[PACKED_BATCH_SIZE /2 ]; +uint64x2_t pkt_mb[PACKED_BATCH_SIZE ]; + + p_desc = &vq->vq_packed.ring.desc[id]; +/* Load high 64 bits of packed descriptor 0,1. */ + desc[0 ] =vld2q_u64 ((uint64_t *)(p_desc)).val[1 ]; +/* Load high 64 bits of packed descriptor 2,3. */ + desc[1 ] =vld2q_u64 ((uint64_t *)(p_desc +2 )).val[1 ]; + +/* Only care avail/used bits. */ +uint32x4_t v_mask =vdupq_n_u32 (PACKED_FLAGS_MASK ); +/* Extract high 32 bits of packed descriptor (id, flags). */ +uint32x4_t v_desc =vuzp2q_u32 (vreinterpretq_u32_u64 (desc[0 ]), +vreinterpretq_u32_u64 (desc[1 ])); +uint32x4_t v_flag =vandq_u32 (v_desc, v_mask); + +uint32x4_t v_used_flag =vdupq_n_u32 (0 ); +if (vq->vq_packed.used_wrap_counter) + v_used_flag =vdupq_n_u32 (PACKED_FLAGS_MASK );Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h line 89
Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6
-0+ +poly128_t desc_stats =vreinterpretq_p128_u32 (~vceqq_u32 (v_flag, v_used_flag)); + +/* Check all descs are used. */ +if (desc_stats) +return -1 ;Same here. You should return 0 here as the queue is full.
Quoted 6 lines
Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.
Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure
Joyce, are you fine if I do the cange while applying?
I have a big series that will conflicts with your patch set, so I'd like to have yours merged ASAP so I can start the rebase.
Thanks,
Maxime
Quoted 63 lines
Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61
-0+ +/* Load 2 mbuf pointers per time. */ + mbp[0 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id]); +vst1q_u64 ((uint64_t *)&rx_pkts[0 ], mbp[0 ].val[0 ]); + + mbp[1 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id +2 ]); +vst1q_u64 ((uint64_t *)&rx_pkts[2 ], mbp[1 ].val[0 ]); + +/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))' + pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( +vreinterpretq_u8_u64 (desc[1 ]), shuf_msk2)); + + pkt_mb[0 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[0 ]), len_adjust)); + pkt_mb[1 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[1 ]), len_adjust)); + pkt_mb[2 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[2 ]), len_adjust)); + pkt_mb[3 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[3 ]), len_adjust)); + +vst1q_u64 ((void *)&rx_pkts[0 ]->rx_descriptor_fields1, pkt_mb[0 ]); +vst1q_u64 ((void *)&rx_pkts[1 ]->rx_descriptor_fields1, pkt_mb[1 ]); +vst1q_u64 ((void *)&rx_pkts[2 ]->rx_descriptor_fields1, pkt_mb[2 ]); +vst1q_u64 ((void *)&rx_pkts[3 ]->rx_descriptor_fields1, pkt_mb[3 ]); + +if (hw->has_rx_offload) { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { +char *addr = (char *)rx_pkts[i]->buf_addr + +RTE_PKTMBUF_HEADROOM - head_size; +virtio_vec_rx_offload (rx_pkts[i], + (struct virtio_net_hdr *)addr); + } + } + +virtio_update_batch_stats (&rxvq->stats, rx_pkts[0 ]->pkt_len, + rx_pkts[1 ]->pkt_len, rx_pkts[2 ]->pkt_len, + rx_pkts[3 ]->pkt_len); + + vq->vq_free_cnt +=PACKED_BATCH_SIZE ; + + vq->vq_used_cons_idx +=PACKED_BATCH_SIZE ; +if (vq->vq_used_cons_idx >= vq->vq_nentries) { + vq->vq_used_cons_idx -= vq->vq_nentries; + vq->vq_packed.used_wrap_counter ^=1 ; + } + +return 0 ; +}
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
Quoted 169 lines · Maxime Coquelin
Quoted 36 lines · Maxime Coquelin
Quoted 33 lines · Joyce Kong
Quoted 30 lines
Optimize packed ring Rx batch path with NEON instructions.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- drivers/net/virtio/virtio_rxtx_packed.h | 15 ++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 150 +++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.hdrivers/net/virtio/virtio_rxtx_packed.h +6
-0@@ -19,9 +19,16 @@#include "virtqueue.h" #define BYTE_SIZE 8 + +#ifdef CC_AVX512_SUPPORT /* flag bits offset in packed ring desc higher 64bits */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE) +#elif defined(RTE_ARCH_ARM ) +/* flag bits offset in packed ring desc from ID */ #define +FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) #endifQuoted 1 line
Quoted 1 line
drivers/net/virtio/virtio_rxtx_packed.h line 32
Quoted 1 line
drivers/net/virtio/virtio_rxtx_packed.h +0
-0#define PACKED_FLAGS_MASK ((0ULL |
VRING_PACKED_DESC_F_AVAIL_USED) << \Quoted 62 lines
Quoted 62 lines
Quoted 48 lines
drivers/net/virtio/virtio_rxtx_packed.h +8
-0FLAGS_BITS_OFFSET)@@ -44,8 +51,16 @@/* net hdr short size mask */ #define NET_HDR_MASK 0x3F +#ifdef RTE_ARCH_ARM +/* The cache line size on different Arm platforms are different, so +* put a four batch size here to match with the minimum cache line +* size and accommodate NEON register size. +*/ +#define PACKED_BATCH_SIZE 4 +#else #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \ sizeof(struct vring_packed_desc)) +#endif #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)#ifdef VIRTIO_GCC_UNROLL_PRAGMA drivers/net/virtio/virtio_rxtx_packed_neon.h new file +23
-0@@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Arm Corporation */ + +#include <stdlib.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> + +#include <rte_net.h> +#include <rte_vect.h> + +#include "virtio_ethdev.h" +#include "virtio_pci.h" +#include "virtio_rxtx_packed.h" +#include "virtqueue.h" + +static inline uint16_t +virtqueue_dequeue_batch_packed_vec (struct virtnet_rx *rxvq, +struct rte_mbuf **rx_pkts) +{ +struct virtqueue *vq = rxvq->vq; +struct virtio_hw *hw = vq->hw;drivers/net/virtio/virtio_rxtx_packed_neon.h line 33
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +10
-0+uint16_t head_size = hw->vtnet_hdr_size; +uint16_t id = vq->vq_used_cons_idx; +struct vring_packed_desc *p_desc; +uint16_t i; + +if (id &PACKED_BATCH_MASK ) +return -1 ; + +if (unlikely ((id +PACKED_BATCH_SIZE ) > vq->vq_nentries)) +return -1 ;This function returns an unsigned short, I think you should return 0 here since it failed to dequeue packets.
Quoted 5 lines
Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h
Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5
-0+/* Map packed descriptor to mbuf fields. */ +uint8x16_t shuf_msk1 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +0 ,1 ,/* octet 1~0, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */
Quoted 9 lines
Quoted 9 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h
Quoted 9 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +9
-0+0 ,1 ,/* octet 1~0, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +uint8x16_t shuf_msk2 = { +0xFF ,0xFF ,0xFF ,0xFF ,/* pkt_type set as unknown */ +8 ,9 ,/* octet 9~8, low 16 bits pkt_len */ +0xFF ,0xFF ,/* skip high 16 bits of pkt_len, zero out */
Quoted 44 lines
Quoted 35 lines
Quoted 35 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +35
-0+8 ,9 ,/* octet 9~8, 16 bits data_len */ +0xFF ,0xFF ,/* vlan tci set as unknown */ +0xFF ,0xFF ,0xFF ,0xFF + }; + +/* Subtract the header length. */ +uint16x8_t len_adjust = { +0 ,0 ,/* ignore pkt_type field */ + head_size,/* sub head_size on pkt_len */ +0 ,/* ignore high 16 bits of pkt_len */ + head_size,/* sub head_size on data_len */ +0 ,0 ,0 /* ignore non-length fields */ + }; + +uint64x2_t desc[PACKED_BATCH_SIZE /2 ]; +uint64x2x2_t mbp[PACKED_BATCH_SIZE /2 ]; +uint64x2_t pkt_mb[PACKED_BATCH_SIZE ]; + + p_desc = &vq->vq_packed.ring.desc[id]; +/* Load high 64 bits of packed descriptor 0,1. */ + desc[0 ] =vld2q_u64 ((uint64_t *)(p_desc)).val[1 ]; +/* Load high 64 bits of packed descriptor 2,3. */ + desc[1 ] =vld2q_u64 ((uint64_t *)(p_desc +2 )).val[1 ]; + +/* Only care avail/used bits. */ +uint32x4_t v_mask =vdupq_n_u32 (PACKED_FLAGS_MASK ); +/* Extract high 32 bits of packed descriptor (id, flags). */ +uint32x4_t v_desc =vuzp2q_u32 (vreinterpretq_u32_u64 (desc[0 ]), +vreinterpretq_u32_u64 (desc[1 ])); +uint32x4_t v_flag =vandq_u32 (v_desc, v_mask); + +uint32x4_t v_used_flag =vdupq_n_u32 (0 ); +if (vq->vq_packed.used_wrap_counter) + v_used_flag =vdupq_n_u32 (PACKED_FLAGS_MASK ); +Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h line 89
Quoted 6 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +6
-0+poly128_t desc_stats =vreinterpretq_p128_u32 (~vceqq_u32 (v_flag, +v_used_flag)); + +/* Check all descs are used. */ +if (desc_stats) +return -1 ;Same here. You should return 0 here as the queue is full.
Quoted 6 lines
Just looked again at the code and at AVX implementation. It should not return 0 here, but any positive value.
Maybe the cleanest way would change the function prototype to int. 0: success
-1: failure
Quoted 7 lines
Joyce, are you fine if I do the cange while applying?
I have a big series that will conflicts with your patch set, so I'd like to have yours merged ASAP so I can start the rebase.Thanks,
Maxime
Maxime, It's ok if you would do the change while applying.
Thanks,
Joyce
Quoted 63 lines
Quoted 63 lines
Quoted 63 lines
Quoted 62 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +61
-0+ +/* Load 2 mbuf pointers per time. */ + mbp[0 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id]); +vst1q_u64 ((uint64_t *)&rx_pkts[0 ], mbp[0 ].val[0 ]); + + mbp[1 ] =vld2q_u64 ((uint64_t *)&vq->vq_descx[id +2 ]); +vst1q_u64 ((uint64_t *)&rx_pkts[2 ], mbp[1 ].val[0 ]); + +/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))' + pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8( +vreinterpretq_u8_u64 (desc[1 ]), shuf_msk2)); + + pkt_mb[0 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[0 ]), len_adjust)); + pkt_mb[1 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[1 ]), len_adjust)); + pkt_mb[2 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[2 ]), len_adjust)); + pkt_mb[3 ] =vreinterpretq_u64_u16 (vsubq_u16 ( +vreinterpretq_u16_u64 (pkt_mb[3 ]), len_adjust)); + +vst1q_u64 ((void *)&rx_pkts[0 ]->rx_descriptor_fields1, pkt_mb[0 ]); +vst1q_u64 ((void *)&rx_pkts[1 ]->rx_descriptor_fields1, pkt_mb[1 ]); +vst1q_u64 ((void *)&rx_pkts[2 ]->rx_descriptor_fields1, pkt_mb[2 ]); +vst1q_u64 ((void *)&rx_pkts[3 ]->rx_descriptor_fields1, pkt_mb[3 ]); + +if (hw->has_rx_offload) { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { +char *addr = (char *)rx_pkts[i]->buf_addr + +RTE_PKTMBUF_HEADROOM - head_size; +virtio_vec_rx_offload (rx_pkts[i], + (struct virtio_net_hdr *)addr); + } + } + +virtio_update_batch_stats (&rxvq->stats, rx_pkts[0 ]->pkt_len, + rx_pkts[1 ]->pkt_len, rx_pkts[2 ]->pkt_len, + rx_pkts[3 ]->pkt_len); + + vq->vq_free_cnt +=PACKED_BATCH_SIZE ; + + vq->vq_used_cons_idx +=PACKED_BATCH_SIZE ; +if (vq->vq_used_cons_idx >= vq->vq_nentries) { + vq->vq_used_cons_idx -= vq->vq_nentries; + vq->vq_packed.used_wrap_counter ^=1 ; + } + +return 0 ; +}
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
In reply to Joyce Kongdrivers/net/virtio/virtio_rxtx_packed_neon.h line 110
Quoted 13 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +13
-0+/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
s\'\;
I will fix in next-net but my concern is why this has been not caught by any of our automated builds?
In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
<snip>
Quoted 19 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h
Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5
-0+/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
drivers/net/virtio/virtio_rxtx_packed_neon.h line 110
Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +8
-0+* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))'s\'\;
Quoted 5 lines
I will fix in next-net but my concern is why this has been not caught by any of our automated builds?
In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/
Looking at [1], Travis CI has not run and the UNH CI did not have Arm builds enabled at the time this patch was submitted.
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
Quoted 21 lines · Honnappa Nagarahalli
<snip>
Quoted 19 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h
Quoted 5 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +5
-0+/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
drivers/net/virtio/virtio_rxtx_packed_neon.h line 110
Quoted 8 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +8
-0+* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))'s\'\;
Quoted 8 lines
Quoted 5 lines
I will fix in next-net but my concern is why this has been not caught by any of our automated builds?
In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/Looking at [1], Travis CI has not run and the UNH CI did not have Arm builds enabled at the time this patch was submitted.
Seem my other email. Travis CI doesn't run 'patch-at-a-time' for execution time reasons.
Quoted 1 line
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
In reply to Ferruh YigitQuoted 18 lines · Ferruh Yigit
Quoted 8 lines · Joyce Kong
+ /**
+ * Update data length and packet length for descriptor. + * structure ofpkt_mb:
+ *
--------------------------------------------------------------------
+ * |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci|
+ *Quoted 8 lines
--------------------------------------------------------------------
+ */
+ pkt_mb[0] =vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[0]), shuf_msk1)); + pkt_mb[1] =vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[0]), shuf_msk2)); + pkt_mb[2] =vreinterpretq_u64_u8(vqtbl1q_u8( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))'
Quoted 8 lines
s\'\;
I will fix in next-net but my concern is why this has been not caught by any of our automated builds?
In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/
Thanks Ferruh for spotting and fixing it.
I think the CI was broken at the time it was submitted, it would be great to have a way to manually trigger the CI again!
That plus me changing laptop recently and not having a full multi-arch build system up and running again made this build issue pass through...
Regards,
Maxime
Re: [dpdk-dev] [PATCH v1 2/4] net/virtio: add vectorized packed ring Rx NEON path
In reply to Ferruh YigitQuoted 17 lines · Ferruh Yigit
drivers/net/virtio/virtio_rxtx_packed_neon.h line 110
Quoted 13 lines · Joyce Kong
drivers/net/virtio/virtio_rxtx_packed_neon.h new file +13
-0+/** +* Update data length and packet length for descriptor. +* structure of pkt_mb: +* -------------------------------------------------------------------- +* |32 bits pkt_type|32 bits pkt_len|16 bits data_len|16 bits vlan_tci| +* -------------------------------------------------------------------- +*/ + pkt_mb[0 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk1)); + pkt_mb[1 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( +vreinterpretq_u8_u64 (desc[0 ]), shuf_msk2)); + pkt_mb[2 ] =vreinterpretq_u64_u8 (vqtbl1q_u8 ( + vreinterpretq_u8_u64(desc[1]), shuf_msk1))'s\'\;
Quoted 2 lines
I will fix in next-net but my concern is why this has been not caught by any of our automated builds?
That series was flagged for error with Travis:
http://mails.dpdk.org/archives/test-report/2020-November/167602.html
Unfortunately, the build seems to have been purged (since it's from November). But Travis did flag the build as failing. With github actions we hope to pull the full logs into the email.
Quoted 2 lines
In patchwork only test report seems from the 'checkpatch':
https://patches.dpdk.org/patch/84260/
At least the 0-day robot does not submit each patch for separate build. We did that at first, and the robot's queue reached a week of backlog because the build takes a while. Especially when we get 20+ patch series followed by v2-v4 fixing build errors or compile errors.
PATCH 3/4 net/virtio: add vectorized packed ring Tx NEON path
In reply to Joyce KongOptimize packed ring Tx batch path with NEON instructions.
--- drivers/net/virtio/virtio_rxtx_packed.h | 6 +- drivers/net/virtio/virtio_rxtx_packed_neon.h | 143 +++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-)
drivers/net/virtio/virtio_rxtx_packed.h +5 -1
@@ -28,6 +28,8 @@/* flag bits offset in packed ring desc from ID */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#define FLAGS_LEN_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
@@ -36,13 +38,15 @@/* reference count offset in mbuf rearm data */ #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) + +#ifdef CC_AVX512_SUPPORT /* segment number offset in mbuf rearm data */ #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) -/* default rearm data */ #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \ 1ULL << REFCNT_BITS_OFFSET) +#endif /* id bits offset in packed ring desc higher 64bits */ #define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
drivers/net/virtio/virtio_rxtx_packed_neon.h +143 -0
149 lines not on this page · raw message
Signature · 1 line
2.28.0
Re: [dpdk-dev] [PATCH v1 3/4] net/virtio: add vectorized packed ring Tx NEON path
Quoted 184 lines · Joyce Kong
Optimize packed ring Tx batch path with NEON instructions.
Signed-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- drivers/net/virtio/virtio_rxtx_packed.h | 6 +- drivers/net/virtio/virtio_rxtx_packed_neon.h | 143 +++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-)drivers/net/virtio/virtio_rxtx_packed.h +5
-1@@ -28,6 +28,8 @@/* flag bits offset in packed ring desc from ID */ #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ offsetof(struct vring_packed_desc, id)) * BYTE_SIZE) +#define FLAGS_LEN_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \ + offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)#endif #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \@@ -36,13 +38,15 @@/* reference count offset in mbuf rearm data */ #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) + +#ifdef CC_AVX512_SUPPORT /* segment number offset in mbuf rearm data */ #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE) -/* default rearm data */ #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \ 1ULL << REFCNT_BITS_OFFSET) +#endif /* id bits offset in packed ring desc higher 64bits */ #define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \drivers/net/virtio/virtio_rxtx_packed_neon.h +136
-0@@ -16,6 +16,149 @@#include "virtio_rxtx_packed.h" #include "virtqueue.h" +static inline int +virtqueue_enqueue_batch_packed_vec (struct virtnet_tx *txvq, +struct rte_mbuf **tx_pkts) +{ +struct virtqueue *vq = txvq->vq; +uint16_t head_size = vq->hw->vtnet_hdr_size; +uint16_t idx = vq->vq_avail_idx; +struct virtio_net_hdr *hdr; +struct vq_desc_extra *dxp; +struct vring_packed_desc *p_desc; +uint16_t i; + +if (idx &PACKED_BATCH_MASK ) +return -1 ; + +if (unlikely ((idx +PACKED_BATCH_SIZE ) > vq->vq_nentries)) +return -1 ; + +/* Map four refcnt and nb_segs from mbufs to one NEON register. */ +uint8x16_t ref_seg_msk = { +2 ,3 ,4 ,5 , +10 ,11 ,12 ,13 , +18 ,19 ,20 ,21 , +26 ,27 ,28 ,29 + }; + +/* Map four data_off from mbufs to one NEON register. */ +uint8x8_t data_msk = { +0 ,1 , +8 ,9 , +16 ,17 , +24 ,25 + }; + +uint16x8_t net_hdr_msk = { +0xFFFF ,0xFFFF , +0 ,0 ,0 ,0 + }; + +uint16x4_t pkts[PACKED_BATCH_SIZE ]; +uint8x16x2_t mbuf; +/* Load four mbufs rearm data. */ +RTE_BUILD_BUG_ON (REFCNT_BITS_OFFSET >=64 ); + pkts[0 ] =vld1_u16 ((uint16_t *)&tx_pkts[0 ]->rearm_data); + pkts[1 ] =vld1_u16 ((uint16_t *)&tx_pkts[1 ]->rearm_data); + pkts[2 ] =vld1_u16 ((uint16_t *)&tx_pkts[2 ]->rearm_data); + pkts[3 ] =vld1_u16 ((uint16_t *)&tx_pkts[3 ]->rearm_data); + + mbuf.val[0 ] =vreinterpretq_u8_u16 (vcombine_u16 (pkts[0 ], pkts[1 ])); + mbuf.val[1 ] =vreinterpretq_u8_u16 (vcombine_u16 (pkts[2 ], pkts[3 ])); + +/* refcnt = 1 and nb_segs = 1 */ +uint32x4_t def_ref_seg =vdupq_n_u32 (0x10001 ); +/* Check refcnt and nb_segs. */ +uint32x4_t ref_seg =vreinterpretq_u32_u8 (vqtbl2q_u8 (mbuf, ref_seg_msk)); +poly128_t cmp1 =vreinterpretq_p128_u32 (~vceqq_u32 (ref_seg, def_ref_seg)); +if (unlikely (cmp1)) +return -1 ; + +/* Check headroom is enough. */ +uint16x4_t head_rooms =vdup_n_u16 (head_size); +RTE_BUILD_BUG_ON (offsetof(struct rte_mbuf , data_off) != + offsetof(struct rte_mbuf , rearm_data)); +uint16x4_t data_offset =vreinterpret_u16_u8 (vqtbl2_u8 (mbuf, data_msk)); +uint64x1_t cmp2 =vreinterpret_u64_u16 (vclt_u16 (data_offset, head_rooms)); +if (unlikely (vget_lane_u64 (cmp2,0 ))) +return -1 ; + +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { + dxp = &vq->vq_descx[idx + i]; + dxp->ndescs =1 ; + dxp->cookie = tx_pkts[i]; + } + +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { + tx_pkts[i]->data_off -= head_size; + tx_pkts[i]->data_len += head_size; + } + +uint64x2x2_t desc[PACKED_BATCH_SIZE /2 ]; +uint64x2_t base_addr0 = { +VIRTIO_MBUF_ADDR (tx_pkts[0 ], vq) + tx_pkts[0 ]->data_off, +VIRTIO_MBUF_ADDR (tx_pkts[1 ], vq) + tx_pkts[1 ]->data_off + }; +uint64x2_t base_addr1 = { +VIRTIO_MBUF_ADDR (tx_pkts[2 ], vq) + tx_pkts[2 ]->data_off, +VIRTIO_MBUF_ADDR (tx_pkts[3 ], vq) + tx_pkts[3 ]->data_off + }; + + desc[0 ].val[0 ] = base_addr0; + desc[1 ].val[0 ] = base_addr1; + +uint64_t flags = (uint64_t )vq->vq_packed.cached_flags <<FLAGS_LEN_BITS_OFFSET ; +uint64x2_t tx_desc0 = { + flags | (uint64_t )idx <<ID_BITS_OFFSET | tx_pkts[0 ]->data_len, + flags | (uint64_t )(idx +1 ) <<ID_BITS_OFFSET | tx_pkts[1 ]->data_len + }; + +uint64x2_t tx_desc1 = { + flags | (uint64_t )(idx +2 ) <<ID_BITS_OFFSET | tx_pkts[2 ]->data_len, + flags | (uint64_t )(idx +3 ) <<ID_BITS_OFFSET | tx_pkts[3 ]->data_len + }; + + desc[0 ].val[1 ] = tx_desc0; + desc[1 ].val[1 ] = tx_desc1; + +if (!vq->hw->has_tx_offload) { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { + hdr =rte_pktmbuf_mtod_offset (tx_pkts[i], + struct virtio_net_hdr *, -head_size); +/* Clear net hdr. */ +uint16x8_t v_hdr =vld1q_u16 ((void *)hdr); +vst1q_u16 ((void *)hdr,vandq_u16 (v_hdr, net_hdr_msk)); + } + }else { +virtio_for_each_try_unroll (i,0 ,PACKED_BATCH_SIZE ) { + hdr =rte_pktmbuf_mtod_offset (tx_pkts[i], + struct virtio_net_hdr *, -head_size); +virtqueue_xmit_offload (hdr, tx_pkts[i], true); + } + } + +/* Enqueue packet buffers. */ + p_desc = &vq->vq_packed.ring.desc[idx]; +vst2q_u64 ((uint64_t *)p_desc, desc[0 ]); +vst2q_u64 ((uint64_t *)(p_desc +2 ), desc[1 ]); + +virtio_update_batch_stats (&txvq->stats, tx_pkts[0 ]->pkt_len, + tx_pkts[1 ]->pkt_len, tx_pkts[2 ]->pkt_len, + tx_pkts[3 ]->pkt_len); + + vq->vq_avail_idx +=PACKED_BATCH_SIZE ; + vq->vq_free_cnt -=PACKED_BATCH_SIZE ; + +if (vq->vq_avail_idx >= vq->vq_nentries) { + vq->vq_avail_idx -= vq->vq_nentries;
drivers/net/virtio/virtio_rxtx_packed_neon.h line 164
Quoted 10 lines
drivers/net/virtio/virtio_rxtx_packed_neon.h +7
-0+ vq->vq_packed.cached_flags ^= +VRING_PACKED_DESC_F_AVAIL_USED ; + } + +return 0 ; +} +static inline uint16_t virtqueue_dequeue_batch_packed_vec (struct virtnet_rx *rxvq,struct rte_mbuf **rx_pkts)
Thanks,
Maxime
PATCH 4/4 net/virtio: add election for packed vector NEON path
In reply to Joyce KongAdd NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.
Packed ring vectorized neon path need:
NEON is supported by compiler and host VERSION_1 and IN_ORDER features are negotiated mergeable feature is not negotiated LRO offloading is disabled
--- doc/guides/nics/virtio.rst | 6 +++--- drivers/net/virtio/meson.build | 1 + drivers/net/virtio/virtio_ethdev.c | 19 +++++++++++++++---- drivers/net/virtio/virtio_rxtx_packed.c | 2 ++ drivers/net/virtio/virtio_user_ethdev.c | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-)
doc/guides/nics/virtio.rst +3 -3
@@ -483,11 +483,11 @@ according to below configuration: #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. #. Packed virtqueue vectorized Rx path: If building and running environment support - AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && - TCP_LRO Rx offloading is disabled && vectorized option enabled, + (AVX512 || NEON) && in-order feature is negotiated && Rx mergeable + is not negotiated && TCP_LRO Rx offloading is disabled && vectorized option enabled, this path will be selected. #. Packed virtqueue vectorized Tx path: If building and running environment support - AVX512 && in-order feature is negotiated && vectorized option enabled, + (AVX512 || NEON) && in-order feature is negotiated && vectorized option enabled, this path will be selected. Rx/Tx callbacks of each Virtio path
drivers/net/virtio/meson.build +1 -0
@@ -32,6 +32,7 @@ if arch_subdir == 'x86' elif arch_subdir == 'ppc' sources += files('virtio_rxtx_simple_altivec.c') elif arch_subdir == 'arm' and host_machine.cpu_family().startswith('aarch64') + sources += files('virtio_rxtx_packed.c') sources += files('virtio_rxtx_simple_neon.c') endif
drivers/net/virtio/virtio_ethdev.c +15 -4
@@ -1967,12 +1967,12 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)if (!vtpci_packed_queue (hw)) { hw->use_vec_rx =1 ; }else { -#if !defined(CC_AVX512_SUPPORT ) -PMD_DRV_LOG (INFO , -"building environment do not support packed ring vectorized" ); -#else +#if defined(CC_AVX512_SUPPORT ) || defined(RTE_ARCH_ARM ) hw->use_vec_rx =1 ; hw->use_vec_tx =1 ; +#else +PMD_DRV_LOG (INFO , +"building environment do not support packed ring vectorized" );#endif } }
@@ -2320,6 +2320,17 @@ virtio_dev_configure(struct rte_eth_dev *dev) hw->use_vec_rx =0 ; hw->use_vec_tx = 0; } +#elif defined(RTE_ARCH_ARM) +if ((hw->use_vec_rx || hw->use_vec_tx) && + (!rte_cpu_get_flag_enabled (RTE_CPUFLAG_NEON ) || + !vtpci_with_feature (hw,VIRTIO_F_IN_ORDER ) || + !vtpci_with_feature (hw,VIRTIO_F_VERSION_1 ) || +rte_vect_get_max_simd_bitwidth () <RTE_VECT_SIMD_128 )) { +PMD_DRV_LOG (INFO , +"disabled packed ring vectorized path for requirements not met" ); + hw->use_vec_rx =0 ; + hw->use_vec_tx =0 ; + }#else hw->use_vec_rx =0 ; hw->use_vec_tx =0 ;
drivers/net/virtio/virtio_rxtx_packed.c +2 -0
@@ -18,6 +18,8 @@#ifdef CC_AVX512_SUPPORT #include "virtio_rxtx_packed_avx.h" +#elif defined(RTE_ARCH_ARM ) +#include "virtio_rxtx_packed_neon.h" #endif uint16_t
drivers/net/virtio/virtio_user_ethdev.c +1 -1
@@ -856,7 +856,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)if (vectorized) {if (packed_vq) { -#if defined(CC_AVX512_SUPPORT ) +#if defined(CC_AVX512_SUPPORT ) || defined(RTE_ARCH_ARM ) hw->use_vec_rx =1 ; hw->use_vec_tx =1 ;#else
Signature · 1 line
2.28.0
Re: [dpdk-dev] [PATCH v1 4/4] net/virtio: add election for packed vector NEON path
Quoted 11 lines · Joyce Kong
Add NEON vectorized path selection logic. Default setting comes from vectorized devarg, then checks each criteria.
Packed ring vectorized neon path need:
NEON is supported by compiler and hostVERSION_1andIN_ORDERfeatures are negotiated mergeable feature is not negotiated LRO offloading is disabledSigned-off-by Joyce Kong <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>
Quoted 8 lines
--- doc/guides/nics/virtio.rst | 6 +++--- drivers/net/virtio/meson.build | 1 + drivers/net/virtio/virtio_ethdev.c | 19 +++++++++++++++---- drivers/net/virtio/virtio_rxtx_packed.c | 2 ++ drivers/net/virtio/virtio_user_ethdev.c | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-)
Thanks,
Maxime
Re: [dpdk-dev] [PATCH v1 0/4] Vectorize packed ring RX/TX path with NEON
In reply to Joyce KongQuoted 13 lines · Joyce Kong
This patch set introduces vectorized RX/TX path for packed ring with NEON intrinsics.
With this patch set, PVP case has 1.5% perf uplift for the packed vectorized NEON path compared with the non-vector packed path, under 0.001% acceptable loss with 2 cores on vhost side and 1 core on virtio side.
Joyce Kong (4):
net/virtio: move AVX based Rx and Tx code to separate file net/virtio: add vectorized packed ring Rx NEON path net/virtio: add vectorized packed ring Tx NEON path net/virtio: add election for packed vector NEON path
Quoted 17 lines
doc/guides/nics/virtio.rst | 6 +- drivers/net/virtio/meson.build | 5 +- drivers/net/virtio/virtio_ethdev.c | 19 +- drivers/net/virtio/virtio_rxtx.c | 6 +- drivers/net/virtio/virtio_rxtx_packed.c | 139 ++++ drivers/net/virtio/virtio_rxtx_packed.h | 317 ++++++++++ drivers/net/virtio/virtio_rxtx_packed_avx.c | 626 ------------------- drivers/net/virtio/virtio_rxtx_packed_avx.h | 239 +++++++ drivers/net/virtio/virtio_rxtx_packed_neon.h | 293 +++++++++ drivers/net/virtio/virtio_user_ethdev.c | 2 +- 10 files changed, 1012 insertions(+), 640 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed.h delete mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.h create mode 100644 drivers/net/virtio/virtio_rxtx_packed_neon.h
Series applied to dpdk-next-virtio/main.
Thanks,
Maxime