net: optimize raw checksum computation
Subsystems (2)
PATCH v10 1/1 net: optimize raw checksum computation
From: Scott Mitchell <[email protected]>
__rte_raw_cksum uses a loop with memcpy on each iteration. GCC 15+ is able to vectorize the loop but Clang 18.1 is not. Replacing the memcpy with unaligned_uint16_t pointer access enables both GCC and Clang to vectorize with SSE/AVX/AVX-512.
Performance results from cksum_perf_autotest on Intel Xeon (Cascade Lake, AVX-512) built with Clang 18.1 (TSC cycles/byte):
Block size Before After Improvement
100 0.40 0.24 ~40%
1500 0.50 0.06 ~8x
9000 0.49 0.06 ~8x---
Changes in v10:
- Fixed patch metadata format: removed literal separator markers from v9 changelog text that would be interpreted as patch delimiters.
Changes in v9:
- Fixed patch metadata format: v8 had duplicate separator which caused changelog to not render on patches.dpdk.org. v9 uses correct format.
Changes in v8:
- __rte_raw_cksum: use native pointer arithmetic instead of RTE_PTR_ADD to avoid incorrect results with -O3 for UDP checksums. Also improves performance due to less assembly generated with Clang.
- Added __rte_no_ubsan_alignment attribute to suppress false UBSAN warnings
- Added RTE_SUPPRESS_UNINITIALIZED_WARNING macro for GCC workaround
- Fixed C++ compilation errors in rte_ip4.h and rte_ip6.h
- Formatted complex ternary expressions for readability
Changes in v7:
- Replaced pointer arithmetic with RTE_PTR_ADD/RTE_ALIGN_FLOOR for DPDK consistency (BROKEN - Clang alias analysis bug causes incorrect UDP checksums)
Changes in v6:
- Fixed GCC -Wmaybe-uninitialized false positive in mlx5 driver
Changes in v5:
- Replaced memcpy loop with direct pointer access to enable vectorization
- Fixed GCC -Wmaybe-uninitialized false-positive warnings in rte_ipv4_phdr_cksum and rte_ipv6_phdr_cksum using compiler barriers (no assembly impact)
- Refactored hinic driver's local phdr_cksum implementations to call common functions, eliminating duplication
- Simplified phdr_cksum struct initialization with designated initializers
Changes in v4:
- Replaced manual 64-byte loop unrolling with simple pointer iteration
- Removed __rte_no_ubsan_alignment macro (no longer needed)
- Removed rte_ip6.h false-positive warning fix (unable to repro locally)
- Updated performance numbers: AVX-512 vectorization on Clang achieves ~8x improvement for large packets vs scalar implementation
Changes in v3:
- Added __rte_no_ubsan_alignment macro to suppress false-positive UBSAN alignment warnings when using unaligned_uint16_t
- Fixed false-positive GCC maybe-uninitialized warning in rte_ip6.h exposed by optimization (can be split to separate patch once verified on CI)
Changes in v2:
- Fixed UndefinedBehaviorSanitizer errors by adding uint32_t casts to prevent signed integer overflow in addition chains
- Restored uint32_t sum accumulator instead of uint64_t
- Added 64k length to test_cksum_perf.c
app/test/meson.build | 1 + app/test/test_cksum_fuzz.c | 240 +++++++++++++++++++++++++++++++ app/test/test_cksum_perf.c | 2 +- drivers/net/hinic/hinic_pmd_tx.c | 38 +---- drivers/net/mlx5/mlx5_flow_dv.c | 2 + lib/eal/include/rte_common.h | 22 +++ lib/net/rte_cksum.h | 18 +-- lib/net/rte_ip4.h | 26 ++-- lib/net/rte_ip6.h | 17 ++- 9 files changed, 294 insertions(+), 72 deletions(-) create mode 100644 app/test/test_cksum_fuzz.c
app/test/meson.build +1 -0
@@ -38,6 +38,7 @@ source_file_deps = { 'test_byteorder.c': [], 'test_cfgfile.c': ['cfgfile'], 'test_cksum.c': ['net'], + 'test_cksum_fuzz.c': ['net'], 'test_cksum_perf.c': ['net'], 'test_cmdline.c': [], 'test_cmdline_cirbuf.c': [],
app/test/test_cksum_fuzz.c new file +240 -0
240 lines not on this page · raw message
app/test/test_cksum_perf.c +1 -1
@@ -15,7 +15,7 @@#define NUM_BLOCKS 10#define ITERATIONS 1000000 -static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 }; +static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 ,9000 ,9001 ,65536 ,65537 }; static __rte_noinline uint16_tdo_rte_raw_cksum (const void *buf,size_t len)
drivers/net/hinic/hinic_pmd_tx.c +2 -36
@@ -706,47 +706,13 @@ hinic_get_sq_wqe(struct hinic_txq *txq, int wqebb_cnt,static inline uint16_t hinic_ipv4_phdr_cksum (const struct rte_ipv4_hdr *ipv4_hdr,uint64_t ol_flags) { -struct ipv4_psd_header { -uint32_t src_addr;/* IP address of source host. */ -uint32_t dst_addr;/* IP address of destination host. */ -uint8_t zero;/* zero. */ -uint8_t proto;/* L4 protocol type. */ -uint16_t len;/* L4 length. */ - } psd_hdr; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) { - psd_hdr.len =0 ; - }else { - psd_hdr.len = -rte_cpu_to_be_16 (rte_be_to_cpu_16 (ipv4_hdr->total_length) - -rte_ipv4_hdr_len (ipv4_hdr)); - } -return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); +return rte_ipv4_phdr_cksum (ipv4_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); }static inline uint16_t hinic_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum; -struct { -uint32_t len;/* L4 length. */ -uint32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (ipv6_hdr->proto <<24 ); -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; - - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, -sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); - sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum); -return __rte_raw_cksum_reduce (sum); +return rte_ipv6_phdr_cksum (ipv6_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); } static inline void hinic_get_outer_cs_pld_offset(struct rte_mbuf *m,
drivers/net/mlx5/mlx5_flow_dv.c +2 -0
@@ -4445,6 +4445,8 @@ __flow_encap_decap_resource_register(struct rte_eth_dev *dev, .reserve = 0, } }; +RTE_SUPPRESS_UNINITIALIZED_WARNING (encap_decap_key); +struct mlx5_flow_cb_ctx ctx = { .error = error, .data = resource,
lib/eal/include/rte_common.h +22 -0
@@ -546,6 +546,28 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void)#define __rte_no_asan#endif +/** +* Disable UndefinedBehaviorSanitizer alignment check on some code +*/ +#if defined(RTE_CC_CLANG ) || defined(RTE_CC_GCC ) +#define __rte_no_ubsan_alignment __attribute__((no_sanitize("alignment"))) +#else +#define __rte_no_ubsan_alignment +#endif + +/** +* Suppress GCC -Wmaybe-uninitialized false positive on struct initialization. +* This tells the compiler that the variable's memory has been touched, +* preventing the false positive without affecting other optimizations. +*/ +#ifdef RTE_CC_GCC +#define RTE_SUPPRESS_UNINITIALIZED_WARNING (var) do { \ + asm volatile("" : "+m" (var)); \ +} while (0) +#else +#define RTE_SUPPRESS_UNINITIALIZED_WARNING (var) +#endif +/*********** Macros for pointer arithmetic ********/ /**
lib/net/rte_cksum.h +7 -11
@@ -39,23 +39,19 @@ extern "C" {* @return * sum += Sum of all words in the buffer. */ +__rte_no_ubsan_alignment static inline uint32_t__rte_raw_cksum (const void *buf,size_t len,uint32_t sum) { -const void *end; - -for (end =RTE_PTR_ADD (buf,RTE_ALIGN_FLOOR (len,sizeof (uint16_t ))); - buf != end; buf =RTE_PTR_ADD (buf,sizeof (uint16_t ))) { -uint16_t v; - -memcpy (&v, buf,sizeof (uint16_t )); - sum += v; - } +/* Process uint16 chunks to preserve overflow/carry math. GCC/Clang vectorize the loop. */ +const unaligned_uint16_t *buf16 = (const unaligned_uint16_t *)buf; +const unaligned_uint16_t *end = buf16 + (len /sizeof (uint16_t )); +for (; buf16 != end; buf16++) + sum += *buf16;/* if length is odd, keeping it byte order independent */ -if (unlikely (len %2 )) { +if (len &1 ) {uint16_t left =0 ; -memcpy (&left, end,1 ); sum += left; }
lib/net/rte_ip4.h +11 -15
@@ -223,21 +223,17 @@ rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)uint8_t zero;/* zero. */ uint8_t proto;/* L4 protocol type. */ uint16_t len;/* L4 length. */ - } psd_hdr; - -uint32_t l3_len; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) { - psd_hdr.len =0 ; - }else { - l3_len =rte_be_to_cpu_16 (ipv4_hdr->total_length); - psd_hdr.len =rte_cpu_to_be_16 ((uint16_t )(l3_len - -rte_ipv4_hdr_len (ipv4_hdr))); - } + } psd_hdr = { + .src_addr = ipv4_hdr->src_addr, + .dst_addr = ipv4_hdr->dst_addr, + .proto = ipv4_hdr->next_proto_id, + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) + ? (uint16_t )0 + :rte_cpu_to_be_16 ((uint16_t )(rte_be_to_cpu_16 (ipv4_hdr->total_length) - +rte_ipv4_hdr_len (ipv4_hdr))) + }; +RTE_SUPPRESS_UNINITIALIZED_WARNING (psd_hdr); +return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); }
lib/net/rte_ip6.h +0 -0
@@ -560,19 +560,18 @@ rte_ipv6_phdr_cksum(const struct rte_ipv6_hdr *ipv6_hdr, uint64_t ol_flags)static inline uint16_t rte_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum;struct {rte_be32_t len;/* L4 length. */ rte_be32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (uint32_t )(ipv6_hdr->proto <<24 ); -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; + } psd_hdr = { + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) + ? (rte_be32_t )0 + : ipv6_hdr->payload_len, + .proto = (uint32_t )(ipv6_hdr->proto <<24 ) + }; +RTE_SUPPRESS_UNINITIALIZED_WARNING (psd_hdr); - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, +uint32_t sum =__rte_raw_cksum (&ipv6_hdr->src_addr,sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum); --
2.39.5 (Apple Git-154)