net: optimize raw checksum computation
Subsystems (2)
PATCH v7 1/1 net: optimize raw checksum computation
From: Scott Mitchell <[email protected]>
__rte_raw_cksum uses a loop with memcpy on each iteration. GCC 15+ is able to vectorize the loop but Clang 18.1 is not. Replacing the memcpy with unaligned_uint16_t pointer access enables both GCC and Clang to vectorize with SSE/AVX/AVX-512.
Performance results from cksum_perf_autotest on Intel Xeon (Cascade Lake, AVX-512) built with Clang 18.1 (TSC cycles/byte):
Block size Before After Improvement
100 0.40 0.24 ~40%
1500 0.50 0.06 ~8x
9000 0.49 0.06 ~8x---
Changes in v7:
- Replaced pointer arithmetic with RTE_PTR_ADD/RTE_ALIGN_FLOOR for DPDK consistency
Changes in v6:
- Fixed GCC -Wmaybe-uninitialized false positive in mlx5 driver
Changes in v5:
- Replaced memcpy loop with direct pointer access to enable vectorization
- Fixed GCC -Wmaybe-uninitialized false-positive warnings in rte_ipv4_phdr_cksum and rte_ipv6_phdr_cksum using compiler barriers (no assembly impact)
- Refactored hinic driver's local phdr_cksum implementations to call common functions, eliminating duplication
- Simplified phdr_cksum struct initialization with designated initializers
Changes in v4:
- Replaced manual 64-byte loop unrolling with simple pointer iteration
- Removed __rte_no_ubsan_alignment macro (no longer needed)
- Removed rte_ip6.h false-positive warning fix (unable to repro locally)
- Updated performance numbers: AVX-512 vectorization on Clang achieves ~8x improvement for large packets vs scalar implementation
Changes in v3:
- Added __rte_no_ubsan_alignment macro to suppress false-positive UBSAN alignment warnings when using unaligned_uint16_t
- Fixed false-positive GCC maybe-uninitialized warning in rte_ip6.h exposed by optimization (can be split to separate patch once verified on CI)
Changes in v2:
- Fixed UndefinedBehaviorSanitizer errors by adding uint32_t casts to prevent signed integer overflow in addition chains
- Restored uint32_t sum accumulator instead of uint64_t
- Added 64k length to test_cksum_perf.c
app/test/meson.build | 1 + app/test/test_cksum_fuzz.c | 241 +++++++++++++++++++++++++++++++ app/test/test_cksum_perf.c | 2 +- drivers/net/hinic/hinic_pmd_tx.c | 38 +---- drivers/net/mlx5/mlx5_flow_dv.c | 4 + lib/net/rte_cksum.h | 17 +-- lib/net/rte_ip4.h | 28 ++-- lib/net/rte_ip6.h | 19 +-- 8 files changed, 278 insertions(+), 72 deletions(-) create mode 100644 app/test/test_cksum_fuzz.c
app/test/meson.build +1 -0
@@ -38,6 +38,7 @@ source_file_deps = { 'test_byteorder.c': [], 'test_cfgfile.c': ['cfgfile'], 'test_cksum.c': ['net'], + 'test_cksum_fuzz.c': ['net'], 'test_cksum_perf.c': ['net'], 'test_cmdline.c': [], 'test_cmdline_cirbuf.c': [],
app/test/test_cksum_fuzz.c new file +241 -0
241 lines not on this page · raw message
app/test/test_cksum_perf.c +1 -1
@@ -15,7 +15,7 @@#define NUM_BLOCKS 10#define ITERATIONS 1000000 -static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 }; +static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 ,9000 ,9001 ,65536 ,65537 }; static __rte_noinline uint16_tdo_rte_raw_cksum (const void *buf,size_t len)
drivers/net/hinic/hinic_pmd_tx.c +2 -36
@@ -706,47 +706,13 @@ hinic_get_sq_wqe(struct hinic_txq *txq, int wqebb_cnt,static inline uint16_t hinic_ipv4_phdr_cksum (const struct rte_ipv4_hdr *ipv4_hdr,uint64_t ol_flags) { -struct ipv4_psd_header { -uint32_t src_addr;/* IP address of source host. */ -uint32_t dst_addr;/* IP address of destination host. */ -uint8_t zero;/* zero. */ -uint8_t proto;/* L4 protocol type. */ -uint16_t len;/* L4 length. */ - } psd_hdr; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) { - psd_hdr.len =0 ; - }else { - psd_hdr.len = -rte_cpu_to_be_16 (rte_be_to_cpu_16 (ipv4_hdr->total_length) - -rte_ipv4_hdr_len (ipv4_hdr)); - } -return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); +return rte_ipv4_phdr_cksum (ipv4_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); }static inline uint16_t hinic_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum; -struct { -uint32_t len;/* L4 length. */ -uint32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (ipv6_hdr->proto <<24 ); -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; - - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, -sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); - sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum); -return __rte_raw_cksum_reduce (sum); +return rte_ipv6_phdr_cksum (ipv6_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); } static inline void hinic_get_outer_cs_pld_offset(struct rte_mbuf *m,
drivers/net/mlx5/mlx5_flow_dv.c +4 -0
@@ -4445,6 +4445,10 @@ __flow_encap_decap_resource_register(struct rte_eth_dev *dev, .reserve = 0, } }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (encap_decap_key)); +#endif struct mlx5_flow_cb_ctx ctx = { .error = error, .data = resource,
lib/net/rte_cksum.h +6 -11
@@ -42,20 +42,15 @@ extern "C" {static inline uint32_t __rte_raw_cksum (const void *buf,size_t len,uint32_t sum) { -const void *end; - -for (end =RTE_PTR_ADD (buf,RTE_ALIGN_FLOOR (len,sizeof (uint16_t ))); - buf != end; buf =RTE_PTR_ADD (buf,sizeof (uint16_t ))) { -uint16_t v; - -memcpy (&v, buf,sizeof (uint16_t )); - sum += v; - } +/* Process uint16 chunks to preserve overflow/carry math. GCC/Clang vectorize the loop. */ +const void *end =RTE_PTR_ADD (buf,RTE_ALIGN_FLOOR (len,sizeof (uint16_t ))); +for (const unaligned_uint16_t *buf16 = (const unaligned_uint16_t *)buf; + buf16 != end; buf16++) + sum += *buf16;/* if length is odd, keeping it byte order independent */ -if (unlikely (len %2 )) { +if (len &1 ) {uint16_t left =0 ; -memcpy (&left, end,1 ); sum += left; }
lib/net/rte_ip4.h +13 -15
@@ -223,21 +223,19 @@ rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)uint8_t zero;/* zero. */ uint8_t proto;/* L4 protocol type. */ uint16_t len;/* L4 length. */ - } psd_hdr; - -uint32_t l3_len; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) { - psd_hdr.len =0 ; - }else { - l3_len =rte_be_to_cpu_16 (ipv4_hdr->total_length); - psd_hdr.len =rte_cpu_to_be_16 ((uint16_t )(l3_len - -rte_ipv4_hdr_len (ipv4_hdr))); - } + } psd_hdr = { + .src_addr = ipv4_hdr->src_addr, + .dst_addr = ipv4_hdr->dst_addr, + .proto = ipv4_hdr->next_proto_id, + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) ?0 : +rte_cpu_to_be_16 ((uint16_t )(rte_be_to_cpu_16 (ipv4_hdr->total_length) - +rte_ipv4_hdr_len (ipv4_hdr))) + }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (psd_hdr)); +#endif +return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); }
lib/net/rte_ip6.h +10 -9
@@ -560,19 +560,20 @@ rte_ipv6_phdr_cksum(const struct rte_ipv6_hdr *ipv6_hdr, uint64_t ol_flags)static inline uint16_t rte_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum;struct {rte_be32_t len;/* L4 length. */ rte_be32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (uint32_t )(ipv6_hdr->proto <<24 ); -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; + } psd_hdr = { + .proto = (uint32_t )(ipv6_hdr->proto <<24 ), + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) ?0 : + ipv6_hdr->payload_len + }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (psd_hdr)); +#endif - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, +uint32_t sum =__rte_raw_cksum (&ipv6_hdr->src_addr,sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum);
Signature · 1 line
2.39.5 (Apple Git-154)
Quoted 22 lines · On Thu, 8 Jan 2026 09:59:13 -0500 [email protected] wrote:
lib/net/rte_ip4.h +0
-14@@ -223,21 +223,19 @@ rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)uint8_t zero;/* zero. */ uint8_t proto;/* L4 protocol type. */ uint16_t len;/* L4 length. */ - } psd_hdr; - -uint32_t l3_len; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) { - psd_hdr.len =0 ; - }else { - l3_len =rte_be_to_cpu_16 (ipv4_hdr->total_length); - psd_hdr.len =rte_cpu_to_be_16 ((uint16_t )(l3_len - -rte_ipv4_hdr_len (ipv4_hdr)));
lib/net/rte_ip4.h rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags) · line 234
Quoted 10 lines
lib/net/rte_ip4.h +9
-1- } + } psd_hdr = { + .src_addr = ipv4_hdr->src_addr, + .dst_addr = ipv4_hdr->dst_addr, + .proto = ipv4_hdr->next_proto_id, + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) ?0 : +rte_cpu_to_be_16 ((uint16_t )(rte_be_to_cpu_16 (ipv4_hdr->total_length) - +rte_ipv4_hdr_len (ipv4_hdr))) + }; +#ifdef RTE_
I find it hard to read complex expressions in initialiazers. Much better as original if statement.
Sometimes you need a union to convince compiler of aliasing
Quoted 32 lines · Stephen Hemminger
Quoted 28 lines · On Thu, 8 Jan 2026 09:59:13 -0500 [email protected] wrote:
lib/net/rte_ip4.h +5
-15@@ -223,21 +223,19 @@ rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)uint8_t zero;/* zero. */ uint8_t proto;/* L4 protocol type. */ uint16_t len;/* L4 length. */ - } psd_hdr; - -uint32_t l3_len; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) { - psd_hdr.len =0 ; - }else { - l3_len =rte_be_to_cpu_16 (ipv4_hdr->total_length); - psd_hdr.len =rte_cpu_to_be_16 ((uint16_t )(l3_len - -rte_ipv4_hdr_len (ipv4_hdr))); - } + } psd_hdr = { + .src_addr = ipv4_hdr->src_addr, + .dst_addr = ipv4_hdr->dst_addr, + .proto = ipv4_hdr->next_proto_id, + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG)) ? 0 :
Quoted 9 lines
lib/net/rte_ip4.h rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)
Quoted 4 lines
lib/net/rte_ip4.h +4
-0+rte_cpu_to_be_16 ((uint16_t )(rte_be_to_cpu_16 (ipv4_hdr->total_length) - + rte_ipv4_hdr_len(ipv4_hdr))) + }; +#ifdef RTE_ I find it hard to read complex expressions in initialiazers. Much better as original if statement.
Sometimes you need a union to convince compiler of aliasing
I improved the readability of the ternary in the v8 patch (coming soon). Rational for using an initializer is to lean on the c99 spec which guarantees the
memory is initialized. Given the ambiguity around "maybe uninitialized variable use" I made the code in question explicit on initialization. Let me know
if the approach in v8 patch is not inline with dpdk standards/convention and I can rework.