net: optimize raw checksum computation
Subsystems (2)
PATCH v6 1/1 net: optimize raw checksum computation
From: Scott Mitchell <[email protected]>
__rte_raw_cksum uses a loop with memcpy on each iteration. GCC 15+ is able to vectorize the loop but Clang 18.1 is not. Replacing the memcpy with unaligned_uint16_t pointer access enables both GCC and Clang to vectorize with SSE/AVX/AVX-512.
Performance results from cksum_perf_autotest on Intel Xeon (Cascade Lake, AVX-512) built with Clang 18.1 (TSC cycles/byte):
Block size Before After Improvement
100 0.40 0.24 ~40%
1500 0.50 0.06 ~8x
9000 0.49 0.06 ~8x---
Changes in v6:
- Fixed GCC -Wmaybe-uninitialized false positive in mlx5 driver
Changes in v5:
- Simplified __rte_raw_cksum pointer arithmetic using type-safe uint16_t* instead of RTE_PTR_ADD byte arithmetic
- Fixed GCC -Wmaybe-uninitialized false-positive warnings in rte_ipv4_phdr_cksum and rte_ipv6_phdr_cksum using compiler barriers (no assembly impact)
- Refactored hinic driver's local phdr_cksum implementations to call common functions, eliminating duplication
- Simplified phdr_cksum struct initialization with designated initializers
Changes in v4:
- Replaced manual 64-byte loop unrolling with simple pointer iteration
- Removed __rte_no_ubsan_alignment macro (no longer needed)
- Removed rte_ip6.h false-positive warning fix (unable to repro locally)
- Updated performance numbers: AVX-512 vectorization on Clang achieves ~8x improvement for large packets vs scalar implementation
Changes in v3:
- Added __rte_no_ubsan_alignment macro to suppress false-positive UBSAN alignment warnings when using unaligned_uint16_t
- Fixed false-positive GCC maybe-uninitialized warning in rte_ip6.h exposed by optimization (can be split to separate patch once verified on CI)
Changes in v2:
- Fixed UndefinedBehaviorSanitizer errors by adding uint32_t casts to prevent signed integer overflow in addition chains
- Restored uint32_t sum accumulator instead of uint64_t
- Added 64k length to test_cksum_perf.c
app/test/meson.build | 1 + app/test/test_cksum_fuzz.c | 241 +++++++++++++++++++++++++++++++ app/test/test_cksum_perf.c | 2 +- drivers/net/hinic/hinic_pmd_tx.c | 38 +---- drivers/net/mlx5/mlx5_flow_dv.c | 4 + lib/net/rte_cksum.h | 19 +-- lib/net/rte_ip4.h | 28 ++-- lib/net/rte_ip6.h | 19 +-- 8 files changed, 280 insertions(+), 72 deletions(-) create mode 100644 app/test/test_cksum_fuzz.c
app/test/meson.build +1 -0
@@ -38,6 +38,7 @@ source_file_deps = { 'test_byteorder.c': [], 'test_cfgfile.c': ['cfgfile'], 'test_cksum.c': ['net'], + 'test_cksum_fuzz.c': ['net'], 'test_cksum_perf.c': ['net'], 'test_cmdline.c': [], 'test_cmdline_cirbuf.c': [],
app/test/test_cksum_fuzz.c new file +241 -0
241 lines not on this page · raw message
app/test/test_cksum_perf.c +1 -1
@@ -15,7 +15,7 @@#define NUM_BLOCKS 10#define ITERATIONS 1000000 -static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 }; +static const size_t data_sizes[] = {20 ,21 ,100 ,101 ,1500 ,1501 ,9000 ,9001 ,65536 ,65537 }; static __rte_noinline uint16_tdo_rte_raw_cksum (const void *buf,size_t len)
drivers/net/hinic/hinic_pmd_tx.c +2 -36
@@ -706,47 +706,13 @@ hinic_get_sq_wqe(struct hinic_txq *txq, int wqebb_cnt,static inline uint16_t hinic_ipv4_phdr_cksum (const struct rte_ipv4_hdr *ipv4_hdr,uint64_t ol_flags) { -struct ipv4_psd_header { -uint32_t src_addr;/* IP address of source host. */ -uint32_t dst_addr;/* IP address of destination host. */ -uint8_t zero;/* zero. */ -uint8_t proto;/* L4 protocol type. */ -uint16_t len;/* L4 length. */ - } psd_hdr; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) { - psd_hdr.len =0 ; - }else { - psd_hdr.len = -rte_cpu_to_be_16 (rte_be_to_cpu_16 (ipv4_hdr->total_length) - -rte_ipv4_hdr_len (ipv4_hdr)); - } -return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); +return rte_ipv4_phdr_cksum (ipv4_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); }static inline uint16_t hinic_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum; -struct { -uint32_t len;/* L4 length. */ -uint32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (ipv6_hdr->proto <<24 ); -if (ol_flags &RTE_MBUF_F_TX_TCP_SEG ) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; - - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, -sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); - sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum); -return __rte_raw_cksum_reduce (sum); +return rte_ipv6_phdr_cksum (ipv6_hdr, ol_flags &RTE_MBUF_F_TX_TCP_SEG ); } static inline void hinic_get_outer_cs_pld_offset(struct rte_mbuf *m,
drivers/net/mlx5/mlx5_flow_dv.c +4 -0
@@ -4445,6 +4445,10 @@ __flow_encap_decap_resource_register(struct rte_eth_dev *dev, .reserve = 0, } }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (encap_decap_key)); +#endif struct mlx5_flow_cb_ctx ctx = { .error = error, .data = resource,
lib/net/rte_cksum.h +8 -11
@@ -42,20 +42,17 @@ extern "C" {static inline uint32_t __rte_raw_cksum (const void *buf,size_t len,uint32_t sum) { -const void *end; - -for (end =RTE_PTR_ADD (buf,RTE_ALIGN_FLOOR (len,sizeof (uint16_t ))); - buf != end; buf =RTE_PTR_ADD (buf,sizeof (uint16_t ))) { -uint16_t v; - -memcpy (&v, buf,sizeof (uint16_t )); - sum += v; - } +/* Process as uint16 chunks to preserve overflow/carry math. */ +const unaligned_uint16_t *buf16 = (const unaligned_uint16_t *)buf; +/* Divide byte length by 2 to get number of uint16_t elements. */ +const unaligned_uint16_t *end = buf16 + (len >>1 ); +/* GCC and Clang vectorize the loop. */ +for (; buf16 != end; buf16++) + sum += *buf16;/* if length is odd, keeping it byte order independent */ -if (unlikely (len %2 )) { +if (len &1 ) {uint16_t left =0 ; -memcpy (&left, end,1 ); sum += left; }
lib/net/rte_ip4.h +13 -15
@@ -223,21 +223,19 @@ rte_ipv4_phdr_cksum(const struct rte_ipv4_hdr *ipv4_hdr, uint64_t ol_flags)uint8_t zero;/* zero. */ uint8_t proto;/* L4 protocol type. */ uint16_t len;/* L4 length. */ - } psd_hdr; - -uint32_t l3_len; - - psd_hdr.src_addr = ipv4_hdr->src_addr; - psd_hdr.dst_addr = ipv4_hdr->dst_addr; - psd_hdr.zero =0 ; - psd_hdr.proto = ipv4_hdr->next_proto_id; -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) { - psd_hdr.len =0 ; - }else { - l3_len =rte_be_to_cpu_16 (ipv4_hdr->total_length); - psd_hdr.len =rte_cpu_to_be_16 ((uint16_t )(l3_len - -rte_ipv4_hdr_len (ipv4_hdr))); - } + } psd_hdr = { + .src_addr = ipv4_hdr->src_addr, + .dst_addr = ipv4_hdr->dst_addr, + .proto = ipv4_hdr->next_proto_id, + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) ?0 : +rte_cpu_to_be_16 ((uint16_t )(rte_be_to_cpu_16 (ipv4_hdr->total_length) - +rte_ipv4_hdr_len (ipv4_hdr))) + }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (psd_hdr)); +#endif +return rte_raw_cksum (&psd_hdr,sizeof (psd_hdr)); }
lib/net/rte_ip6.h +10 -9
@@ -560,19 +560,20 @@ rte_ipv6_phdr_cksum(const struct rte_ipv6_hdr *ipv6_hdr, uint64_t ol_flags)static inline uint16_t rte_ipv6_phdr_cksum (const struct rte_ipv6_hdr *ipv6_hdr,uint64_t ol_flags) { -uint32_t sum;struct {rte_be32_t len;/* L4 length. */ rte_be32_t proto;/* L4 protocol - top 3 bytes must be zero */ - } psd_hdr; - - psd_hdr.proto = (uint32_t )(ipv6_hdr->proto <<24 ); -if (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) - psd_hdr.len =0 ; -else - psd_hdr.len = ipv6_hdr->payload_len; + } psd_hdr = { + .proto = (uint32_t )(ipv6_hdr->proto <<24 ), + .len = (ol_flags & (RTE_MBUF_F_TX_TCP_SEG |RTE_MBUF_F_TX_UDP_SEG )) ?0 : + ipv6_hdr->payload_len + }; +#ifdef RTE_CC_GCC +/* Suppress GCC -Wmaybe-uninitialized false positive. No assembly/runtime impacts. */ + asmvolatile ("" :"+m" (psd_hdr)); +#endif - sum =__rte_raw_cksum (&ipv6_hdr->src_addr, +uint32_t sum =__rte_raw_cksum (&ipv6_hdr->src_addr,sizeof (ipv6_hdr->src_addr) +sizeof (ipv6_hdr->dst_addr),0 ); sum =__rte_raw_cksum (&psd_hdr,sizeof (psd_hdr), sum);
Signature · 1 line
2.39.5 (Apple Git-154)