net: introduce net_aligned_data
Subsystems (5)
v2 cover net: introduce net_aligned_data
____cacheline_aligned_in_smp on small fields like tcp_memory_allocated and udp_memory_allocated is not good enough.
It makes sure to put these fields at the start of a cache line, but does not prevent the linker from using the cache line for other fields, with potential performance impact.
nm -v vmlinux|egrep -5 "tcp_memory_allocated|udp_memory_allocated"
...ffffffff83e35cc0 B tcp_memory_allocatedffffffff83e35cc8 b __key.0ffffffff83e35cc8 b __tcp_tx_delay_enabled.1ffffffff83e35ce0 b tcp_orphan_timer
...ffffffff849dddc0 B udp_memory_allocatedffffffff849dddc8 B udp_encap_needed_keyffffffff849dddd8 B udpv6_encap_needed_keyffffffff849dddf0 b inetsw_lock
One solution is to move these sensitive fields to a structure, so that the compiler is forced to add empty holes between each member.
nm -v vmlinux|egrep -2 "tcp_memory_allocated|udp_memory_allocated|net_aligned_data"
ffffffff885af970 b mem_id_initffffffff885af980 b __key.0ffffffff885af9c0 B net_aligned_dataffffffff885afa80 B page_pool_mem_providersffffffff885afa90 b __key.2
v2: added <linux/atomic.h>, report from kernel test robot <[email protected]>
v1: https://lore.kernel.org/netdev/[email protected]/
Eric Dumazet (4):
net: add struct net_aligned_data
net: move net_cookie into net_aligned_data
tcp: move tcp_memory_allocated into net_aligned_data
udp: move udp_memory_allocated into net_aligned_data
include/net/aligned_data.h | 22 ++++++++++++++++++++++ include/net/tcp.h | 1 - include/net/udp.h | 1 - net/core/hotdata.c | 5 +++++ net/core/net_namespace.c | 8 ++------ net/ipv4/tcp.c | 2 -- net/ipv4/tcp_ipv4.c | 3 ++- net/ipv4/udp.c | 4 +--- net/ipv4/udp_impl.h | 1 + net/ipv4/udplite.c | 2 +- net/ipv6/tcp_ipv6.c | 3 ++- net/ipv6/udp.c | 2 +- net/ipv6/udp_impl.h | 1 + net/ipv6/udplite.c | 2 +- net/mptcp/protocol.c | 3 ++- 15 files changed, 41 insertions(+), 19 deletions(-) create mode 100644 include/net/aligned_data.h
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
PATCH v2 1/4 net: add struct net_aligned_data
This structure will hold networking data that must consume a full cache line to avoid accidental false sharing.
--- include/net/aligned_data.h | 16 ++++++++++++++++ net/core/hotdata.c | 3 +++ 2 files changed, 19 insertions(+) create mode 100644 include/net/aligned_data.h
include/net/aligned_data.h new file +16 -0
@@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _NET_ALIGNED_DATA_H +#define _NET_ALIGNED_DATA_H + +#include <linux/types.h> + +/* Structure holding cacheline aligned fields on SMP builds. +* Each field or group should have an ____cacheline_aligned_in_smp +* attribute to ensure no accidental false sharing can happen. +*/ +struct net_aligned_data { +}; + +extern struct net_aligned_data net_aligned_data; + +#endif /* _NET_ALIGNED_DATA_H */
net/core/hotdata.c +3 -0
@@ -2,6 +2,7 @@#include <linux/cache.h> #include <linux/jiffies.h> #include <linux/list.h> +#include <net/aligned_data.h> #include <net/hotdata.h> #include <net/proto_memory.h>
@@ -22,3 +23,5 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE };EXPORT_SYMBOL (net_hotdata); + +struct net_aligned_data net_aligned_data;
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
PATCH v2 2/4 net: move net_cookie into net_aligned_data
In reply to Eric DumazetUsing per-cpu data for net->net_cookie generation is overkill, because even busy hosts do not create hundreds of netns per second.
Make sure to put net_cookie in a private cache line to avoid potential false sharing.
--- include/net/aligned_data.h | 2 ++ net/core/net_namespace.c | 8 ++------ 2 files changed, 4 insertions(+), 6 deletions(-)
include/net/aligned_data.h +2 -0
@@ -2,6 +2,7 @@#ifndef _NET_ALIGNED_DATA_H#define _NET_ALIGNED_DATA_H +#include <linux/atomic.h> #include <linux/types.h> /* Structure holding cacheline aligned fields on SMP builds.
@@ -9,6 +10,7 @@* attribute to ensure no accidental false sharing can happen. */ struct net_aligned_data { +atomic64_t net_cookie____cacheline_aligned_in_smp ; };extern struct net_aligned_data net_aligned_data;
net/core/net_namespace.c +2 -6
@@ -19,9 +19,9 @@#include <linux/net_namespace.h> #include <linux/sched/task.h> #include <linux/uidgid.h> -#include <linux/cookie.h> #include <linux/proc_fs.h> +#include <net/aligned_data.h> #include <net/sock.h> #include <net/netlink.h> #include <net/net_namespace.h>
@@ -64,8 +64,6 @@ DECLARE_RWSEM(pernet_ops_rwsem);static unsigned int max_gen_ptrs =INITIAL_NET_GEN_PTRS ; -DEFINE_COOKIE (net_cookie); -static struct net_generic *net_alloc_generic (void ) {unsigned int gen_ptrs =READ_ONCE (max_gen_ptrs);
@@ -434,9 +432,7 @@ static __net_init int setup_net(struct net *net)LIST_HEAD (net_exit_list);int error =0 ; -preempt_disable (); - net->net_cookie =gen_cookie_next (&net_cookie); -preempt_enable (); + net->net_cookie =atomic64_inc_return (&net_aligned_data.net_cookie);list_for_each_entry (ops, &pernet_list, list) { error =ops_init (ops, net);
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
PATCH v2 3/4 tcp: move tcp_memory_allocated into net_aligned_data
In reply to Eric Dumazet____cacheline_aligned_in_smp attribute only makes sure to align a field to a cache line. It does not prevent the linker to use the remaining of the cache line for other variables, causing potential false sharing.
Move tcp_memory_allocated into a dedicated cache line.
--- include/net/aligned_data.h | 3 +++ include/net/tcp.h | 1 - net/core/hotdata.c | 2 ++ net/ipv4/tcp.c | 2 -- net/ipv4/tcp_ipv4.c | 3 ++- net/ipv6/tcp_ipv6.c | 3 ++- net/mptcp/protocol.c | 3 ++- 7 files changed, 11 insertions(+), 6 deletions(-)
include/net/aligned_data.h +3 -0
@@ -11,6 +11,9 @@*/ struct net_aligned_data {atomic64_t net_cookie____cacheline_aligned_in_smp ; +#if defined(CONFIG_INET ) +atomic_long_t tcp_memory_allocated____cacheline_aligned_in_smp ; +#endif };extern struct net_aligned_data net_aligned_data;
include/net/tcp.h +0 -1
@@ -267,7 +267,6 @@ extern long sysctl_tcp_mem[3];#define TCP_RACK_STATIC_REO_WND 0x2/* Use static RACK reo wnd */ #define TCP_RACK_NO_DUPTHRESH 0x4/* Do not use DUPACK threshold in RACK */ -extern atomic_long_t tcp_memory_allocated;DECLARE_PER_CPU (int, tcp_memory_per_cpu_fw_alloc);extern struct percpu_counter tcp_sockets_allocated;
net/core/hotdata.c +2 -0
@@ -4,6 +4,7 @@#include <linux/list.h> #include <net/aligned_data.h> #include <net/hotdata.h> +#include <net/ip.h> #include <net/proto_memory.h> struct net_hotdata net_hotdata __cacheline_aligned = {
@@ -25,3 +26,4 @@ struct net_hotdata net_hotdata __cacheline_aligned = {EXPORT_SYMBOL (net_hotdata);struct net_aligned_data net_aligned_data; +EXPORT_IPV6_MOD (net_aligned_data);
net/ipv4/tcp.c +0 -2
@@ -302,8 +302,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);long sysctl_tcp_mem[3 ]__read_mostly ;EXPORT_IPV6_MOD (sysctl_tcp_mem); -atomic_long_t tcp_memory_allocated____cacheline_aligned_in_smp ;/* Current allocated memory. */ -EXPORT_IPV6_MOD (tcp_memory_allocated);DEFINE_PER_CPU (int, tcp_memory_per_cpu_fw_alloc);EXPORT_PER_CPU_SYMBOL_GPL (tcp_memory_per_cpu_fw_alloc);
net/ipv4/tcp_ipv4.c +2 -1
@@ -59,6 +59,7 @@#include <linux/slab.h> #include <linux/sched.h> +#include <net/aligned_data.h> #include <net/net_namespace.h> #include <net/icmp.h> #include <net/inet_hashtables.h>
@@ -3390,7 +3391,7 @@ struct proto tcp_prot = { .sockets_allocated = &tcp_sockets_allocated, .orphan_count = &tcp_orphan_count, - .memory_allocated = &tcp_memory_allocated, + .memory_allocated = &net_aligned_data.tcp_memory_allocated, .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, .memory_pressure = &tcp_memory_pressure,
net/ipv6/tcp_ipv6.c +2 -1
@@ -41,6 +41,7 @@#include <linux/random.h> #include <linux/indirect_call_wrapper.h> +#include <net/aligned_data.h> #include <net/tcp.h> #include <net/ndisc.h> #include <net/inet6_hashtables.h>
@@ -2356,7 +2357,7 @@ struct proto tcpv6_prot = { .stream_memory_free = tcp_stream_memory_free, .sockets_allocated = &tcp_sockets_allocated, - .memory_allocated = &tcp_memory_allocated, + .memory_allocated = &net_aligned_data.tcp_memory_allocated, .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, .memory_pressure = &tcp_memory_pressure,
net/mptcp/protocol.c +2 -1
@@ -11,6 +11,7 @@#include <linux/netdevice.h> #include <linux/sched/signal.h> #include <linux/atomic.h> +#include <net/aligned_data.h> #include <net/sock.h> #include <net/inet_common.h> #include <net/inet_hashtables.h>
@@ -3729,7 +3730,7 @@ static struct proto mptcp_prot = { .stream_memory_free = mptcp_stream_memory_free, .sockets_allocated = &mptcp_sockets_allocated, - .memory_allocated = &tcp_memory_allocated, + .memory_allocated = &net_aligned_data.tcp_memory_allocated, .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, .memory_pressure = &tcp_memory_pressure,
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
PATCH v2 4/4 udp: move udp_memory_allocated into net_aligned_data
In reply to Eric Dumazet____cacheline_aligned_in_smp attribute only makes sure to align a field to a cache line. It does not prevent the linker to use the remaining of the cache line for other variables, causing potential false sharing.
--- include/net/aligned_data.h | 1 + include/net/udp.h | 1 - net/ipv4/udp.c | 4 +--- net/ipv4/udp_impl.h | 1 + net/ipv4/udplite.c | 2 +- net/ipv6/udp.c | 2 +- net/ipv6/udp_impl.h | 1 + net/ipv6/udplite.c | 2 +- 8 files changed, 7 insertions(+), 7 deletions(-)
include/net/aligned_data.h +1 -0
@@ -13,6 +13,7 @@ struct net_aligned_data {atomic64_t net_cookie____cacheline_aligned_in_smp ;#if defined(CONFIG_INET )atomic_long_t tcp_memory_allocated____cacheline_aligned_in_smp ; +atomic_long_t udp_memory_allocated____cacheline_aligned_in_smp ;#endif };
include/net/udp.h +0 -1
@@ -205,7 +205,6 @@ static inline void udp_hash4_dec(struct udp_hslot *hslot2)extern struct proto udp_prot; -extern atomic_long_t udp_memory_allocated;DECLARE_PER_CPU (int, udp_memory_per_cpu_fw_alloc);/* sysctl variables for udp */
net/ipv4/udp.c +1 -3
@@ -127,8 +127,6 @@ struct udp_table udp_table __read_mostly;long sysctl_udp_mem[3 ]__read_mostly ;EXPORT_IPV6_MOD (sysctl_udp_mem); -atomic_long_t udp_memory_allocated____cacheline_aligned_in_smp ; -EXPORT_IPV6_MOD (udp_memory_allocated);DEFINE_PER_CPU (int, udp_memory_per_cpu_fw_alloc);EXPORT_PER_CPU_SYMBOL_GPL (udp_memory_per_cpu_fw_alloc);
@@ -3235,7 +3233,7 @@ struct proto udp_prot = {#ifdef CONFIG_BPF_SYSCALL .psock_update_sk_prot = udp_bpf_update_proto, #endif - .memory_allocated = &udp_memory_allocated, + .memory_allocated = &net_aligned_data.udp_memory_allocated, .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, .sysctl_mem = sysctl_udp_mem,
net/ipv4/udp_impl.h +1 -0
@@ -1,6 +1,7 @@/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _UDP4_IMPL_H#define _UDP4_IMPL_H +#include <net/aligned_data.h> #include <net/udp.h> #include <net/udplite.h> #include <net/protocol.h>
net/ipv4/udplite.c +1 -1
@@ -60,7 +60,7 @@ struct proto udplite_prot = { .rehash = udp_v4_rehash, .get_port = udp_v4_get_port, - .memory_allocated = &udp_memory_allocated, + .memory_allocated = &net_aligned_data.udp_memory_allocated, .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, .sysctl_mem = sysctl_udp_mem,
net/ipv6/udp.c +1 -1
@@ -1925,7 +1925,7 @@ struct proto udpv6_prot = { .psock_update_sk_prot = udp_bpf_update_proto, #endif - .memory_allocated = &udp_memory_allocated, + .memory_allocated = &net_aligned_data.udp_memory_allocated, .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, .sysctl_mem = sysctl_udp_mem,
net/ipv6/udp_impl.h +1 -0
@@ -1,6 +1,7 @@/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _UDP6_IMPL_H#define _UDP6_IMPL_H +#include <net/aligned_data.h> #include <net/udp.h> #include <net/udplite.h> #include <net/protocol.h>
net/ipv6/udplite.c +1 -1
@@ -59,7 +59,7 @@ struct proto udplitev6_prot = { .rehash = udp_v6_rehash, .get_port = udp_v6_get_port, - .memory_allocated = &udp_memory_allocated, + .memory_allocated = &net_aligned_data.udp_memory_allocated, .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc, .sysctl_mem = sysctl_udp_mem,
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Quoted 8 lines · On Mon, 30 Jun 2025 09:35:36 +0000 you wrote:
____cacheline_aligned_in_smpon small fields liketcp_memory_allocatedandudp_memory_allocatedis not good enough.It makes sure to put these fields at the start of a cache line, but does not prevent the linker from using the cache line for other fields, with potential performance impact.
[...]
Here is the summary with links:
- [v2,net-next,1/4] net: add struct net_aligned_data
https://git.kernel.org/netdev/net-next/c/3715b5df09b9 - [v2,net-next,2/4] net: move net_cookie into net_aligned_data https://git.kernel.org/netdev/net-next/c/998642e999d2 - [v2,net-next,3/4] tcp: move tcp_memory_allocated into net_aligned_data https://git.kernel.org/netdev/net-next/c/83081337419c - [v2,net-next,4/4] udp: move udp_memory_allocated into net_aligned_data https://git.kernel.org/netdev/net-next/c/e3d4825124bc
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html