Skip to contentSkip to conversation

net: introduce net_aligned_data

PATCH Applied Eric Dumazet 2 participants 6 messages Jun 30, 2025 – Jul 2, 2025

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

v2 cover net: introduce net_aligned_data

____cacheline_aligned_in_smp on small fields like tcp_memory_allocated and udp_memory_allocated is not good enough.

It makes sure to put these fields at the start of a cache line, but does not prevent the linker from using the cache line for other fields, with potential performance impact.

nm -v vmlinux|egrep -5 "tcp_memory_allocated|udp_memory_allocated"

...
ffffffff83e35cc0 B tcp_memory_allocated
ffffffff83e35cc8 b __key.0
ffffffff83e35cc8 b __tcp_tx_delay_enabled.1
ffffffff83e35ce0 b tcp_orphan_timer
...
ffffffff849dddc0 B udp_memory_allocated
ffffffff849dddc8 B udp_encap_needed_key
ffffffff849dddd8 B udpv6_encap_needed_key
ffffffff849dddf0 b inetsw_lock

One solution is to move these sensitive fields to a structure, so that the compiler is forced to add empty holes between each member.

nm -v vmlinux|egrep -2 "tcp_memory_allocated|udp_memory_allocated|net_aligned_data"

ffffffff885af970 b mem_id_init
ffffffff885af980 b __key.0
ffffffff885af9c0 B net_aligned_data
ffffffff885afa80 B page_pool_mem_providers
ffffffff885afa90 b __key.2

v2: added <linux/atomic.h>, report from kernel test robot <[email protected]>
v1: https://lore.kernel.org/netdev/[email protected]/

Eric Dumazet (4):
net: add struct net_aligned_data
net: move net_cookie into net_aligned_data
tcp: move tcp_memory_allocated into net_aligned_data
udp: move udp_memory_allocated into net_aligned_data

 include/net/aligned_data.h | 22 ++++++++++++++++++++++
 include/net/tcp.h          |  1 -
 include/net/udp.h          |  1 -
 net/core/hotdata.c         |  5 +++++
 net/core/net_namespace.c   |  8 ++------
 net/ipv4/tcp.c             |  2 --
 net/ipv4/tcp_ipv4.c        |  3 ++-
 net/ipv4/udp.c             |  4 +---
 net/ipv4/udp_impl.h        |  1 +
 net/ipv4/udplite.c         |  2 +-
 net/ipv6/tcp_ipv6.c        |  3 ++-
 net/ipv6/udp.c             |  2 +-
 net/ipv6/udp_impl.h        |  1 +
 net/ipv6/udplite.c         |  2 +-
 net/mptcp/protocol.c       |  3 ++-
 15 files changed, 41 insertions(+), 19 deletions(-)
 create mode 100644 include/net/aligned_data.h
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#1 lore raw

PATCH v2 1/4 net: add struct net_aligned_data

This structure will hold networking data that must consume a full cache line to avoid accidental false sharing.

---
 include/net/aligned_data.h | 16 ++++++++++++++++
 net/core/hotdata.c         |  3 +++
 2 files changed, 19 insertions(+)
 create mode 100644 include/net/aligned_data.h
include/net/aligned_data.h new file +16 -0
new file mode 100644
index 0000000000000000000000000000000000000000..cf3329d7c2272ec4424e89352626800cbc282663
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _NET_ALIGNED_DATA_H
+#define _NET_ALIGNED_DATA_H
+
+#include <linux/types.h>
+
+/* Structure holding cacheline aligned fields on SMP builds.
+ * Each field or group should have an ____cacheline_aligned_in_smp
+ * attribute to ensure no accidental false sharing can happen.
+ */
+struct net_aligned_data {
+};
+
+extern struct net_aligned_data net_aligned_data;
+
+#endif /* _NET_ALIGNED_DATA_H */
net/core/hotdata.c +3 -0
index 0bc893d5f07b03b31e08967a2238f63d218020d7..e9c03491ab001cc85fd60ad28533649b32d8a003 100644
@@ -2,6 +2,7 @@
 #include <linux/cache.h>
 #include <linux/jiffies.h>
 #include <linux/list.h>
+#include <net/aligned_data.h>
 #include <net/hotdata.h>
 #include <net/proto_memory.h>
 
@@ -22,3 +23,5 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 	.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE
 };
 EXPORT_SYMBOL(net_hotdata);
+
+struct net_aligned_data net_aligned_data;
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#2 lore raw

PATCH v2 2/4 net: move net_cookie into net_aligned_data

In reply to Eric Dumazet

Using per-cpu data for net->net_cookie generation is overkill, because even busy hosts do not create hundreds of netns per second.

Make sure to put net_cookie in a private cache line to avoid potential false sharing.

---
 include/net/aligned_data.h | 2 ++
 net/core/net_namespace.c   | 8 ++------
 2 files changed, 4 insertions(+), 6 deletions(-)
include/net/aligned_data.h +2 -0
index cf3329d7c2272ec4424e89352626800cbc282663..5c7badf71f043a2bbc871b0063c9a2d2a4ffbfcb 100644
@@ -2,6 +2,7 @@
 #ifndef _NET_ALIGNED_DATA_H
 #define _NET_ALIGNED_DATA_H
 
+#include <linux/atomic.h>
 #include <linux/types.h>
 
 /* Structure holding cacheline aligned fields on SMP builds.
@@ -9,6 +10,7 @@
  * attribute to ensure no accidental false sharing can happen.
  */
 struct net_aligned_data {
+	atomic64_t	net_cookie ____cacheline_aligned_in_smp;
 };
 
 extern struct net_aligned_data net_aligned_data;
net/core/net_namespace.c +2 -6
index d0f607507ee8d0b6d31f11a49421b5f0a985bd3b..e68d208b200dd4be76fc08af73054b3d1dea834c 100644
@@ -19,9 +19,9 @@
 #include <linux/net_namespace.h>
 #include <linux/sched/task.h>
 #include <linux/uidgid.h>
-#include <linux/cookie.h>
 #include <linux/proc_fs.h>
 
+#include <net/aligned_data.h>
 #include <net/sock.h>
 #include <net/netlink.h>
 #include <net/net_namespace.h>
@@ -64,8 +64,6 @@ DECLARE_RWSEM(pernet_ops_rwsem);
 
 static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;
 
-DEFINE_COOKIE(net_cookie);
-
 static struct net_generic *net_alloc_generic(void)
 {
 	unsigned int gen_ptrs = READ_ONCE(max_gen_ptrs);
@@ -434,9 +432,7 @@ static __net_init int setup_net(struct net *net)
 	LIST_HEAD(net_exit_list);
 	int error = 0;
 
-	preempt_disable();
-	net->net_cookie = gen_cookie_next(&net_cookie);
-	preempt_enable();
+	net->net_cookie = atomic64_inc_return(&net_aligned_data.net_cookie);
 
 	list_for_each_entry(ops, &pernet_list, list) {
 		error = ops_init(ops, net);
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#3 lore raw

PATCH v2 3/4 tcp: move tcp_memory_allocated into net_aligned_data

In reply to Eric Dumazet

____cacheline_aligned_in_smp attribute only makes sure to align a field to a cache line. It does not prevent the linker to use the remaining of the cache line for other variables, causing potential false sharing.

Move tcp_memory_allocated into a dedicated cache line.

---
 include/net/aligned_data.h | 3 +++
 include/net/tcp.h          | 1 -
 net/core/hotdata.c         | 2 ++
 net/ipv4/tcp.c             | 2 --
 net/ipv4/tcp_ipv4.c        | 3 ++-
 net/ipv6/tcp_ipv6.c        | 3 ++-
 net/mptcp/protocol.c       | 3 ++-
 7 files changed, 11 insertions(+), 6 deletions(-)
include/net/aligned_data.h +3 -0
index 5c7badf71f043a2bbc871b0063c9a2d2a4ffbfcb..bedb4f86b0fea1ce13a10895182681c37ef7e904 100644
@@ -11,6 +11,9 @@
  */
 struct net_aligned_data {
 	atomic64_t	net_cookie ____cacheline_aligned_in_smp;
+#if defined(CONFIG_INET)
+	atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;
+#endif
 };
 
 extern struct net_aligned_data net_aligned_data;
include/net/tcp.h +0 -1
index 761c4a0ad386f95f73d72dc013a0952187342b51..bc08de49805cf6fc2ffbec96e42bf12378fd10cf 100644
@@ -267,7 +267,6 @@ extern long sysctl_tcp_mem[3];
 #define TCP_RACK_STATIC_REO_WND  0x2 /* Use static RACK reo wnd */
 #define TCP_RACK_NO_DUPTHRESH    0x4 /* Do not use DUPACK threshold in RACK */
 
-extern atomic_long_t tcp_memory_allocated;
 DECLARE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
 
 extern struct percpu_counter tcp_sockets_allocated;
net/core/hotdata.c +2 -0
index e9c03491ab001cc85fd60ad28533649b32d8a003..95d0a4df10069e4529fb9e5b58e8391574085cf1 100644
@@ -4,6 +4,7 @@
 #include <linux/list.h>
 #include <net/aligned_data.h>
 #include <net/hotdata.h>
+#include <net/ip.h>
 #include <net/proto_memory.h>
 
 struct net_hotdata net_hotdata __cacheline_aligned = {
@@ -25,3 +26,4 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 EXPORT_SYMBOL(net_hotdata);
 
 struct net_aligned_data net_aligned_data;
+EXPORT_IPV6_MOD(net_aligned_data);
net/ipv4/tcp.c +0 -2
index 8a3c99246d2ed32ba45849b56830bf128e265763..925b2c572ca23b3f2eba48a38820f6553a2724f4 100644
@@ -302,8 +302,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
 long sysctl_tcp_mem[3] __read_mostly;
 EXPORT_IPV6_MOD(sysctl_tcp_mem);
 
-atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;	/* Current allocated memory. */
-EXPORT_IPV6_MOD(tcp_memory_allocated);
 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
 
net/ipv4/tcp_ipv4.c +2 -1
index 56223338bc0f070179efb2ce9996fa7146782adc..b406fd012b2e6a4f5b0bca9d26298bdf26b7989a 100644
@@ -59,6 +59,7 @@
 #include <linux/slab.h>
 #include <linux/sched.h>
 
+#include <net/aligned_data.h>
 #include <net/net_namespace.h>
 #include <net/icmp.h>
 #include <net/inet_hashtables.h>
@@ -3390,7 +3391,7 @@ struct proto tcp_prot = {
 	.sockets_allocated	= &tcp_sockets_allocated,
 	.orphan_count		= &tcp_orphan_count,
 
-	.memory_allocated	= &tcp_memory_allocated,
+	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
 	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
 
 	.memory_pressure	= &tcp_memory_pressure,
net/ipv6/tcp_ipv6.c +2 -1
index 9fb614e17bde99e5806cd56fdbc4d0b0b74a3f57..ed0b891885d848b25667edbaf4e925b3ece46033 100644
@@ -41,6 +41,7 @@
 #include <linux/random.h>
 #include <linux/indirect_call_wrapper.h>
 
+#include <net/aligned_data.h>
 #include <net/tcp.h>
 #include <net/ndisc.h>
 #include <net/inet6_hashtables.h>
@@ -2356,7 +2357,7 @@ struct proto tcpv6_prot = {
 	.stream_memory_free	= tcp_stream_memory_free,
 	.sockets_allocated	= &tcp_sockets_allocated,
 
-	.memory_allocated	= &tcp_memory_allocated,
+	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
 	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
 
 	.memory_pressure	= &tcp_memory_pressure,
net/mptcp/protocol.c +2 -1
index e7972e633236e0451f0321ff4b0a8d1b37282d5f..5f904fc5ac4c63e8b6c7c9aa79f17e8dcdf1a007 100644
@@ -11,6 +11,7 @@
 #include <linux/netdevice.h>
 #include <linux/sched/signal.h>
 #include <linux/atomic.h>
+#include <net/aligned_data.h>
 #include <net/sock.h>
 #include <net/inet_common.h>
 #include <net/inet_hashtables.h>
@@ -3729,7 +3730,7 @@ static struct proto mptcp_prot = {
 	.stream_memory_free	= mptcp_stream_memory_free,
 	.sockets_allocated	= &mptcp_sockets_allocated,
 
-	.memory_allocated	= &tcp_memory_allocated,
+	.memory_allocated	= &net_aligned_data.tcp_memory_allocated,
 	.per_cpu_fw_alloc	= &tcp_memory_per_cpu_fw_alloc,
 
 	.memory_pressure	= &tcp_memory_pressure,
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#4 lore raw

PATCH v2 4/4 udp: move udp_memory_allocated into net_aligned_data

In reply to Eric Dumazet

____cacheline_aligned_in_smp attribute only makes sure to align a field to a cache line. It does not prevent the linker to use the remaining of the cache line for other variables, causing potential false sharing.

---
 include/net/aligned_data.h | 1 +
 include/net/udp.h          | 1 -
 net/ipv4/udp.c             | 4 +---
 net/ipv4/udp_impl.h        | 1 +
 net/ipv4/udplite.c         | 2 +-
 net/ipv6/udp.c             | 2 +-
 net/ipv6/udp_impl.h        | 1 +
 net/ipv6/udplite.c         | 2 +-
 8 files changed, 7 insertions(+), 7 deletions(-)
include/net/aligned_data.h +1 -0
index bedb4f86b0fea1ce13a10895182681c37ef7e904..e1a1c8aedc79753f63eb7c81757639ea20829f0d 100644
@@ -13,6 +13,7 @@ struct net_aligned_data {
 	atomic64_t	net_cookie ____cacheline_aligned_in_smp;
 #if defined(CONFIG_INET)
 	atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp;
+	atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp;
 #endif
 };
 
include/net/udp.h +0 -1
index a772510b2aa58a922dc4dd5bd7cb7f7f0c1ce0e3..f8ae2c4ade141d27e4af3376f5a177e46d5b39ea 100644
@@ -205,7 +205,6 @@ static inline void udp_hash4_dec(struct udp_hslot *hslot2)
 
 extern struct proto udp_prot;
 
-extern atomic_long_t udp_memory_allocated;
 DECLARE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
 
 /* sysctl variables for udp */
net/ipv4/udp.c +1 -3
index 19573ee64a0f18cf55df34ace1956e9c3e20172c..49f43c54cfb0e3ac85c1a6202f3d6a2f1ca6d0ba 100644
@@ -127,8 +127,6 @@ struct udp_table udp_table __read_mostly;
 long sysctl_udp_mem[3] __read_mostly;
 EXPORT_IPV6_MOD(sysctl_udp_mem);
 
-atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp;
-EXPORT_IPV6_MOD(udp_memory_allocated);
 DEFINE_PER_CPU(int, udp_memory_per_cpu_fw_alloc);
 EXPORT_PER_CPU_SYMBOL_GPL(udp_memory_per_cpu_fw_alloc);
 
@@ -3235,7 +3233,7 @@ struct proto udp_prot = {
 #ifdef CONFIG_BPF_SYSCALL
 	.psock_update_sk_prot	= udp_bpf_update_proto,
 #endif
-	.memory_allocated	= &udp_memory_allocated,
+	.memory_allocated	= &net_aligned_data.udp_memory_allocated,
 	.per_cpu_fw_alloc	= &udp_memory_per_cpu_fw_alloc,
 
 	.sysctl_mem		= sysctl_udp_mem,
net/ipv4/udp_impl.h +1 -0
index e1ff3a37599614b18a0b621534019a7bd71ea901..c7142213fc2112c9e423caa5e6d84bafeaca9936 100644
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _UDP4_IMPL_H
 #define _UDP4_IMPL_H
+#include <net/aligned_data.h>
 #include <net/udp.h>
 #include <net/udplite.h>
 #include <net/protocol.h>
net/ipv4/udplite.c +1 -1
index af37af3ab727bffeebe5c41d84cb7c130f49c50d..d3e621a11a1aa4cafb62eb53ddc0ed1ca517a7fe 100644
@@ -60,7 +60,7 @@ struct proto 	udplite_prot = {
 	.rehash		   = udp_v4_rehash,
 	.get_port	   = udp_v4_get_port,
 
-	.memory_allocated  = &udp_memory_allocated,
+	.memory_allocated  = &net_aligned_data.udp_memory_allocated,
 	.per_cpu_fw_alloc  = &udp_memory_per_cpu_fw_alloc,
 
 	.sysctl_mem	   = sysctl_udp_mem,
net/ipv6/udp.c +1 -1
index ebb95d8bc6819f72842fd1567e73fcef4f1e0ed0..6bbdadbd5fecccfb7de99f05c6fb179393e162f2 100644
@@ -1925,7 +1925,7 @@ struct proto udpv6_prot = {
 	.psock_update_sk_prot	= udp_bpf_update_proto,
 #endif
 
-	.memory_allocated	= &udp_memory_allocated,
+	.memory_allocated	= &net_aligned_data.udp_memory_allocated,
 	.per_cpu_fw_alloc	= &udp_memory_per_cpu_fw_alloc,
 
 	.sysctl_mem		= sysctl_udp_mem,
net/ipv6/udp_impl.h +1 -0
index 0590f566379d7d07dfdd1b0ae808b9d8964eb5aa..8a406be25a3a6dee687a6a02ea0b6a28428abb86 100644
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #ifndef _UDP6_IMPL_H
 #define _UDP6_IMPL_H
+#include <net/aligned_data.h>
 #include <net/udp.h>
 #include <net/udplite.h>
 #include <net/protocol.h>
net/ipv6/udplite.c +1 -1
index a60bec9b14f14a5b2d271f9965b5fca3d2a440c8..2cec542437f74eba9540fc464ee70d8b0bc0be79 100644
@@ -59,7 +59,7 @@ struct proto udplitev6_prot = {
 	.rehash		   = udp_v6_rehash,
 	.get_port	   = udp_v6_get_port,
 
-	.memory_allocated  = &udp_memory_allocated,
+	.memory_allocated  = &net_aligned_data.udp_memory_allocated,
 	.per_cpu_fw_alloc  = &udp_memory_per_cpu_fw_alloc,
 
 	.sysctl_mem	   = sysctl_udp_mem,
Signature · 1 line
2.50.0.727.gbf7dc18ff4-goog
#5 lore raw
In reply to Eric Dumazet
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:

Hello:

This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:

Quoted 8 lines · On Mon, 30 Jun 2025 09:35:36 +0000 you wrote:

____cacheline_aligned_in_smp on small fields like tcp_memory_allocated and udp_memory_allocated is not good enough.

It makes sure to put these fields at the start of a cache line, but does not prevent the linker from using the cache line for other fields, with potential performance impact.

[...]

Here is the summary with links:
- [v2,net-next,1/4] net: add struct net_aligned_data

    https://git.kernel.org/netdev/net-next/c/3715b5df09b9
  - [v2,net-next,2/4] net: move net_cookie into net_aligned_data
    https://git.kernel.org/netdev/net-next/c/998642e999d2
  - [v2,net-next,3/4] tcp: move tcp_memory_allocated into net_aligned_data
    https://git.kernel.org/netdev/net-next/c/83081337419c
  - [v2,net-next,4/4] udp: move udp_memory_allocated into net_aligned_data
    https://git.kernel.org/netdev/net-next/c/e3d4825124bc

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html