net: three additions to net_hotdata
Subsystems (6)
cover net: three additions to net_hotdata
This series moves three fast path sysctls to net_hotdata.
To avoid <net/hotdata.h> inclusion from <net/sock.h>, create <net/proto_memory.h> to hold proto memory definitions.
Eric Dumazet (5):
net: move sysctl_max_skb_frags to net_hotdata
net: move sysctl_skb_defer_max to net_hotdata
tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
net: add <net/proto_memory.h>
net: move sysctl_mem_pcpu_rsv to net_hotdata
include/linux/skbuff.h | 2 - include/net/hotdata.h | 3 ++ include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++ include/net/sock.h | 78 ----------------------------------- include/net/tcp.h | 10 +---- net/core/dev.c | 1 - net/core/dev.h | 1 - net/core/hotdata.c | 7 +++- net/core/skbuff.c | 7 +--- net/core/sock.c | 2 +- net/core/sysctl_net_core.c | 7 ++-- net/ipv4/proc.c | 1 + net/ipv4/tcp.c | 14 ++++++- net/ipv4/tcp_input.c | 1 + net/ipv4/tcp_output.c | 1 + net/mptcp/protocol.c | 3 +- net/sctp/sm_statefuns.c | 1 + 17 files changed, 117 insertions(+), 105 deletions(-) create mode 100644 include/net/proto_memory.h
Signature · 1 line
2.44.0.769.g3c40516874-goog
PATCH 1/5 net: move sysctl_max_skb_frags to net_hotdata
sysctl_max_skb_frags is used in TCP and MPTCP fast paths, move it to net_hodata for better cache locality.
--- include/linux/skbuff.h | 2 -- include/net/hotdata.h | 1 + net/core/hotdata.c | 1 + net/core/skbuff.c | 5 +---- net/core/sysctl_net_core.c | 2 +- net/ipv4/tcp.c | 3 ++- net/mptcp/protocol.c | 3 ++- 7 files changed, 8 insertions(+), 9 deletions(-)
include/linux/skbuff.h +0 -2
@@ -353,8 +353,6 @@ struct sk_buff;#define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS -extern int sysctl_max_skb_frags; -/* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to * segment using its current segmentation instead. */
include/net/hotdata.h +1 -0
@@ -38,6 +38,7 @@ struct net_hotdata {int max_backlog;int dev_tx_weight;int dev_rx_weight; +int sysctl_max_skb_frags; };#define inet_ehash_secret net_hotdata.tcp_protocol.secret
net/core/hotdata.c +1 -0
@@ -18,5 +18,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .max_backlog = 1000, .dev_tx_weight = 64, .dev_rx_weight = 64, + .sysctl_max_skb_frags = MAX_SKB_FRAGS, };EXPORT_SYMBOL (net_hotdata);
net/core/skbuff.c +1 -4
@@ -109,9 +109,6 @@ static struct kmem_cache *skbuff_ext_cache __ro_after_init;#define SKB_SMALL_HEAD_HEADROOM \ SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE) -int sysctl_max_skb_frags__read_mostly =MAX_SKB_FRAGS ; -EXPORT_SYMBOL (sysctl_max_skb_frags); -/* kcm_write_msgs() relies on casting paged frags to bio_vec to use * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the * netmem is a page.
@@ -7040,7 +7037,7 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,ssize_t skb_splice_from_iter (struct sk_buff *skb,struct iov_iter *iter,ssize_t maxsize,gfp_t gfp) { -size_t frag_limit =READ_ONCE (sysctl_max_skb_frags); +size_t frag_limit =READ_ONCE (net_hotdata.sysctl_max_skb_frags);struct page *pages[8 ], **ppages = pages;ssize_t spliced =0 , ret =0 ;unsigned int i;
net/core/sysctl_net_core.c +1 -1
@@ -595,7 +595,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "max_skb_frags", - .data = &sysctl_max_skb_frags, + .data = &net_hotdata.sysctl_max_skb_frags, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax,
net/ipv4/tcp.c +2 -1
@@ -280,6 +280,7 @@#include <linux/uaccess.h> #include <asm/ioctls.h> #include <net/busy_poll.h> +#include <net/hotdata.h> #include <net/rps.h> /* Track pending CMSGs. */
@@ -1188,7 +1189,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)if (!skb_can_coalesce (skb, i, pfrag->page, pfrag->offset)) { -if (i >=READ_ONCE (sysctl_max_skb_frags)) { +if (i >=READ_ONCE (net_hotdata.sysctl_max_skb_frags)) {tcp_mark_push (tp, skb); goto new_segment; }
net/mptcp/protocol.c +2 -1
@@ -20,6 +20,7 @@#include <net/transp_v6.h> #endif #include <net/mptcp.h> +#include <net/hotdata.h> #include <net/xfrm.h> #include <asm/ioctls.h> #include "protocol.h"
@@ -1272,7 +1273,7 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, i =skb_shinfo (skb)->nr_frags; can_coalesce =skb_can_coalesce (skb, i, dfrag->page, offset); -if (!can_coalesce && i >=READ_ONCE (sysctl_max_skb_frags)) { +if (!can_coalesce && i >=READ_ONCE (net_hotdata.sysctl_max_skb_frags)) {tcp_mark_push (tcp_sk (ssk), skb); goto alloc_skb; }
Signature · 1 line
2.44.0.769.g3c40516874-goog
PATCH 2/5 net: move sysctl_skb_defer_max to net_hotdata
In reply to Eric Dumazetsysctl_skb_defer_max is used in TCP fast path,
move it to net_hodata.
--- include/net/hotdata.h | 1 + net/core/dev.c | 1 - net/core/dev.h | 1 - net/core/hotdata.c | 1 + net/core/skbuff.c | 2 +- net/core/sysctl_net_core.c | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-)
include/net/hotdata.h +1 -0
@@ -39,6 +39,7 @@ struct net_hotdata {int dev_tx_weight;int dev_rx_weight;int sysctl_max_skb_frags; +int sysctl_skb_defer_max; };#define inet_ehash_secret net_hotdata.tcp_protocol.secret
net/core/dev.c +0 -1
@@ -4450,7 +4450,6 @@ EXPORT_SYMBOL(__dev_direct_xmit);*************************************************************************/ static DEFINE_PER_CPU(struct task_struct *, backlog_napi); -unsigned int sysctl_skb_defer_max__read_mostly =64 ;int weight_p__read_mostly =64 ;/* old backlog weight */ int dev_weight_rx_bias__read_mostly =1 ;/* bias for backlog weight */ int dev_weight_tx_bias__read_mostly =1 ;/* bias for output_queue quota */
net/core/dev.h +0 -1
@@ -36,7 +36,6 @@ int dev_addr_init(struct net_device *dev);void dev_addr_check (struct net_device *dev);/* sysctls not referred to from outside net/core/ */ -extern unsigned int sysctl_skb_defer_max;extern int netdev_unregister_timeout_secs;extern int weight_p;extern int dev_weight_rx_bias;
net/core/hotdata.c +1 -0
@@ -19,5 +19,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .dev_tx_weight = 64, .dev_rx_weight = 64, .sysctl_max_skb_frags = MAX_SKB_FRAGS, + .sysctl_skb_defer_max = 64, };EXPORT_SYMBOL (net_hotdata);
net/core/skbuff.c +1 -1
@@ -6985,7 +6985,7 @@ nodefer: kfree_skb_napi_cache(skb);DEBUG_NET_WARN_ON_ONCE (skb->destructor); sd = &per_cpu (softnet_data, cpu); - defer_max =READ_ONCE (sysctl_skb_defer_max); + defer_max =READ_ONCE (net_hotdata.sysctl_skb_defer_max);if (READ_ONCE (sd->defer_count) >= defer_max) goto nodefer;
net/core/sysctl_net_core.c +1 -1
@@ -654,7 +654,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "skb_defer_max", - .data = &sysctl_skb_defer_max, + .data = &net_hotdata.sysctl_skb_defer_max, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_dointvec_minmax,
Signature · 1 line
2.44.0.769.g3c40516874-goog
PATCH 3/5 tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
In reply to Eric Dumazettcp_out_of_memory() has a single caller: tcp_check_oom().
Following patch will also make sk_memory_allocated() not anymore visible from <net/sock.h> and <net/tcp.h>
Add const qualifier to sock argument of tcp_out_of_memory() and tcp_check_oom().
--- include/net/tcp.h | 10 +--------- net/ipv4/tcp.c | 10 +++++++++- 2 files changed, 10 insertions(+), 10 deletions(-)
include/net/tcp.h +1 -9
@@ -296,14 +296,6 @@ static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)return seq3 - seq2 >= seq1 - seq2; } -static inline bool tcp_out_of_memory (struct sock *sk) -{ -if (sk->sk_wmem_queued >SOCK_MIN_SNDBUF && -sk_memory_allocated (sk) >sk_prot_mem_limits (sk,2 )) -return true; -return false; -} -static inline void tcp_wmem_free_skb (struct sock *sk,struct sk_buff *skb) {sk_wmem_queued_add (sk, -skb->truesize);
@@ -316,7 +308,7 @@ static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)void sk_forced_mem_schedule (struct sock *sk,int size); -bool tcp_check_oom (struct sock *sk,int shift); +bool tcp_check_oom (const struct sock *sk,int shift);extern struct proto tcp_prot;
net/ipv4/tcp.c +9 -1
@@ -2752,7 +2752,15 @@ static bool tcp_too_many_orphans(int shift)READ_ONCE (sysctl_tcp_max_orphans); } -bool tcp_check_oom (struct sock *sk,int shift) +static bool tcp_out_of_memory (const struct sock *sk) +{ +if (sk->sk_wmem_queued >SOCK_MIN_SNDBUF && +sk_memory_allocated (sk) >sk_prot_mem_limits (sk,2 )) +return true; +return false; +} + +bool tcp_check_oom (const struct sock *sk,int shift) {bool too_many_orphans, out_of_socket_memory;
Signature · 1 line
2.44.0.769.g3c40516874-goog
PATCH 4/5 net: add <net/proto_memory.h>
In reply to Eric DumazetMove some proto memory definitions out of <net/sock.h>
Very few files need them, and following patch will include <net/hotdata.h> from <net/proto_memory.h>
--- include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++ include/net/sock.h | 78 ----------------------------------- net/core/sock.c | 1 + net/core/sysctl_net_core.c | 1 + net/ipv4/proc.c | 1 + net/ipv4/tcp.c | 1 + net/ipv4/tcp_input.c | 1 + net/ipv4/tcp_output.c | 1 + net/sctp/sm_statefuns.c | 1 + 9 files changed, 90 insertions(+), 78 deletions(-) create mode 100644 include/net/proto_memory.h
include/net/proto_memory.h new file +83 -0
@@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _PROTO_MEMORY_H +#define _PROTO_MEMORY_H + +#include <net/sock.h> + +/* 1 MB per cpu, in page units */ +#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) +extern int sysctl_mem_pcpu_rsv; + +static inline bool sk_has_memory_pressure (const struct sock *sk) +{ +return sk->sk_prot->memory_pressure !=NULL ; +} + +static inline bool +proto_memory_pressure (const struct proto *prot) +{ +if (!prot->memory_pressure) +return false; +return !!READ_ONCE (*prot->memory_pressure); +} + +static inline bool sk_under_global_memory_pressure (const struct sock *sk) +{ +return proto_memory_pressure (sk->sk_prot); +} + +static inline bool sk_under_memory_pressure (const struct sock *sk) +{ +if (!sk->sk_prot->memory_pressure) +return false; + +if (mem_cgroup_sockets_enabled && sk->sk_memcg && +mem_cgroup_under_socket_pressure (sk->sk_memcg)) +return true; + +return !!READ_ONCE (*sk->sk_prot->memory_pressure); +} + +static inline long +proto_memory_allocated (const struct proto *prot) +{ +return max (0L ,atomic_long_read (prot->memory_allocated)); +} + +static inline long +sk_memory_allocated (const struct sock *sk) +{ +return proto_memory_allocated (sk->sk_prot); +} + +static inline void proto_memory_pcpu_drain (struct proto *proto) +{ +int val =this_cpu_xchg (*proto->per_cpu_fw_alloc,0 ); + +if (val) +atomic_long_add (val, proto->memory_allocated); +} + +static inline void +sk_memory_allocated_add (const struct sock *sk,int val) +{ +struct proto *proto = sk->sk_prot; + + val =this_cpu_add_return (*proto->per_cpu_fw_alloc, val); + +if (unlikely (val >=READ_ONCE (sysctl_mem_pcpu_rsv))) +proto_memory_pcpu_drain (proto); +} + +static inline void +sk_memory_allocated_sub (const struct sock *sk,int val) +{ +struct proto *proto = sk->sk_prot; + + val =this_cpu_sub_return (*proto->per_cpu_fw_alloc, val); + +if (unlikely (val <= -READ_ONCE (sysctl_mem_pcpu_rsv))) +proto_memory_pcpu_drain (proto); +} + +#endif /* _PROTO_MEMORY_H */
include/net/sock.h +0 -78
90 lines not on this page · raw message
net/core/sock.c +1 -0
@@ -127,6 +127,7 @@#include <net/net_namespace.h> #include <net/request_sock.h> #include <net/sock.h> +#include <net/proto_memory.h> #include <linux/net_tstamp.h> #include <net/xfrm.h> #include <linux/ipsec.h>
net/core/sysctl_net_core.c +1 -0
@@ -24,6 +24,7 @@#include <net/busy_poll.h> #include <net/pkt_sched.h> #include <net/hotdata.h> +#include <net/proto_memory.h> #include <net/rps.h> #include "dev.h"
net/ipv4/proc.c +1 -0
@@ -33,6 +33,7 @@#include <net/protocol.h> #include <net/tcp.h> #include <net/mptcp.h> +#include <net/proto_memory.h> #include <net/udp.h> #include <net/udplite.h> #include <linux/bottom_half.h>
net/ipv4/tcp.c +1 -0
@@ -272,6 +272,7 @@#include <net/inet_common.h> #include <net/tcp.h> #include <net/mptcp.h> +#include <net/proto_memory.h> #include <net/xfrm.h> #include <net/ip.h> #include <net/sock.h>
net/ipv4/tcp_input.c +1 -0
@@ -72,6 +72,7 @@#include <linux/prefetch.h> #include <net/dst.h> #include <net/tcp.h> +#include <net/proto_memory.h> #include <net/inet_common.h> #include <linux/ipsec.h> #include <asm/unaligned.h>
net/ipv4/tcp_output.c +1 -0
@@ -39,6 +39,7 @@#include <net/tcp.h> #include <net/mptcp.h> +#include <net/proto_memory.h> #include <linux/compiler.h> #include <linux/gfp.h>
net/sctp/sm_statefuns.c +1 -0
@@ -38,6 +38,7 @@#include <linux/inet.h> #include <linux/slab.h> #include <net/sock.h> +#include <net/proto_memory.h> #include <net/inet_ecn.h> #include <linux/skbuff.h> #include <net/sctp/sctp.h>
Signature · 1 line
2.44.0.769.g3c40516874-goog
PATCH 5/5 net: move sysctl_mem_pcpu_rsv to net_hotdata
In reply to Eric Dumazetsysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.
--- include/net/hotdata.h | 1 + include/net/proto_memory.h | 6 +++--- net/core/hotdata.c | 5 +++-- net/core/sock.c | 1 - net/core/sysctl_net_core.c | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-)
include/net/hotdata.h +1 -0
@@ -40,6 +40,7 @@ struct net_hotdata {int dev_rx_weight;int sysctl_max_skb_frags;int sysctl_skb_defer_max; +int sysctl_mem_pcpu_rsv; };#define inet_ehash_secret net_hotdata.tcp_protocol.secret
include/net/proto_memory.h +3 -3
@@ -3,10 +3,10 @@#define _PROTO_MEMORY_H#include <net/sock.h> +#include <net/hotdata.h> /* 1 MB per cpu, in page units */ #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) -extern int sysctl_mem_pcpu_rsv;static inline bool sk_has_memory_pressure (const struct sock *sk) {
@@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val) val =this_cpu_add_return (*proto->per_cpu_fw_alloc, val); -if (unlikely (val >=READ_ONCE (sysctl_mem_pcpu_rsv))) +if (unlikely (val >=READ_ONCE (net_hotdata.sysctl_mem_pcpu_rsv)))proto_memory_pcpu_drain (proto); }
@@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val) val =this_cpu_sub_return (*proto->per_cpu_fw_alloc, val); -if (unlikely (val <= -READ_ONCE (sysctl_mem_pcpu_rsv))) +if (unlikely (val <= -READ_ONCE (net_hotdata.sysctl_mem_pcpu_rsv)))proto_memory_pcpu_drain (proto); }
net/core/hotdata.c +3 -2
@@ -1,9 +1,9 @@// SPDX-License-Identifier: GPL-2.0-or-later -#include <net/hotdata.h> #include <linux/cache.h> #include <linux/jiffies.h> #include <linux/list.h> - +#include <net/hotdata.h> +#include <net/proto_memory.h> struct net_hotdata net_hotdata __cacheline_aligned = { .offload_base =LIST_HEAD_INIT (net_hotdata.offload_base),
@@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .dev_rx_weight = 64, .sysctl_max_skb_frags = MAX_SKB_FRAGS, .sysctl_skb_defer_max = 64, + .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE };EXPORT_SYMBOL (net_hotdata);
net/core/sock.c +0 -1
@@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;EXPORT_SYMBOL (sysctl_rmem_max);__u32 sysctl_wmem_default__read_mostly =SK_WMEM_MAX ;__u32 sysctl_rmem_default__read_mostly =SK_RMEM_MAX ; -int sysctl_mem_pcpu_rsv__read_mostly =SK_MEMORY_PCPU_RESERVE ;int sysctl_tstamp_allow_data__read_mostly =1 ;
net/core/sysctl_net_core.c +1 -1
@@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "mem_pcpu_rsv", - .data = &sysctl_mem_pcpu_rsv, + .data = &net_hotdata.sysctl_mem_pcpu_rsv, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax,
Signature · 1 line
2.44.0.769.g3c40516874-goog
Quoted 23 lines · Eric Dumazet
This series moves three fast path sysctls to
net_hotdata.To avoid <
net/hotdata.h> inclusion from <net/sock.h>, create <net/proto_memory.h> to hold proto memory definitions.Eric Dumazet (5):
net: movesysctl_max_skb_fragstonet_hotdata
net: movesysctl_skb_defer_maxtonet_hotdata
tcp: movetcp_out_of_memory()tonet/ipv4/tcp.c
net: add <net/proto_memory.h>
net: movesysctl_mem_pcpu_rsvtonet_hotdatainclude/linux/skbuff.h | 2 - include/net/hotdata.h | 3 ++ include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++ include/net/sock.h | 78 ----------------------------------- include/net/tcp.h | 10 +---- net/core/dev.c | 1 - net/core/dev.h | 1 - net/core/hotdata.c | 7 +++- net/core/skbuff.c | 7 +--- net/core/sock.c | 2 +- net/core/sysctl_net_core.c | 7 ++--
Quoted 9 lines
net/ipv4/proc.c | 1 + net/ipv4/tcp.c | 14 ++++++- net/ipv4/tcp_input.c | 1 + net/ipv4/tcp_output.c | 1 + net/mptcp/protocol.c | 3 +- net/sctp/sm_statefuns.c | 1 + 17 files changed, 117 insertions(+), 105 deletions(-) create mode 100644 include/net/proto_memory.h
For the set:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Hello:
This series was applied to netdev/net-next.git (main) by Jakub Kicinski <[email protected]>:
Quoted 5 lines · On Mon, 29 Apr 2024 13:40:20 +0000 you wrote:
This series moves three fast path sysctls to
net_hotdata.To avoid <
net/hotdata.h> inclusion from <net/sock.h>, create <net/proto_memory.h> to hold proto memory definitions.
Quoted 8 lines
Eric Dumazet (5):
net: movesysctl_max_skb_fragstonet_hotdata
net: movesysctl_skb_defer_maxtonet_hotdata
tcp: movetcp_out_of_memory()tonet/ipv4/tcp.c
net: add <net/proto_memory.h>
net: movesysctl_mem_pcpu_rsvtonet_hotdata[...]
Here is the summary with links:
- [net-next,1/5] net: move sysctl_max_skb_frags to net_hotdata
https://git.kernel.org/netdev/net-next/c/a86a0661b86f - [net-next,2/5] net: move sysctl_skb_defer_max to net_hotdata https://git.kernel.org/netdev/net-next/c/d480dc76d9f8 - [net-next,3/5] tcp: move tcp_out_of_memory() to net/ipv4/tcp.c https://git.kernel.org/netdev/net-next/c/dda4d96acb20 - [net-next,4/5] net: add <net/proto_memory.h> https://git.kernel.org/netdev/net-next/c/f3d93817fba3 - [net-next,5/5] net: move sysctl_mem_pcpu_rsv to net_hotdata https://git.kernel.org/netdev/net-next/c/c204fef97ee6
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html