NET : defines an infrastructure to keep 'inuse' changes in an efficent SMP/NUMA way.
Subsystems (3)
PATCH 1/4 NET : defines an infrastructure to keep 'inuse' changes in an efficent SMP/NUMA way.
"struct proto" currently uses an array stats[NR_CPUS] to track change on 'inuse' sockets per protocol.
If NR_CPUS is big, this means we use a big memory area for this.
Moreover, all this memory area is located on a single node on NUMA machines, increasing memory pressure on the boot node.
In this patch, I tried to :
- Keep a fast !CONFIG_SMP implementation
- Keep a fast CONFIG_SMP implementation for often used protocols (tcp,udp,raw,...)
- Introduce a NUMA efficient implementation
Some helper macros are defined in include/net/sock.h These macros take into account CONFIG_SMP
If a "struct proto" is declared without using DEFINE_PROTO_INUSE / REF_PROTO_INUSE macros, it will automatically use a default implementation, using a dynamically allocated percpu zone. This default implementation will be NUMA efficient, but might use 32/64 bytes per possible cpu because of current alloc_percpu() implementation.
However it still should be better than previous implementation based on stats[NR_CPUS] field.
When a "struct proto" is changed to use the new macros, we use a single static "int" percpu variable, lowering the memory and cpu costs, still preserving NUMA efficiency.
include/net/sock.h | 63 ++++++++++++++++++++++++++++++++++++++----- net/core/sock.c | 48 ++++++++++++++++++++++++++++++++ net/ipv4/proc.c | 19 ++---------- net/ipv6/proc.c | 19 ++---------- 4 files changed, 112 insertions(+), 37 deletions(-)
include/net/sock.h +57 -6
@@ -560,6 +560,14 @@ struct proto {void (*unhash)(struct sock *sk);int (*get_port)(struct sock *sk,unsigned short snum); +#ifdef CONFIG_SMP +/* Keeping track of sockets in use */ +void (*inuse_add)(struct proto *prot,int inc); +int (*inuse_getval)(const struct proto *prot); +int *inuse_ptr; +#else +int inuse; +#endif /* Memory pressure */ void (*enter_memory_pressure)(void );atomic_t *memory_allocated;/* Current allocated memory. */
@@ -592,12 +600,38 @@ struct proto {#ifdef SOCK_REFCNT_DEBUG atomic_t socks;#endif -struct { -int inuse; -u8 __pad[SMP_CACHE_BYTES -sizeof (int )]; - } stats[NR_CPUS ]; }; +/* +* Special macros to let protos use a fast version of inuse{get|add} +* using a static percpu variable per proto instead of an allocated one, +* saving one dereference. +* This might be changed if/when dynamic percpu vars become fast. +*/ +#ifdef CONFIG_SMP +# define DEFINE_PROTO_INUSE (NAME ) \ +static DEFINE_PER_CPU(int, NAME##_inuse); \ +static void NAME##_inuse_add(struct proto *prot, int inc) \ +{ \ + __get_cpu_var(NAME##_inuse) += inc; \ +} \ + \ +static int NAME##_inuse_getval(const struct proto *prot)\ +{ \ + int res = 0, cpu; \ + \ + for_each_possible_cpu(cpu) \ + res += per_cpu(NAME##_inuse, cpu); \ + return res; \ +} +# define REF_PROTO_INUSE (NAME ) \ + .inuse_add = NAME##_inuse_add, \ + .inuse_getval = NAME##_inuse_getval, +#else +# define DEFINE_PROTO_INUSE (NAME ) +# define REF_PROTO_INUSE (NAME ) +#endif +extern int proto_register (struct proto *prot,int alloc_slab);extern void proto_unregister (struct proto *prot);
@@ -629,12 +663,29 @@ static inline void sk_refcnt_debug_release(const struct sock *sk)/* Called with local bh disabled */ static __inline__void sock_prot_inc_use (struct proto *prot) { - prot->stats[smp_processor_id ()].inuse++; +#ifdef CONFIG_SMP + prot->inuse_add (prot,1 ); +#else + prot->inuse++; +#endif }static __inline__void sock_prot_dec_use (struct proto *prot) { - prot->stats[smp_processor_id ()].inuse--; +#ifdef CONFIG_SMP + prot->inuse_add (prot,-1 ); +#else + prot->inuse--; +#endif +} + +static __inline__int sock_prot_inuse (struct proto *proto) +{ +#ifdef CONFIG_SMP +return proto->inuse_getval (proto); +#else +return proto->inuse; +#endif }/* With per-bucket locks this operation is not-atomic, so that
net/core/sock.c +47 -1
78 lines not on this page · raw message
net/ipv4/proc.c +4 -15
@@ -46,17 +46,6 @@#include <net/sock.h> #include <net/raw.h> -static int fold_prot_inuse (struct proto *proto) -{ -int res =0 ; -int cpu; - -for_each_possible_cpu (cpu) - res += proto->stats[cpu].inuse; - -return res; -} -/* * Report socket allocation statistics [[email protected]] */
@@ -64,12 +53,12 @@ static int sockstat_seq_show(struct seq_file *seq, void *v) {socket_seq_show (seq);seq_printf (seq,"TCP: inuse %d orphan %d tw %d alloc %d mem %d\n" , -fold_prot_inuse (&tcp_prot),atomic_read (&tcp_orphan_count), +sock_prot_inuse (&tcp_prot),atomic_read (&tcp_orphan_count), tcp_death_row.tw_count,atomic_read (&tcp_sockets_allocated),atomic_read (&tcp_memory_allocated)); -seq_printf (seq,"UDP: inuse %d\n" ,fold_prot_inuse (&udp_prot)); -seq_printf (seq,"UDPLITE: inuse %d\n" ,fold_prot_inuse (&udplite_prot)); -seq_printf (seq,"RAW: inuse %d\n" ,fold_prot_inuse (&raw_prot)); +seq_printf (seq,"UDP: inuse %d\n" ,sock_prot_inuse (&udp_prot)); +seq_printf (seq,"UDPLITE: inuse %d\n" ,sock_prot_inuse (&udplite_prot)); +seq_printf (seq,"RAW: inuse %d\n" ,sock_prot_inuse (&raw_prot));seq_printf (seq,"FRAG: inuse %d memory %d\n" ,ip_frag_nqueues (),ip_frag_mem ());return 0 ;
net/ipv6/proc.c +4 -15
@@ -32,27 +32,16 @@static struct proc_dir_entry *proc_net_devsnmp6; -static int fold_prot_inuse (struct proto *proto) -{ -int res =0 ; -int cpu; - -for_each_possible_cpu (cpu) - res += proto->stats[cpu].inuse; - -return res; -} -static int sockstat6_seq_show (struct seq_file *seq,void *v) {seq_printf (seq,"TCP6: inuse %d\n" , -fold_prot_inuse (&tcpv6_prot)); +sock_prot_inuse (&tcpv6_prot));seq_printf (seq,"UDP6: inuse %d\n" , -fold_prot_inuse (&udpv6_prot)); +sock_prot_inuse (&udpv6_prot));seq_printf (seq,"UDPLITE6: inuse %d\n" , -fold_prot_inuse (&udplitev6_prot)); +sock_prot_inuse (&udplitev6_prot));seq_printf (seq,"RAW6: inuse %d\n" , -fold_prot_inuse (&rawv6_prot)); +sock_prot_inuse (&rawv6_prot));seq_printf (seq,"FRAG6: inuse %d memory %d\n" ,ip6_frag_nqueues (),ip6_frag_mem ());return 0 ;