per-cpu change
Subsystems (6)
PATCH 1/1 per-cpu change
Hi David,
To implement kmalloc_percpu to use the same primitives that
normal (static) per-cpu vars use, I have made some changes. This is required for sane per-cpu variable support in modules, but the implementation also makes allocation v. efficient. I've had to make more assumptions about the per-cpu layout though.
IA64 is the only arch which doesn't use the generic struct
(x86-64 seems to be partway 8), so I wanted to check with you. Basically, we over-allocate up front, and pass out the rest through the __alloc_percpu interface. I've made the setup_per_cpu_areas() idempotent, so you should simply be able to call that from your cpu_init() (the later generic call will do nothing).
Please tell me if this is too restrictive for IA64. Ideally
it should be a simplification: you just need to override __get_cpu_var(var) and __get_cpu_ptr(var) for SMP in asm/percpu.h. You might want to set PERCPU_POOL_SIZE (16k would be plenty for my machine, but you currently do 64k).
Thanks,
Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.
Name: kmalloc_percpu to use same percpu operators
Author: Rusty Russell
Status: Tested on 2.5.68-bk11
D: By overallocating the per-cpu data at boot, we can make quite an
D: efficient allocator, and then use it to support per-cpu data in
D: modules (next patch).
include/asm-generic/percpu.h +3 -31
@@ -2,37 +2,9 @@#define _ASM_GENERIC_PERCPU_H_#include <linux/compiler.h> -#define __GENERIC_PER_CPU +/* Some archs may want to keep __per_cpu_offset for this CPU in a register. */ #ifdef CONFIG_SMP - -extern unsigned long __per_cpu_offset[NR_CPUS ]; - -/* Separate out the type, so (int[3], foo) works. */ -#ifndef MODULE -#define DEFINE_PER_CPU (type, name) \ - __attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu -#endif - -/* var is in discarded region: offset to particular copy we want */ -#define per_cpu (var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu]))#define __get_cpu_var (var) per_cpu(var, smp_processor_id()) - -#else /* ! SMP */ - -/* Can't define per-cpu variables in modules. Sorry --RR */ -#ifndef MODULE -#define DEFINE_PER_CPU (type, name) \ - __typeof__(type) name##__per_cpu -#endif - -#define per_cpu (var, cpu) ((void)cpu, var##__per_cpu) -#define __get_cpu_var (var) var##__per_cpu - -#endif /* SMP */ - -#define DECLARE_PER_CPU (type, name) extern __typeof__(type) name##__per_cpu - -#define EXPORT_PER_CPU_SYMBOL (var) EXPORT_SYMBOL(var##__per_cpu) -#define EXPORT_PER_CPU_SYMBOL_GPL (var) EXPORT_SYMBOL_GPL(var##__per_cpu) - +#define __get_cpu_ptr (var) per_cpu_ptr(ptr, smp_processor_id()) +#endif /* SMP */ #endif /* _ASM_GENERIC_PERCPU_H_ */
include/linux/genhd.h +1 -2
@@ -160,10 +160,9 @@ static inline void disk_stat_set_all(str#ifdef CONFIG_SMP static inline int init_disk_stats (struct gendisk *disk) { - disk->dkstats =kmalloc_percpu (sizeof (struct disk_stats ),GFP_KERNEL ); + disk->dkstats = kmalloc_percpu(struct disk_stats);if (!disk->dkstats)return 0 ; -disk_stat_set_all (disk,0 );return 1 ; }
include/linux/percpu.h +60 -46
131 lines not on this page · raw message
include/net/ipv6.h +1 -1
@@ -145,7 +145,7 @@ extern atomic_t inet6_sock_nr;int snmp6_register_dev (struct inet6_dev *idev);int snmp6_unregister_dev (struct inet6_dev *idev); -int snmp6_mib_init (void *ptr[2 ],size_t mibsize); +int snmp6_mib_init (void *ptr[2 ],size_t mibsize,size_t mibalign);void snmp6_mib_free (void *ptr[2 ]);struct ip6_ra_chain
init/main.c +0 -25
@@ -301,35 +301,10 @@ static void __init smp_init(void)#define smp_init () do { } while (0)#endif -static inline void setup_per_cpu_areas (void ) { }static inline void smp_prepare_cpus (unsigned int maxcpus) { }#else -#ifdef __GENERIC_PER_CPU -unsigned long __per_cpu_offset[NR_CPUS ]; - -static void __init setup_per_cpu_areas (void ) -{ -unsigned long size, i; -char *ptr; -/* Created by linker magic */ -extern char __per_cpu_start[], __per_cpu_end[]; - -/* Copy section for each CPU (we discard the original) */ - size =ALIGN (__per_cpu_end - __per_cpu_start,SMP_CACHE_BYTES ); -if (!size) -return ; - - ptr =alloc_bootmem (size *NR_CPUS ); - -for (i =0 ; i <NR_CPUS ; i++, ptr += size) { - __per_cpu_offset[i] = ptr - __per_cpu_start; -memcpy (ptr, __per_cpu_start, size); - } -} -#endif /* !__GENERIC_PER_CPU */ -/* Called by boot processor to activate the rest. */ static void __init smp_init (void ) {
kernel/ksyms.c +2 -2
@@ -99,7 +99,7 @@ EXPORT_SYMBOL(remove_shrinker);EXPORT_SYMBOL (kmalloc);EXPORT_SYMBOL (kfree);#ifdef CONFIG_SMP -EXPORT_SYMBOL (kmalloc_percpu); +EXPORT_SYMBOL (__alloc_percpu);EXPORT_SYMBOL (kfree_percpu);EXPORT_SYMBOL (percpu_counter_mod);#endif
@@ -607,7 +607,7 @@ EXPORT_SYMBOL(init_thread_union);EXPORT_SYMBOL (tasklist_lock);EXPORT_SYMBOL (find_task_by_pid);EXPORT_SYMBOL (next_thread); -#if defined(CONFIG_SMP ) && defined(__GENERIC_PER_CPU) +#if defined(CONFIG_SMP )EXPORT_SYMBOL (__per_cpu_offset);#endif
mm/Makefile +1 -0
@@ -12,3 +12,4 @@ obj-y := bootmem.o filemap.o mempool.o slab.o swap.o truncate.o vcache.o vmscan.o $(mmu-y) obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o +obj-$(CONFIG_SMP) += percpu.o
mm/percpu.c +218 -0
218 lines not on this page · raw message
mm/slab.c +0 -70
82 lines not on this page · raw message
net/ipv4/af_inet.c +10 -43
64 lines not on this page · raw message
net/ipv4/route.c +1 -9
@@ -2656,17 +2656,9 @@ int __init ip_rt_init(void) ipv4_dst_ops.gc_thresh = (rt_hash_mask +1 ); ip_rt_max_size = (rt_hash_mask +1 ) *16 ; - rt_cache_stat =kmalloc_percpu (sizeof (struct rt_cache_stat ), -GFP_KERNEL ); + rt_cache_stat = kmalloc_percpu(struct rt_cache_stat);if (!rt_cache_stat) goto out_enomem1; -for (i =0 ; i <NR_CPUS ; i++) { -if (cpu_possible (i)) { -memset (per_cpu_ptr (rt_cache_stat, i),0 , -sizeof (struct rt_cache_stat )); - } - } -devinet_init ();ip_fib_init ();
net/ipv6/af_inet6.c +9 -14
@@ -632,28 +632,20 @@ inet6_unregister_protosw(struct inet_pro }int -snmp6_mib_init (void *ptr[2 ],size_t mibsize) +snmp6_mib_init (void *ptr[2 ],size_t mibsize,size_t mibalign) {int i;if (ptr ==NULL )return -EINVAL ; - ptr[0 ] =kmalloc_percpu (mibsize,GFP_KERNEL ); + ptr[0 ] =__alloc_percpu (mibsize, mibalign);if (!ptr[0 ]) goto err0; - ptr[1 ] =kmalloc_percpu (mibsize,GFP_KERNEL ); + ptr[1 ] =__alloc_percpu (mibsize, mibalign);if (!ptr[1 ]) goto err1; - -/* Zero percpu version of the mibs */ -for (i =0 ; i <NR_CPUS ; i++) { -if (cpu_possible (i)) { -memset (per_cpu_ptr (ptr[0 ], i),0 , mibsize); -memset (per_cpu_ptr (ptr[1 ], i),0 , mibsize); - } - }return 0 ; err1:
@@ -675,11 +667,14 @@ snmp6_mib_free(void *ptr[2])static int __init init_ipv6_mibs (void ) { -if (snmp6_mib_init ((void **)ipv6_statistics,sizeof (struct ipv6_mib )) <0 ) +if (snmp6_mib_init ((void **)ipv6_statistics,sizeof (struct ipv6_mib ), + __alignof__(struct ipv6_mib )) <0 ) goto err_ip_mib; -if (snmp6_mib_init ((void **)icmpv6_statistics,sizeof (struct icmpv6_mib )) <0 ) +if (snmp6_mib_init ((void **)icmpv6_statistics,sizeof (struct icmpv6_mib ), + __alignof__(struct icmpv6_mib )) <0 ) goto err_icmp_mib; -if (snmp6_mib_init ((void **)udp_stats_in6,sizeof (struct udp_mib )) <0 ) +if (snmp6_mib_init ((void **)udp_stats_in6,sizeof (struct udp_mib ), + __alignof__(struct udp_mib )) <0 ) goto err_udp_mib;return 0 ;
net/ipv6/proc.c +1 -1
@@ -226,7 +226,7 @@ int snmp6_register_dev(struct inet6_devif (!idev || !idev->dev)return -EINVAL ; -if (snmp6_mib_init ((void **)idev->stats.icmpv6,sizeof (struct icmpv6_mib )) <0 ) +if (snmp6_mib_init ((void **)idev->stats.icmpv6,sizeof (struct icmpv6_mib ), __alignof__(struct icmpv6_mib )) <0 ) goto err_icmp;#ifdef CONFIG_PROC_FS
net/sctp/protocol.c +2 -14
@@ -853,26 +853,14 @@ static int __init init_sctp_mibs(void) {int i; - sctp_statistics[0 ] =kmalloc_percpu (sizeof (struct sctp_mib ), -GFP_KERNEL ); + sctp_statistics[0] = kmalloc_percpu(struct sctp_mib);if (!sctp_statistics[0 ])return -ENOMEM ; - sctp_statistics[1 ] =kmalloc_percpu (sizeof (struct sctp_mib ), -GFP_KERNEL ); + sctp_statistics[1] = kmalloc_percpu(struct sctp_mib);if (!sctp_statistics[1 ]) {kfree_percpu (sctp_statistics[0 ]);return -ENOMEM ; } - -/* Zero all percpu versions of the mibs */ -for (i =0 ; i <NR_CPUS ; i++) { -if (cpu_possible (i)) { -memset (per_cpu_ptr (sctp_statistics[0 ], i),0 , -sizeof (struct sctp_mib )); -memset (per_cpu_ptr (sctp_statistics[1 ], i),0 , -sizeof (struct sctp_mib )); - } - }return 0 ; }
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line · Rusty Russell
Rusty> Hi David, To implement kmalloc_percpu to use the same Rusty> primitives that normal (static) per-cpu vars use, I have made Rusty> some changes. This is required for sane per-cpu variable Rusty> support in modules, but the implementation also makes Rusty> allocation v. efficient. I've had to make more assumptions Rusty> about the per-cpu layout though.
Rusty> IA64 is the only arch which doesn't use the generic
Rusty> struct (x86-64 seems to be partway 8), so I wanted to check Rusty> with you. Basically, we over-allocate up front, and pass out Rusty> the rest through the __alloc_percpu interface. I've made the Rusty> setup_per_cpu_areas() idempotent, so you should simply be Rusty> able to call that from your cpu_init() (the later generic Rusty> call will do nothing).
Rusty> Please tell me if this is too restrictive for IA64.
Rusty> Ideally it should be a simplification: you just need to Rusty> override __get_cpu_var(var) and __get_cpu_ptr(var) for SMP in Rusty> asm/percpu.h. You might want to set PERCPU_POOL_SIZE (16k Rusty> would be plenty for my machine, but you currently do 64k).
We pin the per-CPU area into the TLB, so the size of this area is limited to one of the supported page sizes (yes, we _could_ pin multiple TLB entries, but I do not think that would be economical). At the moment, we use a 64KB page size. So as long as the per-CPU area is guaranteed never to be bigger than that, we should be fine.
Two other comments:
- On NUMA, you want to allocate the per-CPU areas in node-local memory.
- It's important to initialize the per-CPU area early enough. We currently do it in cpu_init(). Anything later would probably break things.
I am a little concerned with supporting per-CPU storage in modules. What happens if the per-CPU area fills up? With the normal kernel, you'll get a link-time error, but with modules, you won't know until you try to insert a module. But the alternative of using something similar to the user-level TLS model doesn't look exactly attractive either.
--david
In message <[email protected]> you write:
Quoted 1 line
- On NUMA, you want to allocate the per-CPU areas in node-local memory.
Thanks: I'd forgotten about this. Even "big smp" machines often have different memory latencies.
I need to allow archs to override the core allocation.
Quoted 3 lines
- It's important to initialize the per-CPU area early enough. We currently do it in
cpu_init(). Anything later would probably break things.
Yep, I imagine you'd explicitly call setup_per_cpu_areas() in cpu_init. (the later call will do nothing).
Quoted 4 lines
I am a little concerned with supporting per-CPU storage in modules. What happens if the per-CPU area fills up? With the normal kernel, you'll get a link-time error, but with modules, you won't know until you try to insert a module.
Yes. In fact, you get a boot error not a link error in my implementation.
My previous implementation used a linked-list of memory chunks, like so:
Core kernel percpu First dynamic percpu Second dynamic percpu +----+----+---...-+ -> +----+----+---...-+ -> +----+----+---...-+ -> CPU0 CPU1 CPU2...
You can then use the same __per_cpu_offset[], as the spacing between each cpu's variables in the dynamically allocated sections is the same as in the core kernel. You can __alloc_percpu any size up to the initial inter-cpu spacing (which was equal to the initial amount of static per-cpu data, or 4k, whatever is more).
But if the cpu's variables are not consecutive (ie your NUMA point), this scheme breaks down: you'd need to allocate areas with the same spacing between them, which won't happen.
So I think the current, simpler choice is preferable.
Quoted 2 lines
But the alternative of using something similar to the user-level TLS model doesn't look exactly attractive either.
Yes, I spoke with Alan Modra about what he had to do on PPC64, and ran screaming. We can live with more constraints, in the kernel, I think, especially when the result is so fast and simple.
Thanks,
Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.
In message <[email protected]> you write:
Quoted 1 line
Quoted 1 line
- On NUMA, you want to allocate the per-CPU areas in node-local memory.
Quoted 2 lines · Rusty Russell
Thanks: I'd forgotten about this. Even "big smp" machines often have different memory latencies.
I have already a patch to do this for ia32 NUMA systems (which present issues not present on sane architectures).
I also didn't notice the lack of an override; #undef'ing the macro __GENERIC_PER_CPU appeared to suffice for me.
-- wli