percpu-2.5.63-bk5-1 (properly generated)
Subsystems (6)
This patch does 3 different things:
(1) shoves per-cpu areas into node-local memory
(2) creates a new per-node thing analogous to per-cpu
(3) uses (1) and (2) to shove several frequently-accessed things into node-local memory
Tested, boots, and runs on NUMA-Q. Trims 6s of 41s off kernel compiles. Compiletested for walmart x86 SMP/UP, and could use runtime testing. A few non-x86 arches probably need fixups for per_cpu irq_stat[].
Also available at:
ftp://ftp.kernel.org/pub/linux/kernel/people/wli/percpu/
-- wli
arch/i386/kernel/apic.c | 2 arch/i386/kernel/io_apic.c | 2 arch/i386/kernel/irq.c | 2 arch/i386/kernel/nmi.c | 4 - arch/i386/kernel/process.c | 2 arch/i386/mm/discontig.c | 83 ++++++++++++++++++++++++--- arch/i386/mm/init.c | 4 - arch/i386/vmlinux.lds.S | 4 + include/asm-generic/percpu.h | 4 - include/asm-generic/pernode.h | 39 ++++++++++++ include/asm-i386/numaq.h | 9 +- include/asm-i386/percpu.h | 5 + include/asm-i386/pernode.h | 11 +++ include/asm-i386/srat.h | 3 include/asm-i386/tlb.h | 128 +++++++++++++++++++++++++++++++++++++++++- include/linux/irq_cpustat.h | 10 +-- include/linux/mm.h | 6 + init/main.c | 30 +++++++++ kernel/fork.c | 10 +-- kernel/ksyms.c | 2 kernel/sched.c | 18 ++--- kernel/softirq.c | 2 mm/page_alloc.c | 6 - mm/slab.c | 6 - 24 files changed, 338 insertions(+), 54 deletions(-)
arch/i386/kernel/apic.c +1 -1
@@ -1060,7 +1060,7 @@ void smp_apic_timer_interrupt(struct pt_/* * the NMI deadlock-detector uses this. */ - irq_stat[cpu].apic_timer_irqs++; +per_cpu (irq_stat, cpu).apic_timer_irqs++;/* * NOTE! We'd better ACK the irq immediately,
arch/i386/kernel/io_apic.c +1 -1
@@ -237,7 +237,7 @@ struct irq_cpu_info {#define IRQ_DELTA (cpu,irq) (irq_cpu_data[cpu].irq_delta[irq])#define IDLE_ENOUGH (cpu,now) \ - (idle_cpu(cpu) && ((now) - irq_stat[(cpu)].idle_timestamp > 1)) + (idle_cpu(cpu) && ((now) - per_cpu(irq_stat, cpu).idle_timestamp > 1))#define IRQ_ALLOWED (cpu,allowed_mask) \ ((1 << cpu) & (allowed_mask))
arch/i386/kernel/irq.c +1 -1
@@ -171,7 +171,7 @@ int show_interrupts(struct seq_file *p,seq_printf (p,"LOC: " );for (j =0 ; j <NR_CPUS ; j++)if (cpu_online (j)) - p +=seq_printf (p,"%10u " , irq_stat[j].apic_timer_irqs); + p +=seq_printf (p,"%10u " ,per_cpu (irq_stat, j).apic_timer_irqs);seq_putc (p,'\n' );#endif seq_printf (p,"ERR: %10u\n" ,atomic_read (&irq_err_count));
arch/i386/kernel/nmi.c +2 -2
@@ -76,7 +76,7 @@ int __init check_nmi_watchdog (void)printk (KERN_INFO "testing NMI watchdog ... " );for (cpu =0 ; cpu <NR_CPUS ; cpu++) - prev_nmi_count[cpu] = irq_stat[cpu].__nmi_count; + prev_nmi_count[cpu] =per_cpu (irq_stat, cpu).__nmi_count;local_irq_enable ();mdelay ((10 *1000 )/nmi_hz);// wait 10 ticks
@@ -358,7 +358,7 @@ void nmi_watchdog_tick (struct pt_regs **/ int sum, cpu =smp_processor_id (); - sum = irq_stat[cpu].apic_timer_irqs; + sum =per_cpu (irq_stat, cpu).apic_timer_irqs;if (last_irq_sums[cpu] == sum) {/*
arch/i386/kernel/process.c +1 -1
@@ -141,7 +141,7 @@ void cpu_idle (void)void (*idle)(void ) = pm_idle;if (!idle) idle = default_idle; - irq_stat[smp_processor_id ()].idle_timestamp = jiffies; +per_cpu (irq_stat,smp_processor_id ()).idle_timestamp = jiffies;while (!need_resched ())idle ();schedule ();
arch/i386/mm/discontig.c +74 -9
101 lines not on this page · raw message
arch/i386/mm/init.c +3 -1
@@ -41,7 +41,7 @@#include <asm/tlbflush.h> #include <asm/sections.h> -struct mmu_gather mmu_gathers[NR_CPUS ]; +DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);unsigned long highstart_pfn, highend_pfn;/*
@@ -372,7 +372,9 @@ void __init paging_init(void)__flush_tlb_all ();kmap_init (); +#ifndef CONFIG_DISCONTIGMEM zone_sizes_init (); +#endif }/*
arch/i386/vmlinux.lds.S +4 -0
@@ -83,6 +83,10 @@ SECTIONS .data.percpu : { *(.data.percpu) } __per_cpu_end = .; . = ALIGN(4096); + __per_node_start = .; + .data.pernode : { *(.data.pernode) } + __per_node_end = .; + . = ALIGN(4096); __init_end = .; /* freed after init ends here */
include/asm-generic/percpu.h +2 -2
@@ -25,8 +25,8 @@ extern unsigned long __per_cpu_offset[NR __typeof__(type) name##__per_cpu#endif -#define per_cpu (var, cpu) ((void)cpu, var##__per_cpu) -#define __get_cpu_var (var) var##__per_cpu +#define per_cpu (var, cpu) ( (void)(cpu), var##__per_cpu ) +#define __get_cpu_var (var) var##__per_cpu#endif /* SMP */
include/asm-generic/pernode.h +39 -0
@@ -0,0 +1,39 @@ +#ifndef _ASM_GENERIC_PERNODE_H_ +#define _ASM_GENERIC_PERNODE_H_ +#include <linux/config.h> +#include <linux/compiler.h> + +#define __GENERIC_PER_NODE +#ifdef CONFIG_DISCONTIGMEM + +extern unsigned long __per_node_offset[MAX_NR_NODES ]; + +/* Separate out the type, so (int[3], foo) works. */ +#ifndef MODULE +#define DEFINE_PER_NODE (type, name) \ + __attribute__((__section__(".data.pernode"))) __typeof__(type) name##__per_node +#endif + +/* var is in discarded region: offset to particular copy we want */ +#define per_node (var, node) (*RELOC_HIDE(&var##__per_node, __per_node_offset[node])) +#define __get_node_var (var) per_node(var, numa_node_id()) + +#else /* !CONFIG_DISCONTIGMEM */ + +/* Can't define per-node variables in modules. Sorry -- wli */ +#ifndef MODULE +#define DEFINE_PER_NODE (type, name) \ + __typeof__(type) name##__per_node +#endif + +#define per_node (var, node) ( (void)(node), var##__per_node ) +#define __get_node_var (var) var##__per_node + +#endif /* CONFIG_DISCONTIGMEM */ + +#define DECLARE_PER_NODE (type, name) extern __typeof__(type) name##__per_node + +#define EXPORT_PER_NODE_SYMBOL (var) EXPORT_SYMBOL(var##__per_node) +#define EXPORT_PER_NODE_SYMBOL_GPL (var) EXPORT_SYMBOL_GPL(var##__per_node) + +#endif /* _ASM_GENERIC_PERNODE_H_ */
include/asm-i386/numaq.h +5 -4
@@ -39,8 +39,9 @@extern int physnode_map[];#define pfn_to_nid (pfn) ({ physnode_map[(pfn) / PAGES_PER_ELEMENT]; })#define pfn_to_pgdat (pfn) NODE_DATA(pfn_to_nid(pfn)) -#define PHYSADDR_TO_NID (pa) pfn_to_nid(pa >> PAGE_SHIFT) -#define MAX_NUMNODES 8 +#define PHYSADDR_TO_NID (pa) pfn_to_nid((pa) >> PAGE_SHIFT) +#define MAX_NUMNODES 16 +#define MAX_NODE_CPUS 4extern void get_memcfg_numaq (void );#define get_memcfg_numa () get_memcfg_numaq()
@@ -169,9 +170,9 @@ struct sys_cfg_data {struct eachquadmem eq[MAX_NUMNODES ];/* indexed by quad id */ }; -static inline unsigned long get_zholes_size (int nid) +static inline unsigned long *get_zholes_size (int nid) { -return 0 ; +return NULL ; }#endif /* CONFIG_X86_NUMAQ */ #endif /* NUMAQ_H */
include/asm-i386/percpu.h +5 -0
@@ -3,4 +3,9 @@#include <asm-generic/percpu.h> +#ifdef CONFIG_NUMA +#undef __GENERIC_PER_CPU +void setup_per_cpu_areas (void ); +#endif +#endif /* __ARCH_I386_PERCPU__ */
include/asm-i386/pernode.h +11 -0
@@ -0,0 +1,11 @@ +#ifndef __ARCH_I386_PERNODE__ +#define __ARCH_I386_PERNODE__ + +#include <asm-generic/pernode.h> + +#ifdef CONFIG_DISCONTIGMEM +#undef __GENERIC_PER_NODE +void setup_per_node_areas (void ); +#endif + +#endif /* __ARCH_I386_PERNODE__ */
include/asm-i386/srat.h +2 -1
@@ -37,8 +37,9 @@extern int pfnnode_map[];#define pfn_to_nid (pfn) ({ pfnnode_map[PFN_TO_ELEMENT(pfn)]; })#define pfn_to_pgdat (pfn) NODE_DATA(pfn_to_nid(pfn)) -#define PHYSADDR_TO_NID (pa) pfn_to_nid(pa >> PAGE_SHIFT) +#define PHYSADDR_TO_NID (pa) pfn_to_nid((pa) >> PAGE_SHIFT)#define MAX_NUMNODES 8 +#define MAX_NODE_CPUS 4extern void get_memcfg_from_srat (void );extern unsigned long *get_zholes_size (int );#define get_memcfg_numa () get_memcfg_from_srat()
include/asm-i386/tlb.h +127 -1
139 lines not on this page · raw message
include/linux/irq_cpustat.h +4 -6
@@ -17,14 +17,12 @@* definitions instead of differing sets for each arch. */ -extern irq_cpustat_t irq_stat[];/* defined in asm/hardirq.h */ +/* defined in kernel/softirq.c */ +DECLARE_PER_CPU (irq_cpustat_t, irq_stat);#ifndef __ARCH_IRQ_STAT/* Some architectures can do this more efficiently */ -#ifdef CONFIG_SMP -#define __IRQ_STAT (cpu, member) (irq_stat[cpu].member) -#else -#define __IRQ_STAT (cpu, member) ((void)(cpu), irq_stat[0].member) -#endif + +#define __IRQ_STAT (cpu, member) (per_cpu(irq_stat, cpu).member)#endif /* arch independent irq_stat fields */
include/linux/mm.h +4 -2
@@ -26,6 +26,7 @@ extern int page_cluster;#include <asm/page.h> #include <asm/pgtable.h> #include <asm/atomic.h> +#include <asm/pernode.h> /* * Linux kernel virtual memory manager primitives.
@@ -318,11 +319,12 @@ static inline void put_page(struct page#define ZONE_SHIFT (BITS_PER_LONG - 8)struct zone ; -extern struct zone *zone_table[]; +DECLARE_PER_NODE(struct zone *[MAX_NR_ZONES], zone_table);static inline struct zone *page_zone (struct page *page) { -return zone_table[page->flags >>ZONE_SHIFT ]; +unsigned long zone = page->flags >>ZONE_SHIFT ; +return per_node (zone_table, zone/MAX_NR_ZONES )[zone %MAX_NR_ZONES ]; }static inline void set_page_zone (struct page *page,unsigned long zone_num)
init/main.c +30 -0
@@ -29,6 +29,7 @@#include <linux/tty.h> #include <linux/gfp.h> #include <linux/percpu.h> +#include <asm/pernode.h> #include <linux/kernel_stat.h> #include <linux/security.h> #include <linux/workqueue.h>
@@ -277,6 +278,10 @@ __setup("init=", init_setup);extern void setup_arch (char **);extern void cpu_idle (void ); +#ifndef CONFIG_NUMA +static inline void setup_per_node_areas (void ) { } +#endif +#ifndef CONFIG_SMP #ifdef CONFIG_X86_LOCAL_APIC
@@ -317,6 +322,30 @@ static void __init setup_per_cpu_areas(v }#endif /* !__GENERIC_PER_CPU */ +#if defined(__GENERIC_PER_NODE) && defined(CONFIG_NUMA ) +unsigned long __per_node_offset[MAX_NR_NODES ]; + +static void __init setup_per_node_areas (void ) +{ +unsigned long size, i; +char *ptr; +/* Created by linker magic */ +extern char __per_node_start[], __per_node_end[]; + +/* Copy section for each CPU (we discard the original) */ + size =ALIGN (__per_node_end - __per_node_start,SMP_CACHE_BYTES ); +if (!size) +return ; + + ptr =alloc_bootmem (size *MAX_NR_NODES ); + +for (i =0 ; i <MAX_NR_NODES ; i++, ptr += size) { + __per_node_offset[i] = ptr - __per_node_start; +memcpy (ptr, __per_node_start, size); + } +} +#endif /* __GENERIC_PER_NODE && CONFIG_NUMA */ +/* Called by boot processor to activate the rest. */ static void __init smp_init (void ) {
@@ -376,6 +405,7 @@ asmlinkage void __init start_kernel(voidprintk (linux_banner);setup_arch (&command_line);setup_per_cpu_areas (); +setup_per_node_areas ();/* * Mark the boot cpu "online" so that it can call console drivers in
kernel/fork.c +5 -5
@@ -58,7 +58,7 @@ rwlock_t tasklist_lock __cacheline_align* the very last portion of sys_exit() is executed with * preemption turned off. */ -static task_t *task_cache[NR_CPUS] __cacheline_aligned; +static DEFINE_PER_CPU(task_t *, task_cache);int nr_processes (void ) {
@@ -86,12 +86,12 @@ static void free_task_struct(struct task }else {int cpu =get_cpu (); - tsk = task_cache[cpu]; + tsk =per_cpu (task_cache, cpu);if (tsk) {free_thread_info (tsk->thread_info);kmem_cache_free (task_struct_cachep,tsk); } - task_cache[cpu] = current; +per_cpu (task_cache, cpu) = current;put_cpu (); } }
@@ -214,8 +214,8 @@ static struct task_struct *dup_task_strustruct thread_info *ti;int cpu =get_cpu (); - tsk = task_cache[cpu]; - task_cache[cpu] =NULL ; + tsk =per_cpu (task_cache, cpu); +per_cpu (task_cache, cpu) =NULL ;put_cpu ();if (!tsk) { ti =alloc_thread_info ();
kernel/ksyms.c +1 -1
@@ -405,7 +405,7 @@ EXPORT_SYMBOL(add_timer);EXPORT_SYMBOL (del_timer);EXPORT_SYMBOL (request_irq);EXPORT_SYMBOL (free_irq); -EXPORT_SYMBOL (irq_stat); +EXPORT_PER_CPU_SYMBOL (irq_stat);/* waitqueue handling */ EXPORT_SYMBOL (add_wait_queue);
kernel/sched.c +9 -9
@@ -32,6 +32,7 @@#include <linux/delay.h> #include <linux/timer.h> #include <linux/rcupdate.h> +#include <asm/pernode.h> /* * Convert user-nice values [ -20 ... 0 ... 19 ]
@@ -166,9 +167,9 @@ struct runqueue {atomic_t nr_iowait; }____cacheline_aligned ; -static struct runqueue runqueues[NR_CPUS] __cacheline_aligned; +static DEFINE_PER_CPU(struct runqueue, runqueues) = {{ 0 }}; -#define cpu_rq (cpu) (runqueues + (cpu)) +#define cpu_rq (cpu) (&per_cpu(runqueues, cpu))#define this_rq () cpu_rq(smp_processor_id())#define task_rq (p) cpu_rq(task_cpu(p))#define cpu_curr (cpu) (cpu_rq(cpu)->curr)
@@ -189,12 +190,11 @@ static struct runqueue runqueues[NR_CPUS* Keep track of running tasks. */ -static atomic_t node_nr_running[MAX_NUMNODES] ____cacheline_maxaligned_in_smp = - {[0 ...MAX_NUMNODES -1 ] =ATOMIC_INIT (0 )}; +static DEFINE_PER_NODE(atomic_t, node_nr_running) = ATOMIC_INIT(0);static inline void nr_running_init (struct runqueue *rq) { - rq->node_nr_running = &node_nr_running[0 ]; + rq->node_nr_running = &per_node (node_nr_running,0 ); }static inline void nr_running_inc (runqueue_t *rq)
@@ -214,7 +214,7 @@ __init void node_nr_running_init(void)int i;for (i =0 ; i <NR_CPUS ; i++) - cpu_rq(i)->node_nr_running = &node_nr_running[cpu_to_node(i)]; + cpu_rq(i)->node_nr_running = &per_node(node_nr_running, cpu_to_node(i)); }#else /* !CONFIG_NUMA */
@@ -748,7 +748,7 @@ static int sched_best_cpu(struct task_st minload =10000000 ;for (i =0 ; i < numnodes; i++) { - load =atomic_read (&node_nr_running[i]); + load =atomic_read (&per_node (node_nr_running, i));if (load < minload) { minload = load; node = i;
@@ -790,13 +790,13 @@ static int find_busiest_node(int this_noint i, node =-1 , load, this_load, maxload; this_load = maxload = (this_rq ()->prev_node_load[this_node] >>1 ) - +atomic_read (&node_nr_running[this_node]); + +atomic_read (&per_node (node_nr_running, this_node));this_rq ()->prev_node_load[this_node] = this_load;for (i =0 ; i < numnodes; i++) {if (i == this_node)continue ; load = (this_rq ()->prev_node_load[i] >>1 ) - +atomic_read (&node_nr_running[i]); + +atomic_read (&per_node (node_nr_running, i));this_rq ()->prev_node_load[i] = load;if (load > maxload && (100 *load >NODE_THRESHOLD *this_load)) { maxload = load;
kernel/softirq.c +1 -1
@@ -32,7 +32,7 @@- Tasklets: serialized wrt itself. */ -irq_cpustat_t irq_stat[NR_CPUS ]____cacheline_aligned ; +DEFINE_PER_CPU (irq_cpustat_t, irq_stat); static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
mm/page_alloc.c +3 -3
@@ -44,8 +44,8 @@ int sysctl_lower_zone_protection = 0;* Used by page_zone() to look up the address of the struct zone whose * id is encoded in the upper bits of page->flags */ -struct zone *zone_table[MAX_NR_ZONES *MAX_NR_NODES ]; -EXPORT_SYMBOL (zone_table); +DEFINE_PER_NODE(struct zone *[MAX_NR_ZONES], zone_table); +EXPORT_PER_NODE_SYMBOL (zone_table);static char *zone_names[MAX_NR_ZONES ] = {"DMA" ,"Normal" ,"HighMem" };static int zone_balance_ratio[MAX_NR_ZONES ]__initdata = {128 ,128 ,128 , };
@@ -1170,7 +1170,7 @@ static void __init free_area_init_core(sunsigned long size, realsize;unsigned long batch; - zone_table[nid *MAX_NR_ZONES + j] = zone; +per_node (zone_table, nid)[j] = zone; realsize = size = zones_size[j];if (zholes_size) realsize -= zholes_size[j];
mm/slab.c +3 -3
@@ -462,7 +462,7 @@ enum {FULL } g_cpucache_up; -static struct timer_list reap_timers[NR_CPUS ]; +static DEFINE_PER_CPU(struct timer_list, reap_timers);static void reap_timer_fnc (unsigned long data);
@@ -516,7 +516,7 @@ static void __slab_error(const char *fun*/ static void start_cpu_timer (int cpu) { -struct timer_list *rt = &reap_timers[cpu]; +struct timer_list *rt = &per_cpu (reap_timers, cpu);if (rt->function ==NULL ) {init_timer (rt);
@@ -2180,7 +2180,7 @@ next:static void reap_timer_fnc (unsigned long data) {int cpu =smp_processor_id (); -struct timer_list *rt = &reap_timers[cpu]; +struct timer_list *rt = &per_cpu (reap_timers, cpu);cache_reap ();mod_timer (rt, jiffies +REAPTIMEOUT_CPUC + cpu);
Quoted 10 lines · William Lee Irwin III
This patch does 3 different things:
(1) shoves per-cpu areas into node-local memory
(2) creates a new per-node thing analogous to per-cpu
(3) uses (1) and (2) to shove several frequently-accessed things into node-local memory
Tested, boots, and runs on NUMA-Q. Trims 6s of 41s off kernel compiles. Compiletested for walmart x86 SMP/UP, and could use runtime testing. A few non-x86 arches probably need fixups forper_cpuirq_stat[]. Also available at:
ftp://ftp.kernel.org/pub/linux/kernel/people/wli/percpu/
Okay, I got requests for a more detailed changelog, so here it is:
This patch does 19 different things when put under the microscope and/or in an excessively finegrained subdivision of simple concepts.
(1) reuse the arch/i386/discontigmem.c per-node mem_map[] virtual remap to remap node-local memory backing per_cpu and per_node areas
(2) make irq_stat[] per_cpu, x86-only (3) make mmu_gathers[] per_cpu, with comcomitant divorce from asm-generic (4) delay discontig zone_sizes_init() to dodge bootstrap order issues (5) add .data.pernode section handling in vmlinux.lds.S (6) per_cpu()/__get_cpu_var() needs to parenthesize the cpu arg (7) introduced asm-generic/pernode.h to do similar things as percpu.h (8) added MAX_NODE_CPUS for pessimistic sizing of virtual remapping arenas (9) fix return type error in NUMA-Q get_zholes_size()
(10) #undef asm-i386/per{cpu,node}.h's __GENERIC_PER_{CPU,NODE}
(11) declare setup_per_cpu_areas() in asm-i386/percpu.h
(12) make an asm-i386/pernode.h stub header like include/asm-generic/pernode.h
(13) declare MAX_NODE_CPUS in include/asm-i386/srat.h
(14) make zone_table[] per_node
(15) call setup_per_node_areas() in init/main.c, with analogous hooks
(16) make task_cache per_cpu
(17) make runqueues[] per_cpu
(18) make node_nr_running[] per_node
(19) make reap_timers[] per_cpu
-- wli