percpu-2.5.63-bkcurr
Shove per-cpu areas into node-local memory for i386 discontigmem, or at least NUMA-Q. You'll have to plop down early_cpu_to_node() and early_node_to_cpumask() stubs to use it on, say Summit.
-- wli
===== arch/i386/mm/discontig.c 1.9 vs edited =====
arch/i386/mm/discontig.c +62 -9
@@ -48,8 +48,6 @@extern unsigned long totalram_pages;extern unsigned long totalhigh_pages; -#define LARGE_PAGE_BYTES (PTRS_PER_PTE * PAGE_SIZE) -unsigned long node_remap_start_pfn[MAX_NUMNODES ];unsigned long node_remap_size[MAX_NUMNODES ];unsigned long node_remap_offset[MAX_NUMNODES ];
@@ -67,6 +65,20 @@ node_end_pfn[nid] = max_pfn; } +extern char __per_cpu_start[], __per_cpu_end[]; +unsigned long __per_cpu_offset[NR_CPUS ]; + +#define PER_CPU_PAGES PFN_UP((unsigned long)(__per_cpu_end-__per_cpu_start)) +#define MEM_MAP_SIZE (n) PFN_UP((node_end_pfn[n]-node_start_pfn[n]+1)*sizeof(struct page)) + +#ifdef CONFIG_X86_NUMAQ +#define early_cpu_to_node (cpu) ((cpu)/4) +#define early_node_to_cpumask (node) (0xFUL << (4*(node))) +#else +#define early_cpu_to_node (cpu) cpu_to_node(cpu) +#define early_node_to_cpumask (node) node_to_cpumask(node) +#endif +/* * Allocate memory for the pg_data_t via a crude pre-bootmem method * We ought to relocate these onto their own node later on during boot.
@@ -82,6 +94,44 @@ } } +static void __init allocate_one_cpu_area (int cpu) +{ +int cpu_in_node, node =early_cpu_to_node (cpu); +unsigned long nodemask =early_node_to_cpumask (node); +unsigned long node_vaddr = (unsigned long )node_remap_start_vaddr[node]; + +if (!PER_CPU_PAGES ) +return ; + +if (!node) { + __per_cpu_offset[cpu] = min_low_pfn*PAGE_SIZE + +PAGE_OFFSET + - (unsigned long )__per_cpu_start; + min_low_pfn +=PER_CPU_PAGES ; +return ; + } + + cpu_in_node =hweight32 (nodemask & ((1UL << cpu) -1 )); + __per_cpu_offset[cpu] = node_vaddr +MEM_MAP_SIZE (node)*PAGE_SIZE + +PFN_UP (sizeof (pg_data_t))*PAGE_SIZE + +PER_CPU_PAGES *cpu_in_node*PAGE_SIZE + - (unsigned long )__per_cpu_start; +} + +void __init setup_per_cpu_areas (void ) +{ +int node, cpu; +for (node =0 ; node < numnodes; ++node) { +for (cpu =0 ; cpu <NR_CPUS ; ++cpu) { +if (early_cpu_to_node (cpu) == node) { +memcpy (RELOC_HIDE ((char *)__per_cpu_start, __per_cpu_offset[cpu]), + __per_cpu_start, +PER_CPU_PAGES *PAGE_SIZE ); + } + } + } +} +/* * Register fully available low RAM pages with the bootmem allocator. */
@@ -144,13 +194,11 @@unsigned long size, reserve_pages =0 ;for (nid =1 ; nid < numnodes; nid++) { -/* calculate the size of the mem_map needed in bytes */ - size = (node_end_pfn[nid] - node_start_pfn[nid] +1 ) - *sizeof (struct page ) +sizeof (pg_data_t); -/* convert size to large (pmd size) pages, rounding up */ - size = (size +LARGE_PAGE_BYTES -1 ) /LARGE_PAGE_BYTES ; -/* now the roundup is correct, convert to PAGE_SIZE pages */ - size = size *PTRS_PER_PTE ; +/* calculate the size of the mem_map needed in pages */ + size =MEM_MAP_SIZE (nid) +PFN_UP (sizeof (pg_data_t)) + +PER_CPU_PAGES *hweight32 (early_node_to_cpumask (nid)); +/* round up to nearest pmd boundary */ + size = (size +PTRS_PER_PTE -1 ) & ~(PTRS_PER_PTE -1 );printk ("Reserving %ld pages of KVA for lmem_map of node %d\n" , size, nid); node_remap_size[nid] = size;
@@ -196,9 +244,14 @@ printk("Low memory ends at vaddr %08lx\n", (ulong) pfn_to_kaddr(max_low_pfn)); for (nid = 0; nid < numnodes; nid++) { + int cpu; node_remap_start_vaddr[nid] = pfn_to_kaddr( highstart_pfn - node_remap_offset[nid]); allocate_pgdat(nid); + for (cpu = 0; cpu < NR_CPUS; ++cpu) { + if (early_cpu_to_node(cpu) == nid) + allocate_one_cpu_area(cpu); + } printk ("node %d will remap to vaddr %08lx - %08lx\n", nid, (ulong) node_remap_start_vaddr[nid], (ulong) pfn_to_kaddr(highstart_pfn
===== include/asm-i386/numaq.h 1.7 vs edited =====
include/asm-i386/numaq.h +2 -2
@@ -169,9 +169,9 @@struct eachquadmem eq[MAX_NUMNODES ];/* indexed by quad id */ }; -static inline unsigned long get_zholes_size (int nid) +static inline unsigned long *get_zholes_size (int nid) { -return 0 ; +return NULL ; }#endif /* CONFIG_X86_NUMAQ */ #endif /* NUMAQ_H */
===== include/asm-i386/percpu.h 1.1 vs edited =====
include/asm-i386/percpu.h +24 -0
@@ -1,6 +1,30 @@#ifndef __ARCH_I386_PERCPU__#define __ARCH_I386_PERCPU__ +#include <linux/config.h> +#include <linux/compiler.h> + +#ifndef CONFIG_DISCONTIGMEM #include <asm-generic/percpu.h> +#else /* CONFIG_DISCONTIGMEM */ + +extern unsigned long __per_cpu_offset[NR_CPUS ]; +void setup_per_cpu_areas (void ); + +/* Separate out the type, so (int[3], foo) works. */ +#ifndef MODULE +#define DEFINE_PER_CPU (type, name) \ + __attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu +#endif + +/* var is in discarded region: offset to particular copy we want */ +#define per_cpu (var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu])) +#define __get_cpu_var (var) per_cpu(var, smp_processor_id()) + +#define DECLARE_PER_CPU (type, name) extern __typeof__(type) name##__per_cpu +#define EXPORT_PER_CPU_SYMBOL (var) EXPORT_SYMBOL(var##__per_cpu) +#define EXPORT_PER_CPU_SYMBOL_GPL (var) EXPORT_SYMBOL_GPL(var##__per_cpu) + +#endif /* CONFIG_DISCONTIGMEM */ #endif /* __ARCH_I386_PERCPU__ */
Quoted 3 lines · William Lee Irwin III
Shove per-cpu areas into node-local memory for i386 discontigmem, or at least NUMA-Q. You'll have to plop down
early_cpu_to_node()andearly_node_to_cpumask()stubs to use it on, say Summit.
Tentative followup #1 (thanks Zwane!)
Use per-cpu rq's in the sched.c to avoid remote cache misses there. It actually means something now.
Index: linux-2.5.59/kernel/sched.c
===================================================================
RCS file: /build/cvsroot/linux-2.5.59/kernel/sched.c,v retrieving revision 1.1.1.1
kernel/sched.c +2 -2
@@ -160,9 +160,9 @@atomic_t nr_iowait; }____cacheline_aligned ; -static struct runqueue runqueues[NR_CPUS] __cacheline_aligned; +static DEFINE_PER_CPU(struct runqueue, runqueues) = {{ 0 }}; -#define cpu_rq (cpu) (runqueues + (cpu)) +#define cpu_rq (cpu) (&per_cpu(runqueues, cpu))#define this_rq () cpu_rq(smp_processor_id())#define task_rq (p) cpu_rq(task_cpu(p))#define cpu_curr (cpu) (cpu_rq(cpu)->curr)
Signature · 1 line
function.linuxpower.ca
Quoted 3 lines · William Lee Irwin III
Quoted 3 lines
Shove per-cpu areas into node-local memory for i386 discontigmem, or at least NUMA-Q. You'll have to plop down
early_cpu_to_node()andearly_node_to_cpumask()stubs to use it on, say Summit.
Quoted 3 lines · William Lee Irwin III
Tentative followup #1 (thanks Zwane!)
Use per-cpu rq's in thesched.cto avoid remote cache misses there. It actually means something now.
Tentative followup #2 -- totally untested, at some point I have to figure out how to avoid breaking the compile for non-NUMA-Q with this.
arch/i386/mm/discontig.c +36 -1
@@ -66,10 +66,12 @@ }extern char __per_cpu_start[], __per_cpu_end[]; -unsigned long __per_cpu_offset[NR_CPUS ]; +extern char __per_node_start[], __per_node_end[]; +unsigned long __per_cpu_offset[NR_CPUS ], __per_node_offset[MAX_NR_NODES ];#define PER_CPU_PAGES PFN_UP((unsigned long)(__per_cpu_end-__per_cpu_start))#define MEM_MAP_SIZE (n) PFN_UP((node_end_pfn[n]-node_start_pfn[n]+1)*sizeof(struct page)) +#define PER_NODE_PAGES PFN_UP((unsigned long)(__per_node_end-__per_node_start))#ifdef CONFIG_X86_NUMAQ #define early_cpu_to_node (cpu) ((cpu)/4)
@@ -94,6 +96,26 @@ } } +static void __init allocate_per_node_area (int node) +{ +unsigned long node_vaddr = (unsigned long )node_remap_start_vaddr[node]; + +if (!PER_CPU_PAGES ) +return ; + +if (!node) { + __per_node_offset[node] = min_low_pfn*PAGE_SIZE + +PAGE_OFFSET + - (unsigned long )__per_node_start; + min_low_pfn +=PER_NODE_PAGES ; +return ; + } + + __per_node_offset[node] = node_vaddr +MEM_MAP_SIZE (node)*PAGE_SIZE + +PFN_UP (sizeof (pg_data_t))*PAGE_SIZE + - (unsigned long )__per_node_start; +} +static void __init allocate_one_cpu_area (int cpu) {int cpu_in_node, node =early_cpu_to_node (cpu);
@@ -115,9 +137,21 @@ __per_cpu_offset[cpu] = node_vaddr +MEM_MAP_SIZE (node)*PAGE_SIZE +PFN_UP (sizeof (pg_data_t))*PAGE_SIZE +PER_CPU_PAGES *cpu_in_node*PAGE_SIZE + +PER_NODE_PAGES *PAGE_SIZE - (unsigned long )__per_cpu_start; } +void __init setup_per_node_areas (void ) +{ +int node; +for (node =0 ; node < numnodes; ++node) { +memcpy (RELOC_HIDE ((char *)__per_node_start, + __per_node_offset[node]), + __per_node_start, +PER_NODE_PAGES *PAGE_SIZE ); + } +} +void __init setup_per_cpu_areas (void ) {int node, cpu;
@@ -248,6 +282,7 @@ node_remap_start_vaddr[nid] =pfn_to_kaddr ( highstart_pfn - node_remap_offset[nid]);allocate_pgdat (nid); +allocate_per_node_area (nid);for (cpu =0 ; cpu <NR_CPUS ; ++cpu) {if (early_cpu_to_node (cpu) == nid)allocate_one_cpu_area (cpu);
arch/i386/vmlinux.lds.S +4 -0
@@ -83,6 +83,10 @@ .data.percpu : { *(.data.percpu) } __per_cpu_end = .; . = ALIGN(4096); + __per_node_start = .; + .data.pernode : { *(.data.pernode) } + __per_node_end = .; + . = ALIGN(4096); __init_end = .; /* freed after init ends here */
include/asm-i386/pernode.h +29 -0
@@ -0,0 +1,29 @@ +#ifndef __ARCH_I386_PERNODE__ +#define __ARCH_I386_PERNODE__ + +#include <linux/config.h> +#include <linux/compiler.h> + +#ifndef CONFIG_DISCONTIGMEM +#include <asm-generic/percpu.h> +#else /* CONFIG_DISCONTIGMEM */ + +extern unsigned long __per_node_offset[NR_CPUS ]; +void setup_per_node_areas (void ); + +/* Separate out the type, so (int[3], foo) works. */ +#ifndef MODULE +#define DEFINE_PER_NODE (type, name) \ + __attribute__((__section__(".data.pernode"))) __typeof__(type) name##__per_node +#endif + +/* var is in discarded region: offset to particular copy we want */ +#define per_node (var, node) (*RELOC_HIDE(&var##__per_node, __per_node_offset[node])) + +#define DECLARE_PER_NODE (type, name) extern __typeof__(type) name##__per_node +#define EXPORT_PER_NODE_SYMBOL (var) EXPORT_SYMBOL(var##__per_node) +#define EXPORT_PER_NODE_SYMBOL_GPL (var) EXPORT_SYMBOL_GPL(var##__per_node) + +#endif /* CONFIG_DISCONTIGMEM */ + +#endif /* __ARCH_I386_PERNODE__ */
init/main.c +6 -0
@@ -38,6 +38,7 @@#include <asm/io.h> #include <asm/bugs.h> +#include <asm/pernode.h> #ifdef CONFIG_X86_LOCAL_APIC #include <asm/smp.h>
@@ -317,6 +318,10 @@ }#endif /* !__GENERIC_PER_CPU */ +#ifndef CONFIG_NUMA +static inline void setup_per_node_areas (void ) { } +#endif +/* Called by boot processor to activate the rest. */ static void __init smp_init (void ) {
@@ -376,6 +381,7 @@printk (linux_banner);setup_arch (&command_line);setup_per_cpu_areas (); +setup_per_node_areas ();/* * Mark the boot cpu "online" so that it can call console drivers in
kernel/sched.c +9 -9
@@ -32,6 +32,7 @@#include <linux/delay.h> #include <linux/timer.h> #include <linux/rcupdate.h> +#include <asm/pernode.h> /* * Convert user-nice values [ -20 ... 0 ... 19 ]
@@ -166,9 +167,9 @@atomic_t nr_iowait; }____cacheline_aligned ; -static struct runqueue runqueues[NR_CPUS] __cacheline_aligned; +static DEFINE_PER_CPU(struct runqueue, runqueues) = {{ 0 }}; -#define cpu_rq (cpu) (runqueues + (cpu)) +#define cpu_rq (cpu) (&per_cpu(runqueues, cpu))#define this_rq () cpu_rq(smp_processor_id())#define task_rq (p) cpu_rq(task_cpu(p))#define cpu_curr (cpu) (cpu_rq(cpu)->curr)
@@ -189,12 +190,11 @@* Keep track of running tasks. */ -static atomic_t node_nr_running[MAX_NUMNODES] ____cacheline_maxaligned_in_smp = - {[0 ...MAX_NUMNODES -1 ] =ATOMIC_INIT (0 )}; +static DEFINE_PER_NODE(atomic_t, node_nr_running) = ATOMIC_INIT(0);static inline void nr_running_init (struct runqueue *rq) { - rq->node_nr_running = &node_nr_running[0 ]; + rq->node_nr_running = &per_node (node_nr_running,0 ); }static inline void nr_running_inc (runqueue_t *rq)
@@ -214,7 +214,7 @@int i;for (i =0 ; i <NR_CPUS ; i++) - cpu_rq(i)->node_nr_running = &node_nr_running[cpu_to_node(i)]; + cpu_rq(i)->node_nr_running = &per_node(node_nr_running, cpu_to_node(i)); }#else /* !CONFIG_NUMA */
@@ -748,7 +748,7 @@ minload =10000000 ;for (i =0 ; i < numnodes; i++) { - load =atomic_read (&node_nr_running[i]); + load =atomic_read (&per_node (node_nr_running, i));if (load < minload) { minload = load; node = i;
@@ -790,13 +790,13 @@int i, node =-1 , load, this_load, maxload; this_load = maxload = (this_rq ()->prev_node_load[this_node] >>1 ) - +atomic_read (&node_nr_running[this_node]); + +atomic_read (&per_node (node_nr_running, this_node));this_rq ()->prev_node_load[this_node] = this_load;for (i =0 ; i < numnodes; i++) {if (i == this_node)continue ; load = (this_rq ()->prev_node_load[i] >>1 ) - +atomic_read (&node_nr_running[i]); + +atomic_read (&per_node (node_nr_running, i));this_rq ()->prev_node_load[i] = load;if (load > maxload && (100 *load >NODE_THRESHOLD *this_load)) { maxload = load;
Quoted 3 lines · William Lee Irwin III
Tentative followup #1 (thanks Zwane!)
Quoted 2 lines
Use per-cpu rq's in the
sched.cto avoid remote cache misses there. It actually means something now.
Quoted 2 lines · William Lee Irwin III
Tentative followup #2 -- totally untested, at some point I have to figure out how to avoid breaking the compile for non-NUMA-Q with this.
woops
arch/i386/mm/discontig.c +1 -0
@@ -230,6 +230,7 @@for (nid =1 ; nid < numnodes; nid++) {/* calculate the size of the mem_map needed in pages */ size =MEM_MAP_SIZE (nid) +PFN_UP (sizeof (pg_data_t)) + +PER_NODE_PAGES +PER_CPU_PAGES *hweight32 (early_node_to_cpumask (nid));/* round up to nearest pmd boundary */ size = (size +PTRS_PER_PTE -1 ) & ~(PTRS_PER_PTE -1 );
Quoted 2 lines · William Lee Irwin III
Quoted 2 lines
Quoted 2 lines
Use per-cpu rq's in the
sched.cto avoid remote cache misses there. It actually means something now.
Quoted 2 lines · William Lee Irwin III
Quoted 2 lines
Tentative followup #2 -- totally untested, at some point I have to figure out how to avoid breaking the compile for non-NUMA-Q with this.
Tested. it works.
-- wli