core/percpu for v2.6.27
Subsystems (3)
Linus,
Please pull the latest core/percpu git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git core/percpu
Thanks,
Ingo
------------------>
Eric Dumazet (1):
percpu: introduce DEFINE_PER_CPU_PAGE_ALIGNED() macro
Heiko Carstens (1):
percpu: zero based percpu build error on s390
Mike Travis (2):
x86: extend percpu ops to 64 bit
Zero based percpu: infrastructure to rebase the per cpu area to zero
arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/vmlinux_32.lds.S | 1 + include/asm-generic/percpu.h | 11 ++++- include/asm-generic/sections.h | 10 ++++ include/asm-generic/vmlinux.lds.h | 17 ++++++++ include/asm-x86/percpu.h | 83 ++++++++++++++++++++++++++++++++++-- include/linux/percpu.h | 24 ++++++++++- kernel/lockdep.c | 4 +- kernel/module.c | 7 ++- 9 files changed, 146 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/common.c +1 -1
@@ -21,7 +21,7 @@#include "cpu.h" -DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = { +DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { [GDT_ENTRY_KERNEL_CS ] = { { {0x0000ffff ,0x00cf9a00 } } }, [GDT_ENTRY_KERNEL_DS ] = { { {0x0000ffff ,0x00cf9200 } } }, [GDT_ENTRY_DEFAULT_USER_CS ] = { { {0x0000ffff ,0x00cffa00 } } },
arch/x86/kernel/vmlinux_32.lds.S +1 -0
@@ -189,6 +189,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { __per_cpu_start = .; + *(.data.percpu.page_aligned) *(.data.percpu) *(.data.percpu.shared_aligned) __per_cpu_end = .;
include/asm-generic/percpu.h +10 -1
@@ -45,7 +45,12 @@ extern unsigned long __per_cpu_offset[NR_CPUS];* Only S390 provides its own means of moving the pointer. */ #ifndef SHIFT_PERCPU_PTR -#define SHIFT_PERCPU_PTR (__p, __offset) RELOC_HIDE((__p), (__offset)) +# ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +# define SHIFT_PERCPU_PTR (__p, __offset) \ + ((__typeof(__p))(((void *)(__p)) + (__offset))) +# else +# define SHIFT_PERCPU_PTR (__p, __offset) RELOC_HIDE((__p), (__offset)) +# endif /* CONFIG_HAVE_ZERO_BASED_PER_CPU */ #endif /*
@@ -70,6 +75,10 @@ extern void setup_per_cpu_areas(void);#define per_cpu (var, cpu) (*((void)(cpu), &per_cpu_var(var)))#define __get_cpu_var (var) per_cpu_var(var)#define __raw_get_cpu_var (var) per_cpu_var(var) +#ifndef SHIFT_PERCPU_PTR +# define SHIFT_PERCPU_PTR (__p, __offset) (__p) +#endif +#define per_cpu_offset (x) 0L#endif /* SMP */
include/asm-generic/sections.h +10 -0
@@ -9,7 +9,17 @@ extern char __bss_start[], __bss_stop[];extern char __init_begin[], __init_end[];extern char _sinittext[], _einittext[];extern char _end[]; +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +extern char __per_cpu_load[]; +extern char ____per_cpu_size[]; +#define __per_cpu_size ((unsigned long)&____per_cpu_size) +#define __per_cpu_start ((char *)0) +#define __per_cpu_end ((char *)__per_cpu_size) +#else extern char __per_cpu_start[], __per_cpu_end[]; +#define __per_cpu_load __per_cpu_start +#define __per_cpu_size (__per_cpu_end - __per_cpu_start) +#endif extern char __kprobes_text_start[], __kprobes_text_end[];extern char __initdata_begin[], __initdata_end[];extern char __start_rodata[], __end_rodata[];
include/asm-generic/vmlinux.lds.h +17 -0
@@ -344,11 +344,28 @@ *(.initcall7.init) \ *(.initcall7s.init) +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +#define PERCPU (align) \ + . = ALIGN(align); \ + percpu : { } :percpu \ + __per_cpu_load = .; \ + .data.percpu 0 : AT(__per_cpu_load - LOAD_OFFSET) { \ + *(.data.percpu.first) \ + *(.data.percpu.shared_aligned) \ + *(.data.percpu) \ + *(.data.percpu.page_aligned) \ + ____per_cpu_size = .; \ + } \ + . = __per_cpu_load + ____per_cpu_size; \ + data : { } :data +#else #define PERCPU (align) \ . = ALIGN(align); \ __per_cpu_start = .; \ .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \ + *(.data.percpu.page_aligned) \ *(.data.percpu) \ *(.data.percpu.shared_aligned) \ } \ __per_cpu_end = .; +#endif
include/asm-x86/percpu.h +78 -5
100 lines not on this page · raw message
include/linux/percpu.h +23 -1
@@ -23,13 +23,35 @@ __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ ____cacheline_aligned_in_smp + +#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.page_aligned"))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + +#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU +#define DEFINE_PER_CPU_FIRST(type, name) \ + __attribute__((__section__(".data.percpu.first"))) \ + PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name #else +#define DEFINE_PER_CPU_FIRST(type, name) \ + DEFINE_PER_CPU(type, name) +#endif + +#else /* !CONFIG_SMP */ + #define DEFINE_PER_CPU(type, name) \ PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ DEFINE_PER_CPU(type, name) -#endif + +#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) + +#define DEFINE_PER_CPU_FIRST(type, name) \ + DEFINE_PER_CPU(type, name) + +#endif /* !CONFIG_SMP */ #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
kernel/lockdep.c +2 -2
@@ -609,8 +609,8 @@ static int static_obj(void *obj)* percpu var? */ for_each_possible_cpu (i) { - start = (unsigned long ) &__per_cpu_start +per_cpu_offset (i); - end = (unsigned long ) &__per_cpu_start +PERCPU_ENOUGH_ROOM + start = (unsigned long ) __per_cpu_start +per_cpu_offset (i); + end = (unsigned long ) __per_cpu_start +PERCPU_ENOUGH_ROOM +per_cpu_offset (i);if ((addr >= start) && (addr < end))
kernel/module.c +4 -3
@@ -44,6 +44,7 @@#include <linux/unwind.h> #include <asm/uaccess.h> #include <asm/cacheflush.h> +#include <asm/sections.h> #include <linux/license.h> #include <asm/sections.h>
@@ -365,7 +366,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align, align =PAGE_SIZE ; } - ptr = __per_cpu_start; + ptr = __per_cpu_load;for (i =0 ; i < pcpu_num_used; ptr +=block_size (pcpu_size[i]), i++) {/* Extra for alignment requirement. */ extra =ALIGN ((unsigned long )ptr, align) - (unsigned long )ptr;
@@ -400,7 +401,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,static void percpu_modfree (void *freeme) {unsigned int i; -void *ptr = __per_cpu_start +block_size (pcpu_size[0 ]); +void *ptr = __per_cpu_load +block_size (pcpu_size[0 ]);/* First entry is core kernel percpu data. */ for (i =1 ; i < pcpu_num_used; ptr +=block_size (pcpu_size[i]), i++) {
@@ -451,7 +452,7 @@ static int percpu_modinit(void) pcpu_size =kmalloc (sizeof (pcpu_size[0 ]) * pcpu_num_allocated,GFP_KERNEL );/* Static in-kernel percpu data (used). */ - pcpu_size[0 ] = -(__per_cpu_end-__per_cpu_start); + pcpu_size[0 ] = -__per_cpu_size;/* Free room. */ pcpu_size[1 ] =PERCPU_ENOUGH_ROOM + pcpu_size[0 ];if (pcpu_size[1 ] <0 ) {
Quoted 4 lines · Ingo Molnar
Please pull the latest core/percpu git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.gitcore/percpu
Can you explain what this does and who needs it? The percpu_xchg() looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.
Linus
Quoted 1 line · Linus Torvalds
Can you explain what this does and who needs it? The
percpu_xchg()looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.
The essence of those changes is to work towards unifying the i386 and x86-64 percpu mechanisms. But I wasn't terribly convinced by some of the frills around the edges, like the xchg operation. The work is being driven by Mike Travis and the huge numa people, so perhaps they have a use for it.
J
Quoted 11 lines · Jeremy Fitzhardinge
Quoted 4 lines · Linus Torvalds
Can you explain what this does and who needs it? The
percpu_xchg()looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.The essence of those changes is to work towards unifying the i386 and x86-64 percpu mechanisms. But I wasn't terribly convinced by some of the frills around the edges, like the xchg operation. The work is being driven by Mike Travis and the huge numa people, so perhaps they have a use for it.
yeah. It all centers around the '4096 CPUs' topic which has so wide impact that it had to be scattered out a bit.
x86_percpu_xchg():
seems like a sensible extension of the API family to me. Linus is right about the LOCK overhead but i'm not all that sure that LOCKed instructions will always be slower. Combined with single-pointer per-cpu (not the PDA-based redirection we do now) it can be done safely without disabling preemption - and that would make the basic percpu ops usable in preemptible code.
But indeed there are no very strong reasons.
These are the currently pending related topics:
# tip/cpus4096 - ready for upstream, i'll send the pull request once # core/rcu is upstream
earth4:~/tip> git-log-line linus..cpus4096
9982fbf: Revert "cpumask: introduce new APIs"
68083e0: Merge commit 'v2.6.26-rc9' into cpus4096
7baac8b: cpumask: make for_each_cpu_mask a bit smaller 3f9b48a: net: Pass reference to cpumask variable in net/sunrpc/svc.c
cad0e45: clocksource/events: use performance variant for_each_cpu_mask_nr 5d7bfd0: infiniband: use performance variant for_each_cpu_mask_nr 334ef7a: x86: use performance variant for_each_cpu_mask_nr 0e12f84: net: use performance variant for_each_cpu_mask_nr 6d6a436: mm: use performance variant for_each_cpu_mask_nr 363ab6f: core: use performance variant for_each_cpu_mask_nr 068b127: cpufreq: use performance variant for_each_cpu_mask_nr 141ad06: acpi: use performance variant for_each_cpu_mask_nr 41df0d61: x86: Add performance variants of cpumask operators 143aa5c: cpu: change some globals to statics in drivers/base/cpu.c v2
a953e45: sched: replace MAX_NUMNODES with nr_node_ids in kernel/sched.c
# tip/core/percpu-zerobased: stalled - current approach is
# fragile to binutils bugs
earth4:~/tip> git-log-line linus..core/percpu-zerobased
a6d5d88: x86, 64-bit: replace xxx_pda() operations with x86_xxx_percpu(). 6b17441: x86, 64-bit: replace cpu_pda ops with percpu ops 6aa952c: x86, 64-bit: reference zero-based percpu variables offset from gs 8c76b15: x86, 64-bit: fold pda into per cpu area
673698e: x86, 64-bit: fix early references to cpumask_of_cpu
in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.
Ingo
Ingo Molnar wrote:
...
Quoted 5 lines
in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.
Ingo
Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)
Btw, is there a list of "bad" gcc's for kernel building? Or better yet, can the Makefile script provide a warning when a known "bad" gcc is being used to compile the kernel? I seem to recall that Peter provided this list:
4.2.3 is fine; he was using 4.2.0 before, and as far as I know, 4.2.0 and 4.2.1 are known broken for the kernel.
Thanks!
Mike
H. Peter Anvin
reviewerQuoted 4 lines · Mike Travis
4.2.3 is fine; he was using 4.2.0 before, and as far as I know, 4.2.0 and 4.2.1 are known broken for the kernel.
I might have been misremembering about 4.2.[01]; Adrian Bunk reminds me that 4.*1*.[01] are definitely broken, and I can't simply remember the purported problem with 4.2, so it *may* be a case of off-by-one mental error.
-hpa
Quoted 4 lines · Mike Travis
Ingo Molnar wrote:
...
Quoted 1 line
Quoted 8 lines
Quoted 4 lines
in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.
Ingo
Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)
well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?
Ingo
Quoted 9 lines · Ingo Molnar
Quoted 7 lines · Mike Travis
Ingo Molnar wrote:
...
Quoted 4 lines
in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.
Ingo
Quoted 8 lines
Quoted 3 lines
Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)
well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?
Ingo
I will test with gcc-4.2.3 as soon as I'm able, and will let you know. (There have been changes from when you hit the panic.)
I will resubmit against tip/master.
Thanks,
Mike
Quoted 9 lines · Ingo Molnar
Quoted 7 lines · Mike Travis
Ingo Molnar wrote:
...
Quoted 4 lines
in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.
Ingo
Quoted 8 lines
Quoted 3 lines
Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)
well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?
Ingo
YES! Patcheset to follow... ;-)
Thanks,
Mike