percpu: Make PER_CPU_BASE_SECTION overridable by arches
Subsystems (3)
PATCH 1/3 percpu: Make PER_CPU_BASE_SECTION overridable by arches
Impact: bug fix
IA-64 needs to put percpu data in the seperate section even on UP. Fixes regression caused by "percpu: refactor percpu.h"
--- arch/ia64/include/asm/percpu.h | 4 ++-- include/linux/percpu.h | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-)
arch/ia64/include/asm/percpu.h +2 -2
@@ -27,12 +27,12 @@ extern void *per_cpu_init(void);#else /* ! SMP */ -#define PER_CPU_ATTRIBUTES __attribute__((__section__(".data.percpu"))) -#define per_cpu_init () (__phys_per_cpu_start)#endif /* SMP */ +#define PER_CPU_BASE_SECTION ".data.percpu" +/* * Be extremely careful when taking the address of this variable! Due to virtual * remapping, it is different from the canonical address returned by __get_cpu_var(var)!
include/linux/percpu.h +7 -1
@@ -8,8 +8,15 @@#include <asm/percpu.h> +#ifndef PER_CPU_BASE_SECTION #ifdef CONFIG_SMP #define PER_CPU_BASE_SECTION ".data.percpu" +#else +#define PER_CPU_BASE_SECTION ".data" +#endif +#endif + +#ifdef CONFIG_SMP #ifdef MODULE #define PER_CPU_SHARED_ALIGNED_SECTION ""
@@ -20,7 +27,6 @@#else -#define PER_CPU_BASE_SECTION ".data"#define PER_CPU_SHARED_ALIGNED_SECTION ""#define PER_CPU_FIRST_SECTION ""
Signature · 1 line
1.6.1
PATCH 2/3 x86: Use linker to offset symbols by __per_cpu_load
Impact: cleanup and bug fix
Use the linker to create symbols for certain per-cpu variables that are offset by __per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.
--- arch/x86/include/asm/percpu.h | 22 ++++++++++++++++++++++ arch/x86/include/asm/processor.h | 2 ++ arch/x86/kernel/cpu/common.c | 6 +----- arch/x86/kernel/head_64.S | 21 ++------------------- arch/x86/kernel/vmlinux_64.lds.S | 8 ++++++++ 5 files changed, 35 insertions(+), 24 deletions(-)
arch/x86/include/asm/percpu.h +22 -0
@@ -34,6 +34,12 @@#define PER_CPU_VAR (var) per_cpu__##var#endif /* SMP */ +#ifdef CONFIG_X86_64_SMP +#define INIT_PER_CPU_VAR (var) init_per_cpu__##var +#else +#define INIT_PER_CPU_VAR (var) per_cpu__##var +#endif +#else /* ...!ASSEMBLY */ #include <linux/stringify.h>
@@ -45,6 +51,22 @@#define __percpu_arg (x) "%" #x#endif +/* +* Initialized pointers to per-cpu variables needed for the boot +* processor need to use these macros to get the proper address +* offset from __per_cpu_load on SMP. +* +* There also must be an entry in vmlinux_64.lds.S +*/ +#define DECLARE_INIT_PER_CPU (var) \ + extern typeof(per_cpu_var(var)) init_per_cpu_var(var) + +#ifdef CONFIG_X86_64_SMP +#define init_per_cpu_var (var) init_per_cpu__##var +#else +#define init_per_cpu_var (var) per_cpu_var(var) +#endif +/* For arch-specific code, we can use direct single-insn ops (they * don't give an lvalue though). */ extern void __bad_percpu_size (void );
arch/x86/include/asm/processor.h +2 -0
@@ -393,6 +393,8 @@ union irq_stack_union { }; DECLARE_PER_CPU(union irq_stack_union, irq_stack_union); +DECLARE_INIT_PER_CPU (irq_stack_union); + DECLARE_PER_CPU(char *, irq_stack_ptr);#endif
arch/x86/kernel/cpu/common.c +1 -5
@@ -902,12 +902,8 @@ struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; DEFINE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union)__aligned(PAGE_SIZE) ; -#ifdef CONFIG_SMP -DEFINE_PER_CPU(char *, irq_stack_ptr); /* will be set during per cpu init */ -#else DEFINE_PER_CPU(char *, irq_stack_ptr) = -per_cpu_var (irq_stack_union.irq_stack) +IRQ_STACK_SIZE -64 ; -#endif +init_per_cpu_var (irq_stack_union.irq_stack) +IRQ_STACK_SIZE -64 ; DEFINE_PER_CPU(unsigned long, kernel_stack) = (unsigned long )&init_thread_union -KERNEL_STACK_OFFSET +THREAD_SIZE ;
arch/x86/kernel/head_64.S +2 -19
@@ -205,19 +205,6 @@ ENTRY(secondary_startup_64) pushq $0 popfq -#ifdef CONFIG_SMP - /* - * Fix up static pointers that need __per_cpu_load added. The assembler - * is unable to do this directly. This is only needed for the boot cpu. - * These values are set up with the correct base addresses by C code for - * secondary cpus. - */ - movq initial_gs(%rip), %rax - cmpl $0, per_cpu__cpu_number(%rax) - jne 1f - addq %rax, early_gdt_descr_base(%rip) -1: -#endif /* * We must switch to a new descriptor in kernel space for the GDT * because soon the kernel won't have access anymore to the userspace
@@ -275,11 +262,7 @@ ENTRY(secondary_startup_64) ENTRY(initial_code) .quad x86_64_start_kernel ENTRY(initial_gs) -#ifdef CONFIG_SMP - .quad __per_cpu_load -#else - .quad PER_CPU_VAR(irq_stack_union) -#endif + .quad INIT_PER_CPU_VAR(irq_stack_union) __FINITDATA ENTRY(stack_start)
@@ -425,7 +408,7 @@ NEXT_PAGE(level2_spare_pgt) early_gdt_descr: .word GDT_ENTRIES*8-1 early_gdt_descr_base: - .quad per_cpu__gdt_page + .quad INIT_PER_CPU_VAR(gdt_page) ENTRY(phys_base) /* This must match the first entry in level2_kernel_pgt */
arch/x86/kernel/vmlinux_64.lds.S +8 -0
@@ -257,6 +257,14 @@ SECTIONS DWARF_DEBUG } + /* + * Per-cpu symbols which need to be offset from __per_cpu_load + * for the boot processor. + */ +#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load +INIT_PER_CPU(gdt_page); +INIT_PER_CPU(irq_stack_union); + /* * Build-time check on the image size: */
Signature · 1 line
1.6.1
Re: [PATCH 2/3] x86: Use linker to offset symbols by __per_cpu_load
Quoted 9 lines · Brian Gerst
Impact: cleanup and bug fix
Use the linker to create symbols for certain per-cpu variables that are offset by
__per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.Signed-off-by Brian Gerst <[email protected]>Cc Jiri Slaby <[email protected]>
Quoted 7 lines
--- arch/x86/include/asm/percpu.h | 22 ++++++++++++++++++++++ arch/x86/include/asm/processor.h | 2 ++ arch/x86/kernel/cpu/common.c | 6 +----- arch/x86/kernel/head_64.S | 21 ++------------------- arch/x86/kernel/vmlinux_64.lds.S | 8 ++++++++ 5 files changed, 35 insertions(+), 24 deletions(-)
applied to tip/core/percpu, thanks!
Also added these tags:
Ingo
Re: [PATCH 2/3] x86: Use linker to offset symbols by __per_cpu_load
Quoted 19 lines · Ingo Molnar
Quoted 9 lines · Brian Gerst
Impact: cleanup and bug fix
Use the linker to create symbols for certain per-cpu variables that are offset by
__per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.Signed-off-by Brian Gerst <[email protected]>Cc Jiri Slaby <[email protected]>Quoted 7 lines
--- arch/x86/include/asm/percpu.h | 22 ++++++++++++++++++++++ arch/x86/include/asm/processor.h | 2 ++ arch/x86/kernel/cpu/common.c | 6 +----- arch/x86/kernel/head_64.S | 21 ++------------------- arch/x86/kernel/vmlinux_64.lds.S | 8 ++++++++ 5 files changed, 35 insertions(+), 24 deletions(-)
Quoted 6 lines
applied to tip/core/percpu, thanks!
Also added these tags:
Reported-by Jiri Slaby <[email protected]>Acked-by Jiri Slaby <[email protected]>
Ok, fixes the problem for me, thanks.
PATCH 3/3 x86: Fix abuse of per_cpu_offset
In reply to Brian GerstImpact: bug fix
Don't use per_cpu_offset() to determine if it valid to access a per-cpu variable for a given cpu number. It is not a valid assumption on x86-64 anymore. Use cpu_possible() instead.
--- arch/x86/mm/numa_64.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/mm/numa_64.c +2 -2
@@ -702,7 +702,7 @@ void __cpuinit numa_set_node(int cpu, int node) }#ifdef CONFIG_DEBUG_PER_CPU_MAPS -if (cpu >= nr_cpu_ids || !per_cpu_offset (cpu)) { +if (cpu >= nr_cpu_ids || !cpu_possible (cpu)) {printk (KERN_ERR "numa_set_node: invalid cpu# (%d)\n" , cpu);dump_stack ();return ;
@@ -790,7 +790,7 @@ int early_cpu_to_node(int cpu)if (early_per_cpu_ptr (x86_cpu_to_node_map))return early_per_cpu_ptr (x86_cpu_to_node_map)[cpu]; -if (!per_cpu_offset (cpu)) { +if (!cpu_possible (cpu)) {printk (KERN_WARNING "early_cpu_to_node(%d): no per_cpu area!\n" , cpu);dump_stack ();
Signature · 1 line
1.6.1
Ingo Molnar
AppliedRe: [PATCH 3/3] x86: Fix abuse of per_cpu_offset
Quoted 10 lines · Brian Gerst
Impact: bug fix
Don't use
per_cpu_offset()to determine if it valid to access a per-cpu variable for a given cpu number. It is not a valid assumption on x86-64 anymore. Usecpu_possible()instead.Signed-off-by Brian Gerst <[email protected]>--- arch/x86/mm/numa_64.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
Applied to tip/core/perpcu, thanks Brian!
Ingo
Ingo Molnar
AppliedQuoted 11 lines · Brian Gerst
Impact: bug fix
IA-64 needs to put percpu data in the seperate section even on UP. Fixes regression caused by "percpu: refactor
percpu.h"Signed-off-by Brian Gerst <[email protected]>Acked-by Tony Luck <[email protected]>--- arch/ia64/include/asm/percpu.h | 4 ++-- include/linux/percpu.h | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-)
Applied to tip/core/percpu, thanks!
Ingo