Skip to contentSkip to conversation

percpu: Make PER_CPU_BASE_SECTION overridable by arches

PATCH Applied Brian Gerst 3 participants 7 messages Feb 8, 2009 – Feb 11, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 percpu: Make PER_CPU_BASE_SECTION overridable by arches

Impact: bug fix

IA-64 needs to put percpu data in the seperate section even on UP. Fixes regression caused by "percpu: refactor percpu.h"

---
 arch/ia64/include/asm/percpu.h |    4 ++--
 include/linux/percpu.h         |    8 +++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)
arch/ia64/include/asm/percpu.h +2 -2
index 77f30b6..30cf465 100644
@@ -27,12 +27,12 @@ extern void *per_cpu_init(void);
 
 #else /* ! SMP */
 
-#define PER_CPU_ATTRIBUTES	__attribute__((__section__(".data.percpu")))
-
 #define per_cpu_init()				(__phys_per_cpu_start)
 
 #endif	/* SMP */
 
+#define PER_CPU_BASE_SECTION ".data.percpu"
+
 /*
  * Be extremely careful when taking the address of this variable!  Due to virtual
  * remapping, it is different from the canonical address returned by __get_cpu_var(var)!
include/linux/percpu.h +7 -1
index 0e24202..3577ffd 100644
@@ -8,8 +8,15 @@
 
 #include <asm/percpu.h>
 
+#ifndef PER_CPU_BASE_SECTION
 #ifdef CONFIG_SMP
 #define PER_CPU_BASE_SECTION ".data.percpu"
+#else
+#define PER_CPU_BASE_SECTION ".data"
+#endif
+#endif
+
+#ifdef CONFIG_SMP
 
 #ifdef MODULE
 #define PER_CPU_SHARED_ALIGNED_SECTION ""
@@ -20,7 +27,6 @@
 
 #else
 
-#define PER_CPU_BASE_SECTION ".data"
 #define PER_CPU_SHARED_ALIGNED_SECTION ""
 #define PER_CPU_FIRST_SECTION ""
 
Signature · 1 line
1.6.1
#1 lore raw

PATCH 2/3 x86: Use linker to offset symbols by __per_cpu_load

Impact: cleanup and bug fix

Use the linker to create symbols for certain per-cpu variables that are offset by __per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.

---
 arch/x86/include/asm/percpu.h    |   22 ++++++++++++++++++++++
 arch/x86/include/asm/processor.h |    2 ++
 arch/x86/kernel/cpu/common.c     |    6 +-----
 arch/x86/kernel/head_64.S        |   21 ++-------------------
 arch/x86/kernel/vmlinux_64.lds.S |    8 ++++++++
 5 files changed, 35 insertions(+), 24 deletions(-)
arch/x86/include/asm/percpu.h +22 -0
index 0b64af4..aee103b 100644
@@ -34,6 +34,12 @@
 #define PER_CPU_VAR(var)	per_cpu__##var
 #endif	/* SMP */
 
+#ifdef CONFIG_X86_64_SMP
+#define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
+#else
+#define INIT_PER_CPU_VAR(var)  per_cpu__##var
+#endif
+
 #else /* ...!ASSEMBLY */
 
 #include <linux/stringify.h>
@@ -45,6 +51,22 @@
 #define __percpu_arg(x)		"%" #x
 #endif
 
+/*
+ * Initialized pointers to per-cpu variables needed for the boot
+ * processor need to use these macros to get the proper address
+ * offset from __per_cpu_load on SMP.
+ *
+ * There also must be an entry in vmlinux_64.lds.S
+ */
+#define DECLARE_INIT_PER_CPU(var) \
+       extern typeof(per_cpu_var(var)) init_per_cpu_var(var)
+
+#ifdef CONFIG_X86_64_SMP
+#define init_per_cpu_var(var)  init_per_cpu__##var
+#else
+#define init_per_cpu_var(var)  per_cpu_var(var)
+#endif
+
 /* For arch-specific code, we can use direct single-insn ops (they
  * don't give an lvalue though). */
 extern void __bad_percpu_size(void);
arch/x86/include/asm/processor.h +2 -0
index 656d02e..373d3f6 100644
@@ -393,6 +393,8 @@ union irq_stack_union {
 };
 
 DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);
+DECLARE_INIT_PER_CPU(irq_stack_union);
+
 DECLARE_PER_CPU(char *, irq_stack_ptr);
 #endif
 
arch/x86/kernel/cpu/common.c +1 -5
index 0f73ea4..41b0de6 100644
@@ -902,12 +902,8 @@ struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
 
 DEFINE_PER_CPU_FIRST(union irq_stack_union,
 		     irq_stack_union) __aligned(PAGE_SIZE);
-#ifdef CONFIG_SMP
-DEFINE_PER_CPU(char *, irq_stack_ptr);	/* will be set during per cpu init */
-#else
 DEFINE_PER_CPU(char *, irq_stack_ptr) =
-	per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
-#endif
+	init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
 
 DEFINE_PER_CPU(unsigned long, kernel_stack) =
 	(unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
arch/x86/kernel/head_64.S +2 -19
index a0a2b5c..2e648e3 100644
@@ -205,19 +205,6 @@ ENTRY(secondary_startup_64)
 	pushq $0
 	popfq
 
-#ifdef CONFIG_SMP
-	/*
-	 * Fix up static pointers that need __per_cpu_load added.  The assembler
-	 * is unable to do this directly.  This is only needed for the boot cpu.
-	 * These values are set up with the correct base addresses by C code for
-	 * secondary cpus.
-	 */
-	movq	initial_gs(%rip), %rax
-	cmpl	$0, per_cpu__cpu_number(%rax)
-	jne	1f
-	addq	%rax, early_gdt_descr_base(%rip)
-1:
-#endif
 	/*
 	 * We must switch to a new descriptor in kernel space for the GDT
 	 * because soon the kernel won't have access anymore to the userspace
@@ -275,11 +262,7 @@ ENTRY(secondary_startup_64)
 	ENTRY(initial_code)
 	.quad	x86_64_start_kernel
 	ENTRY(initial_gs)
-#ifdef CONFIG_SMP
-	.quad	__per_cpu_load
-#else
-	.quad	PER_CPU_VAR(irq_stack_union)
-#endif
+	.quad	INIT_PER_CPU_VAR(irq_stack_union)
 	__FINITDATA
 
 	ENTRY(stack_start)
@@ -425,7 +408,7 @@ NEXT_PAGE(level2_spare_pgt)
 early_gdt_descr:
 	.word	GDT_ENTRIES*8-1
 early_gdt_descr_base:
-	.quad	per_cpu__gdt_page
+	.quad	INIT_PER_CPU_VAR(gdt_page)
 
 ENTRY(phys_base)
 	/* This must match the first entry in level2_kernel_pgt */
arch/x86/kernel/vmlinux_64.lds.S +8 -0
index 07f62d2..087a7f2 100644
@@ -257,6 +257,14 @@ SECTIONS
   DWARF_DEBUG
 }
 
+ /*
+  * Per-cpu symbols which need to be offset from __per_cpu_load
+  * for the boot processor.
+  */
+#define INIT_PER_CPU(x) init_per_cpu__##x = per_cpu__##x + __per_cpu_load
+INIT_PER_CPU(gdt_page);
+INIT_PER_CPU(irq_stack_union);
+
 /*
  * Build-time check on the image size:
  */
Signature · 1 line
1.6.1
Acked-by Jiri Slaby on v1 2/3 Applied
#2 lore raw

Re: [PATCH 2/3] x86: Use linker to offset symbols by __per_cpu_load

Quoted 9 lines · Brian Gerst

Impact: cleanup and bug fix

Use the linker to create symbols for certain per-cpu variables that are offset by __per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.

Quoted 7 lines
---
 arch/x86/include/asm/percpu.h    |   22 ++++++++++++++++++++++
 arch/x86/include/asm/processor.h |    2 ++
 arch/x86/kernel/cpu/common.c     |    6 +-----
 arch/x86/kernel/head_64.S        |   21 ++-------------------
 arch/x86/kernel/vmlinux_64.lds.S |    8 ++++++++
 5 files changed, 35 insertions(+), 24 deletions(-)

applied to tip/core/percpu, thanks!

Also added these tags:

Ingo

#3 lore raw

Re: [PATCH 2/3] x86: Use linker to offset symbols by __per_cpu_load

Quoted 19 lines · Ingo Molnar
Quoted 9 lines · Brian Gerst

Impact: cleanup and bug fix

Use the linker to create symbols for certain per-cpu variables that are offset by __per_cpu_load. This allows the removal of the runtime fixup of the GDT pointer, which fixes a bug with resume reported by Jiri Slaby.

Quoted 7 lines
---
 arch/x86/include/asm/percpu.h    |   22 ++++++++++++++++++++++
 arch/x86/include/asm/processor.h |    2 ++
 arch/x86/kernel/cpu/common.c     |    6 +-----
 arch/x86/kernel/head_64.S        |   21 ++-------------------
 arch/x86/kernel/vmlinux_64.lds.S |    8 ++++++++
 5 files changed, 35 insertions(+), 24 deletions(-)
Quoted 6 lines

applied to tip/core/percpu, thanks!

Also added these tags:

Ok, fixes the problem for me, thanks.

#4 lore raw

PATCH 3/3 x86: Fix abuse of per_cpu_offset

In reply to Brian Gerst

Impact: bug fix

Don't use per_cpu_offset() to determine if it valid to access a per-cpu variable for a given cpu number. It is not a valid assumption on x86-64 anymore. Use cpu_possible() instead.

---
 arch/x86/mm/numa_64.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
arch/x86/mm/numa_64.c +2 -2
index 08d140f..deb1c1a 100644
@@ -702,7 +702,7 @@ void __cpuinit numa_set_node(int cpu, int node)
 	}
 
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
-	if (cpu >= nr_cpu_ids || !per_cpu_offset(cpu)) {
+	if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) {
 		printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu);
 		dump_stack();
 		return;
@@ -790,7 +790,7 @@ int early_cpu_to_node(int cpu)
 	if (early_per_cpu_ptr(x86_cpu_to_node_map))
 		return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu];
 
-	if (!per_cpu_offset(cpu)) {
+	if (!cpu_possible(cpu)) {
 		printk(KERN_WARNING
 			"early_cpu_to_node(%d): no per_cpu area!\n", cpu);
 		dump_stack();
Signature · 1 line
1.6.1
Applied
#5 lore raw

Re: [PATCH 3/3] x86: Fix abuse of per_cpu_offset

Quoted 10 lines · Brian Gerst

Impact: bug fix

Don't use per_cpu_offset() to determine if it valid to access a per-cpu variable for a given cpu number. It is not a valid assumption on x86-64 anymore. Use cpu_possible() instead.

---
 arch/x86/mm/numa_64.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

Applied to tip/core/perpcu, thanks Brian!

Ingo