Skip to contentSkip to conversation

core/percpu for v2.6.27

GIT PULL Ingo Molnar 5 participants 9 messages Jul 14, 2008 – Jul 25, 2008

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

Linus,

Please pull the latest core/percpu git tree from:

git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git core/percpu

Thanks,

Ingo

------------------>
Eric Dumazet (1):
percpu: introduce DEFINE_PER_CPU_PAGE_ALIGNED() macro

Heiko Carstens (1):
percpu: zero based percpu build error on s390

Mike Travis (2):
x86: extend percpu ops to 64 bit
Zero based percpu: infrastructure to rebase the per cpu area to zero

 arch/x86/kernel/cpu/common.c      |    2 +-
 arch/x86/kernel/vmlinux_32.lds.S  |    1 +
 include/asm-generic/percpu.h      |   11 ++++-
 include/asm-generic/sections.h    |   10 ++++
 include/asm-generic/vmlinux.lds.h |   17 ++++++++
 include/asm-x86/percpu.h          |   83 ++++++++++++++++++++++++++++++++++--
 include/linux/percpu.h            |   24 ++++++++++-
 kernel/lockdep.c                  |    4 +-
 kernel/module.c                   |    7 ++-
 9 files changed, 146 insertions(+), 13 deletions(-)
arch/x86/kernel/cpu/common.c +1 -1
index d0463a9..b2f54fa 100644
@@ -21,7 +21,7 @@
 
 #include "cpu.h"
 
-DEFINE_PER_CPU(struct gdt_page, gdt_page) = { .gdt = {
+DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
 	[GDT_ENTRY_KERNEL_CS] = { { { 0x0000ffff, 0x00cf9a00 } } },
 	[GDT_ENTRY_KERNEL_DS] = { { { 0x0000ffff, 0x00cf9200 } } },
 	[GDT_ENTRY_DEFAULT_USER_CS] = { { { 0x0000ffff, 0x00cffa00 } } },
arch/x86/kernel/vmlinux_32.lds.S +1 -0
index ce5ed08..0f7c29a 100644
@@ -189,6 +189,7 @@ SECTIONS
   . = ALIGN(PAGE_SIZE);
   .data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {
 	__per_cpu_start = .;
+	*(.data.percpu.page_aligned)
 	*(.data.percpu)
 	*(.data.percpu.shared_aligned)
 	__per_cpu_end = .;
include/asm-generic/percpu.h +10 -1
index b0e63c6..1c02250 100644
@@ -45,7 +45,12 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
  * Only S390 provides its own means of moving the pointer.
  */
 #ifndef SHIFT_PERCPU_PTR
-#define SHIFT_PERCPU_PTR(__p, __offset)	RELOC_HIDE((__p), (__offset))
+# ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU
+#  define SHIFT_PERCPU_PTR(__p, __offset) \
+	((__typeof(__p))(((void *)(__p)) + (__offset)))
+# else
+#  define SHIFT_PERCPU_PTR(__p, __offset)	RELOC_HIDE((__p), (__offset))
+# endif /* CONFIG_HAVE_ZERO_BASED_PER_CPU */
 #endif
 
 /*
@@ -70,6 +75,10 @@ extern void setup_per_cpu_areas(void);
 #define per_cpu(var, cpu)			(*((void)(cpu), &per_cpu_var(var)))
 #define __get_cpu_var(var)			per_cpu_var(var)
 #define __raw_get_cpu_var(var)			per_cpu_var(var)
+#ifndef SHIFT_PERCPU_PTR
+# define SHIFT_PERCPU_PTR(__p, __offset)		(__p)
+#endif
+#define per_cpu_offset(x)			0L
 
 #endif	/* SMP */
 
include/asm-generic/sections.h +10 -0
index 8feeae1..f697620 100644
@@ -9,7 +9,17 @@ extern char __bss_start[], __bss_stop[];
 extern char __init_begin[], __init_end[];
 extern char _sinittext[], _einittext[];
 extern char _end[];
+#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU
+extern char __per_cpu_load[];
+extern char ____per_cpu_size[];
+#define __per_cpu_size ((unsigned long)&____per_cpu_size)
+#define __per_cpu_start ((char *)0)
+#define __per_cpu_end ((char *)__per_cpu_size)
+#else
 extern char __per_cpu_start[], __per_cpu_end[];
+#define __per_cpu_load __per_cpu_start
+#define __per_cpu_size (__per_cpu_end - __per_cpu_start)
+#endif
 extern char __kprobes_text_start[], __kprobes_text_end[];
 extern char __initdata_begin[], __initdata_end[];
 extern char __start_rodata[], __end_rodata[];
include/asm-generic/vmlinux.lds.h +17 -0
index f054778..465a97a 100644
@@ -344,11 +344,28 @@
   	*(.initcall7.init)						\
   	*(.initcall7s.init)
 
+#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU
+#define PERCPU(align)							\
+	. = ALIGN(align);						\
+	percpu : { } :percpu						\
+	__per_cpu_load = .;						\
+	.data.percpu 0 : AT(__per_cpu_load - LOAD_OFFSET) {		\
+		*(.data.percpu.first)					\
+		*(.data.percpu.shared_aligned)				\
+		*(.data.percpu)						\
+		*(.data.percpu.page_aligned)				\
+		____per_cpu_size = .;					\
+	}								\
+	. = __per_cpu_load + ____per_cpu_size;				\
+	data : { } :data
+#else
 #define PERCPU(align)							\
 	. = ALIGN(align);						\
 	__per_cpu_start = .;						\
 	.data.percpu  : AT(ADDR(.data.percpu) - LOAD_OFFSET) {		\
+		*(.data.percpu.page_aligned)				\
 		*(.data.percpu)						\
 		*(.data.percpu.shared_aligned)				\
 	}								\
 	__per_cpu_end = .;
+#endif
include/asm-x86/percpu.h +78 -5

100 lines not on this page · raw message

include/linux/percpu.h +23 -1
index 4cdd393..3d1a7f6 100644
@@ -23,13 +23,35 @@
 	__attribute__((__section__(SHARED_ALIGNED_SECTION)))		\
 	PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name		\
 	____cacheline_aligned_in_smp
+
+#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)			\
+	__attribute__((__section__(".data.percpu.page_aligned")))	\
+	PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
+
+#ifdef CONFIG_HAVE_ZERO_BASED_PER_CPU
+#define DEFINE_PER_CPU_FIRST(type, name)				\
+	__attribute__((__section__(".data.percpu.first")))		\
+	PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
 #else
+#define DEFINE_PER_CPU_FIRST(type, name)				\
+	DEFINE_PER_CPU(type, name)
+#endif
+
+#else /* !CONFIG_SMP */
+
 #define DEFINE_PER_CPU(type, name)					\
 	PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
 
 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)		      \
 	DEFINE_PER_CPU(type, name)
-#endif
+
+#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)		      \
+	DEFINE_PER_CPU(type, name)
+
+#define DEFINE_PER_CPU_FIRST(type, name)				\
+	DEFINE_PER_CPU(type, name)
+
+#endif /* !CONFIG_SMP */
 
 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
kernel/lockdep.c +2 -2
index 81a4e4a..965b9c7 100644
@@ -609,8 +609,8 @@ static int static_obj(void *obj)
 	 * percpu var?
 	 */
 	for_each_possible_cpu(i) {
-		start = (unsigned long) &__per_cpu_start + per_cpu_offset(i);
-		end   = (unsigned long) &__per_cpu_start + PERCPU_ENOUGH_ROOM
+		start = (unsigned long) __per_cpu_start + per_cpu_offset(i);
+		end   = (unsigned long) __per_cpu_start + PERCPU_ENOUGH_ROOM
 					+ per_cpu_offset(i);
 
 		if ((addr >= start) && (addr < end))
kernel/module.c +4 -3
index 5f80478..f525015 100644
@@ -44,6 +44,7 @@
 #include <linux/unwind.h>
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
+#include <asm/sections.h>
 #include <linux/license.h>
 #include <asm/sections.h>
 
@@ -365,7 +366,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,
 		align = PAGE_SIZE;
 	}
 
-	ptr = __per_cpu_start;
+	ptr = __per_cpu_load;
 	for (i = 0; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
 		/* Extra for alignment requirement. */
 		extra = ALIGN((unsigned long)ptr, align) - (unsigned long)ptr;
@@ -400,7 +401,7 @@ static void *percpu_modalloc(unsigned long size, unsigned long align,
 static void percpu_modfree(void *freeme)
 {
 	unsigned int i;
-	void *ptr = __per_cpu_start + block_size(pcpu_size[0]);
+	void *ptr = __per_cpu_load + block_size(pcpu_size[0]);
 
 	/* First entry is core kernel percpu data. */
 	for (i = 1; i < pcpu_num_used; ptr += block_size(pcpu_size[i]), i++) {
@@ -451,7 +452,7 @@ static int percpu_modinit(void)
 	pcpu_size = kmalloc(sizeof(pcpu_size[0]) * pcpu_num_allocated,
 			    GFP_KERNEL);
 	/* Static in-kernel percpu data (used). */
-	pcpu_size[0] = -(__per_cpu_end-__per_cpu_start);
+	pcpu_size[0] = -__per_cpu_size;
 	/* Free room. */
 	pcpu_size[1] = PERCPU_ENOUGH_ROOM + pcpu_size[0];
 	if (pcpu_size[1] < 0) {
#1 lore raw
Quoted 4 lines · Ingo Molnar

Please pull the latest core/percpu git tree from:

git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git core/percpu

Can you explain what this does and who needs it? The percpu_xchg() looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.

Linus

#2 lore raw
Quoted 1 line · Linus Torvalds

Can you explain what this does and who needs it? The percpu_xchg() looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.

The essence of those changes is to work towards unifying the i386 and x86-64 percpu mechanisms. But I wasn't terribly convinced by some of the frills around the edges, like the xchg operation. The work is being driven by Mike Travis and the huge numa people, so perhaps they have a use for it.

J

#3 lore raw
Quoted 11 lines · Jeremy Fitzhardinge
Quoted 4 lines · Linus Torvalds

Can you explain what this does and who needs it? The percpu_xchg() looks particularly pointless, since it's always a locked SMP-safe instruction on x86, so a nonpreemptible load+store will likely be much faster.

The essence of those changes is to work towards unifying the i386 and x86-64 percpu mechanisms. But I wasn't terribly convinced by some of the frills around the edges, like the xchg operation. The work is being driven by Mike Travis and the huge numa people, so perhaps they have a use for it.

yeah. It all centers around the '4096 CPUs' topic which has so wide impact that it had to be scattered out a bit.

x86_percpu_xchg():

seems like a sensible extension of the API family to me. Linus is right about the LOCK overhead but i'm not all that sure that LOCKed instructions will always be slower. Combined with single-pointer per-cpu (not the PDA-based redirection we do now) it can be done safely without disabling preemption - and that would make the basic percpu ops usable in preemptible code.

But indeed there are no very strong reasons.

These are the currently pending related topics:

 # tip/cpus4096 - ready for upstream, i'll send the pull request once
 #                core/rcu is upstream

earth4:~/tip> git-log-line linus..cpus4096
9982fbf: Revert "cpumask: introduce new APIs"
68083e0: Merge commit 'v2.6.26-rc9' into cpus4096
7baac8b: cpumask: make for_each_cpu_mask a bit smaller 3f9b48a: net: Pass reference to cpumask variable in net/sunrpc/svc.c
cad0e45: clocksource/events: use performance variant for_each_cpu_mask_nr 5d7bfd0: infiniband: use performance variant for_each_cpu_mask_nr 334ef7a: x86: use performance variant for_each_cpu_mask_nr 0e12f84: net: use performance variant for_each_cpu_mask_nr 6d6a436: mm: use performance variant for_each_cpu_mask_nr 363ab6f: core: use performance variant for_each_cpu_mask_nr 068b127: cpufreq: use performance variant for_each_cpu_mask_nr 141ad06: acpi: use performance variant for_each_cpu_mask_nr 41df0d61: x86: Add performance variants of cpumask operators 143aa5c: cpu: change some globals to statics in drivers/base/cpu.c v2
a953e45: sched: replace MAX_NUMNODES with nr_node_ids in kernel/sched.c

# tip/core/percpu-zerobased: stalled - current approach is
# fragile to binutils bugs

earth4:~/tip> git-log-line linus..core/percpu-zerobased
a6d5d88: x86, 64-bit: replace xxx_pda() operations with x86_xxx_percpu(). 6b17441: x86, 64-bit: replace cpu_pda ops with percpu ops 6aa952c: x86, 64-bit: reference zero-based percpu variables offset from gs 8c76b15: x86, 64-bit: fold pda into per cpu area
673698e: x86, 64-bit: fix early references to cpumask_of_cpu

in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.

Ingo

#4 lore raw

Ingo Molnar wrote:

...

Quoted 5 lines

in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.

Ingo

Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)

Btw, is there a list of "bad" gcc's for kernel building? Or better yet, can the Makefile script provide a warning when a known "bad" gcc is being used to compile the kernel? I seem to recall that Peter provided this list:

4.2.3 is fine; he was using 4.2.0 before, and as far as I know, 4.2.0 and 4.2.1 are known broken for the kernel.

Thanks!
Mike

#5 lore raw
Quoted 4 lines · Mike Travis

4.2.3 is fine; he was using 4.2.0 before, and as far as I know, 4.2.0 and 4.2.1 are known broken for the kernel.

I might have been misremembering about 4.2.[01]; Adrian Bunk reminds me that 4.*1*.[01] are definitely broken, and I can't simply remember the purported problem with 4.2, so it *may* be a case of off-by-one mental error.

-hpa

#6 lore raw
In reply to Mike Travis
Quoted 4 lines · Mike Travis

Ingo Molnar wrote:

...

Quoted 1 line
Quoted 8 lines
Quoted 4 lines

in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.

Ingo

Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)

well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?

Ingo

#7 lore raw
Quoted 9 lines · Ingo Molnar
Quoted 7 lines · Mike Travis

Ingo Molnar wrote:

...

Quoted 4 lines

in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.

Ingo

Quoted 8 lines
Quoted 3 lines

Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)

well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?

Ingo

I will test with gcc-4.2.3 as soon as I'm able, and will let you know. (There have been changes from when you hit the panic.)

I will resubmit against tip/master.

Thanks,
Mike

#8 lore raw
In reply to Ingo Molnar
Quoted 9 lines · Ingo Molnar
Quoted 7 lines · Mike Travis

Ingo Molnar wrote:

...

Quoted 4 lines

in hindsight core/percpu indeed looks unfinished and direction-less without core/percpu-zerobased - but the latter is not stable yet.

Ingo

Quoted 8 lines
Quoted 3 lines

Well it's very stable using gcc-4.2.4. The earlier problems came about using gcc-4.2.0 and has yet to be determined what exactly went wrong. (And I need to install gcc-3.2 to complete the build/test QA.)

well, the crash report i gave you was with gcc-4.2.3. Or is that one resolved?

Ingo

YES! Patcheset to follow... ;-)

Thanks,
Mike