Skip to contentSkip to conversation

PATCH [0/2] percpu: Local cpu pointer optimizations

PATCH In review Brian Gerst 2 participants 4 messages Sep 4, 2010 – Sep 5, 2010

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

This patch set allows arches to provide an optimized way of shifting a per-cpu pointer for the current cpu.

On an x86-32 defconfig build, these patches save 1304 bytes of text.

text data bss dec hex filename
7298335 717480 1374104 9389919 8f475f vmlinux.orig
7297283 717480 1374104 9388867 8f4343 vmlinux.1
7297031 717480 1374104 9388615 8f4247 vmlinux.2

[PATCH 1/2] x86, percpu: Optimize this_cpu_ptr
[PATCH 2/2] percpu: Optimize __get_cpu_var()

 arch/x86/include/asm/percpu.h |    9 +++++++++
 include/asm-generic/percpu.h  |   14 +++++++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)
#1 lore raw

PATCH 1/2 x86, percpu: Optimize this_cpu_ptr

Allow arches to implement __this_cpu_ptr, and provide an x86 version.

Before:
movq $foo, %rax
movq %gs:this_cpu_off, %rdx
addq %rdx, %rax

After:
movq $foo, %rax
addq %gs:this_cpu_off, %rax

The benefit is doing it in one less instruction and not clobbering a temporary register.

---
 arch/x86/include/asm/percpu.h |    9 +++++++++
 include/asm-generic/percpu.h  |    9 +++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
arch/x86/include/asm/percpu.h +9 -0
index cd28f9a..d854438 100644
@@ -47,6 +47,15 @@
 #ifdef CONFIG_SMP
 #define __percpu_arg(x)		"%%"__stringify(__percpu_seg)":%P" #x
 #define __my_cpu_offset		percpu_read(this_cpu_off)
+#define __this_cpu_ptr(ptr)				\
+({							\
+	typeof(ptr) __ptr = (ptr);			\
+	__verify_pcpu_ptr(ptr);				\
+	asm volatile("add " __percpu_arg(1) ", %0"	\
+		     : "+r" (__ptr)			\
+		     : "m" (this_cpu_off));		\
+	__ptr;						\
+}) 
 #else
 #define __percpu_arg(x)		"%P" #x
 #endif
include/asm-generic/percpu.h +7 -2
index b5043a9..5820fcb 100644
@@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 #define __raw_get_cpu_var(var) \
 	(*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
 
-#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
+#ifndef __this_cpu_ptr
 #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
-
+#endif
+#ifdef CONFIG_DEBUG_PREEMPT
+#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
+#else
+#define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)
+#endif
 
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
Signature · 1 line
1.7.2.2
maintainer
#2 lore raw

Re: [PATCH 1/2] x86, percpu: Optimize this_cpu_ptr

Hello,

Quoted 7 lines · Brian Gerst
arch/x86/include/asm/percpu.h +0 -0
index cd28f9a..d854438 100644
@@ -47,6 +47,15 @@
 #ifdef CONFIG_SMP
 #define __percpu_arg(x)		"%%"__stringify(__percpu_seg)":%P" #x

arch/x86/include/asm/percpu.h line 58

Quoted 10 lines
arch/x86/include/asm/percpu.h +9 -0
 #define __my_cpu_offset		percpu_read(this_cpu_off)
+#define __this_cpu_ptr(ptr)				\
+({							\
+	typeof(ptr) __ptr = (ptr);			\
+	__verify_pcpu_ptr(ptr);				\
+	asm volatile("add " __percpu_arg(1) ", %0"	\
+		     : "+r" (__ptr)			\
+		     : "m" (this_cpu_off));		\
+	__ptr;						\
+}) 

Great, thanks for doing this, but can you please comment on what __this_cpu_ptr() optimizes above? Something similar to the patch description but shorter.

Thanks.

Signature · 1 line
tejun
#3 lore raw

PATCH 2/2 percpu: Optimize __get_cpu_var()

In reply to Brian Gerst

Redefine __get_cpu_var() using this_cpu_ptr() which can be arch-optimized.

---
 include/asm-generic/percpu.h |    7 +++----
 1 files changed, 3 insertions(+), 4 deletions(-)
include/asm-generic/percpu.h +3 -4
index 5820fcb..c6e2c2d 100644
@@ -55,10 +55,6 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
  */
 #define per_cpu(var, cpu) \
 	(*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
-#define __get_cpu_var(var) \
-	(*SHIFT_PERCPU_PTR(&(var), my_cpu_offset))
-#define __raw_get_cpu_var(var) \
-	(*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))
 
 #ifndef __this_cpu_ptr
 #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
@@ -69,6 +65,9 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
 #define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)
 #endif
 
+#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
+#define __raw_get_cpu_var(var) (*__this_cpu_ptr(&(var)))
+
 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
 extern void setup_per_cpu_areas(void);
 #endif
Signature · 1 line
1.7.2.2