PATCH [0/2] percpu: Local cpu pointer optimizations
Subsystems (3)
This patch set allows arches to provide an optimized way of shifting a per-cpu pointer for the current cpu.
On an x86-32 defconfig build, these patches save 1304 bytes of text.
text data bss dec hex filename
7298335 717480 1374104 9389919 8f475f vmlinux.orig 7297283 717480 1374104 9388867 8f4343 vmlinux.1 7297031 717480 1374104 9388615 8f4247 vmlinux.2
[PATCH 1/2] x86, percpu: Optimize this_cpu_ptr
[PATCH 2/2] percpu: Optimize __get_cpu_var()
arch/x86/include/asm/percpu.h | 11 +++++++++++ include/asm-generic/percpu.h | 14 +++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-)
v2: Added comments
PATCH 1/2 x86, percpu: Optimize this_cpu_ptr
Allow arches to implement __this_cpu_ptr, and provide an x86 version.
Before:
movq $foo, %rax
movq %gs:this_cpu_off, %rdx
addq %rdx, %rax
After:
movq $foo, %rax
addq %gs:this_cpu_off, %rax
The benefit is doing it in one less instruction and not clobbering a temporary register.
--- arch/x86/include/asm/percpu.h | 11 +++++++++++ include/asm-generic/percpu.h | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-)
arch/x86/include/asm/percpu.h +11 -0
@@ -47,6 +47,17 @@#ifdef CONFIG_SMP #define __percpu_arg (x) "%%"__stringify(__percpu_seg)":%P" #x#define __my_cpu_offset percpu_read(this_cpu_off) + +/* shift a per-cpu pointer to the current cpu's version */ +#define __this_cpu_ptr (ptr) \ +({ \ + typeof(ptr) __ptr = (ptr); \ + __verify_pcpu_ptr(ptr); \ + asm volatile("add " __percpu_arg(1) ", %0" \ + : "+r" (__ptr) \ + : "m" (this_cpu_off)); \ + __ptr; \ +})#else #define __percpu_arg (x) "%P" #x#endif
include/asm-generic/percpu.h +7 -2
@@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR_CPUS];#define __raw_get_cpu_var (var) \ (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset)) -#define this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#ifndef __this_cpu_ptr#define __this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) - +#endif +#ifdef CONFIG_DEBUG_PREEMPT +#define this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#else +#define this_cpu_ptr (ptr) __this_cpu_ptr(ptr) +#endif #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas (void );
Signature · 1 line
1.7.2.2
Tejun Heo
maintainerPATCH 1/2 x86, percpu: Optimize this_cpu_ptr
Allow arches to implement __this_cpu_ptr, and provide an x86 version.
Before:
movq $foo, %rax
movq %gs:this_cpu_off, %rdx
addq %rdx, %rax
After:
movq $foo, %rax
addq %gs:this_cpu_off, %rax
The benefit is doing it in one less instruction and not clobbering a temporary register.
tj: beefed up the comment a bit and renamed in-macro temp variable to
match neighboring macros.
---
Applied both patches to the following branch. I updated the first patch slightly.
git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git for-next
Thank you.
arch/x86/include/asm/percpu.h | 14 ++++++++++++++ include/asm-generic/percpu.h | 9 +++++++-- 2 files changed, 21 insertions(+), 2 deletions(-)
arch/x86/include/asm/percpu.h +14 -0
@@ -47,6 +47,20 @@#ifdef CONFIG_SMP #define __percpu_arg (x) "%%"__stringify(__percpu_seg)":%P" #x#define __my_cpu_offset percpu_read(this_cpu_off) + +/* +* Compared to the generic __my_cpu_offset version, the following +* saves one instruction and avoids clobbering a temp register. +*/ +#define __this_cpu_ptr (ptr) \ +({ \ + typeof(ptr) tcp_ptr__ = (ptr); \ + __verify_pcpu_ptr(ptr); \ + asm volatile("add " __percpu_arg(1) ", %0" \ + : "+r" (tcp_ptr__) \ + : "m" (this_cpu_off)); \ + tcp_ptr__; \ +})#else #define __percpu_arg (x) "%P" #x#endif
include/asm-generic/percpu.h +7 -2
@@ -60,9 +60,14 @@ extern unsigned long __per_cpu_offset[NR#define __raw_get_cpu_var (var) \ (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset)) -#define this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#ifndef __this_cpu_ptr#define __this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) - +#endif +#ifdef CONFIG_DEBUG_PREEMPT +#define this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) +#else +#define this_cpu_ptr (ptr) __this_cpu_ptr(ptr) +#endif #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas (void );
PATCH 2/2 percpu: Optimize __get_cpu_var()
In reply to Brian GerstRedefine __get_cpu_var() using this_cpu_ptr() which can be arch-optimized.
--- include/asm-generic/percpu.h | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-)
include/asm-generic/percpu.h +3 -4
@@ -55,10 +55,6 @@ extern unsigned long __per_cpu_offset[NR_CPUS];*/ #define per_cpu (var, cpu) \ (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu))) -#define __get_cpu_var (var) \ - (*SHIFT_PERCPU_PTR(&(var), my_cpu_offset)) -#define __raw_get_cpu_var (var) \ - (*SHIFT_PERCPU_PTR(&(var), __my_cpu_offset))#ifndef __this_cpu_ptr#define __this_cpu_ptr (ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
@@ -69,6 +65,9 @@ extern unsigned long __per_cpu_offset[NR_CPUS];#define this_cpu_ptr (ptr) __this_cpu_ptr(ptr)#endif +#define __get_cpu_var (var) (*this_cpu_ptr(&(var))) +#define __raw_get_cpu_var (var) (*__this_cpu_ptr(&(var))) +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas (void );#endif
Signature · 1 line
1.7.2.2