x86: enlightenment for ticket spinlocks - remove NOPs from unlock path
Subsystems (2)
PATCH 3/3 x86: enlightenment for ticket spinlocks - remove NOPs from unlock path
Under the assumption that the nop-s added by the base ticket spinlock enlightenment patch might be considered undesirable (or worse), here is an optional patch to eliminate these nop-s again. This is done through extending the memory operands of the inc instructions used for unlocking ticket locks to the necessary size, using assembler and linker features.
--- arch/x86/Makefile | 3 + arch/x86/include/asm/alternative-asm.h | 59 +++++++++++++++++++++++++++++++++ arch/x86/include/asm/alternative.h | 5 ++ arch/x86/include/asm/spinlock.h | 27 ++++----------- arch/x86/kernel/symdefs.lds | 1 arch/x86/kernel/vmlinux.lds.S | 2 + 6 files changed, 78 insertions(+), 19 deletions(-)
arch/x86/Makefile +3 -0
@@ -87,6 +87,9 @@ ifeq ($(CONFIG_KMEMCHECK),y) KBUILD_CFLAGS += $(call cc-option,-fno-builtin-memcpy) endif +KBUILD_CFLAGS += -Wa,-I$(srctree)/arch/x86/include +LDFLAGS_MODULE += -T $(srctree)/arch/x86/kernel/symdefs.lds + # Stackpointer is addressed different for 32 bit and 64 bit x86 sp-$(CONFIG_X86_32) := esp sp-$(CONFIG_X86_64) := rsp
arch/x86/include/asm/alternative.h +5 -0
@@ -6,6 +6,11 @@#include <linux/stringify.h> #include <asm/asm.h> +#if !defined(__ASSEMBLY__) && !defined(__PIC__) +#include <asm/alternative-asm.h> /* just for tracking the build dependency */ +__asm__(".include \"asm/alternative-asm.h\"" ); +#endif +/* * Alternative inline assembly for SMP. *
arch/x86/include/asm/alternative-asm.h +59 -0
65 lines not on this page · raw message
arch/x86/include/asm/spinlock.h +8 -19
@@ -10,7 +10,6 @@#ifdef CONFIG_ENLIGHTEN_SPINLOCKS #include <asm/alternative.h> -#include <asm/nops.h> /* Including asm/smp.h here causes a cyclic include dependency. */ #include <asm/percpu.h> DECLARE_PER_CPU (int, cpu_number);
@@ -155,20 +154,15 @@ static __always_inline int __ticket_spin static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) {#ifndef CONFIG_ENLIGHTEN_SPINLOCKS - asm volatile( + asm volatile(UNLOCK_LOCK_PREFIX "incb %0" + : "+m" (lock->slock) + :#else unsigned int token;alternative_io (".L%=orig:\n\t" -#endif -UNLOCK_LOCK_PREFIX "incb %0" -#ifndef CONFIG_ENLIGHTEN_SPINLOCKS - : "+m" (lock->slock) - : -#else - "\n\t" -ASM_NOP3 +UNLOCK_LOCK_PREFIX "unary incb %0\n\t" ".L%=done:" ,".L%=alt:\n\t" /* jmp .L%=callout */
@@ -286,20 +280,15 @@ static __always_inline int __ticket_spin static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) {#ifndef CONFIG_ENLIGHTEN_SPINLOCKS - asm volatile( + asm volatile(UNLOCK_LOCK_PREFIX "incw %0" + : "+m" (lock->slock) + :#else unsigned int token, tmp;alternative_io (".L%=orig:\n\t" -#endif -UNLOCK_LOCK_PREFIX "incw %0" -#ifndef CONFIG_ENLIGHTEN_SPINLOCKS - : "+m" (lock->slock) - : -#else - "\n\t" -ASM_NOP2 +UNLOCK_LOCK_PREFIX "unary incw %0\n\t" ".L%=done:" ,".L%=alt:\n\t" /* jmp .L%=callout */
arch/x86/kernel/symdefs.lds new file +1 -0
@@ -0,0 +1 @@ +_$.zero = 0;
arch/x86/kernel/vmlinux.lds.S +2 -0
@@ -27,6 +27,8 @@ #include <asm/cache.h> #include <asm/boot.h> +#include "symdefs.lds" + #undef i386 /* in case the preprocessor is a 32bit one */ OUTPUT_FORMAT(CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT, CONFIG_OUTPUT_FORMAT)
H. Peter Anvin
reviewerQuoted 66 lines · Jan Beulich
Under the assumption that the nop-s added by the base ticket spinlock enlightenment patch might be considered undesirable (or worse), here is an optional patch to eliminate these nop-s again. This is done through extending the memory operands of the inc instructions used for unlocking ticket locks to the necessary size, using assembler and linker features.
arch/x86/include/asm/alternative-asm.h +49
-0@@ -1,3 +1,7 @@ +#if 0 /* Hide this from compiler. */ + .if 0 # Hide assembly source stuff when assembling compiler output. +#endif +#ifdef __ASSEMBLY__#include <asm/asm.h> @@ -16,3 +20,58 @@#endif #endif /* __ASSEMBLY__ */ + +#if 0 /* Hide this from compiler. */ + .else # Code to be used in compiler output: + + .weak _$.zero + + .macro unary opc arg1 arg2 arg3 + .Lempty=2 + .irpc c,"\arg2" + .Lempty=3 + .endr + .irpc c,"\arg3" + .Lempty=0 + .endr + .Lsym=1 + .Lnum=0 + .irpc c,"\arg1" + .irpc m,"(123456789-0" + .ifeqs "\c","\m" + .Lsym=0 + .exitm + .endif + .Lnum=1 + .endr + .exitm + .endr + .if .Lempty ==2 + .if .Lsym + \opc \arg1 + .elseif .Lnum + \opc _$.zero+\arg1 + .else + \opc _$.zero\arg1 + .endif + .elseif .Lempty ==3 + .if .Lsym + \opc \arg1,\arg2 + .elseif .Lnum + \opc _$.zero+\arg1,\arg2 + .else + \opc _$.zero\arg1,\arg2 + .endif + .else + .if .Lsym + \opc \arg1,\arg2,\arg3
arch/x86/include/asm/alternative-asm.h line 77
Quoted 10 lines
arch/x86/include/asm/alternative-asm.h +10
-0+ .elseif .Lnum + \opc _$.zero+\arg1,\arg2,\arg3 + .else + \opc _$.zero\arg1,\arg2,\arg3 + .endif + .endif + .endm + + .endif +#endif
Okay, I have absolutely no idea what this macro either *does* or what it's *supposed to do* nor if it matches... you kind of forgot to describe that. The other bit is that this whole handling with .if and #if is just too ugly to live. Create two include files at the very minimum.
I'd like to figure out if there isn't a better idea to do what you're trying to do here, though...
-hpa
Quoted 23 lines
Quoted 22 lines
Quoted 1 line
"H. Peter Anvin" <[email protected]> 01.02.10 23:54 >>>
arch/x86/include/asm/alternative-asm.h +11
-0@@ -1,3 +1,7 @@ +#if 0 /* Hide this from compiler. */ + .if 0 # Hide assembly source stuff when assembling compiler output. +#endif +#ifdef __ASSEMBLY__#include <asm/asm.h> @@ -16,3 +20,58 @@#endif #endif /* __ASSEMBLY__ */ + +#if 0 /* Hide this from compiler. */ + .else # Code to be used in compiler output: + + .weak _$.zero + + .macro unary opc arg1 arg2 arg3...
Quoted 8 lines
arch/x86/include/asm/alternative-asm.h line 77
Quoted 4 lines
arch/x86/include/asm/alternative-asm.h +4
-0+ .endm + + .endif +#endif Okay, I have absolutely no idea what this macro either *does* or what it's *supposed to do* nor if it matches... you kind of forgot to describe that.
It does what the patch description says - extend the (memory) operand of a unary opcode (inc in the case where it's being used) to maximum width (i.e. to include a 32-bit displacement regardless of whether one really is needed). It just needs to consider the different cases of what operands gcc may output. And it gets more complicated because commas in the operand are being treated as macro operand separators.
Quoted 2 lines
The other bit is that this whole handling with .if and
#if is just too ugly to live. Create two include files at the very minimum.
The new one ought to use the same construct then anyway, as inclusion of the file visible to gcc is desirable for dependency tracking, while wrapping the whole construct in an __asm__() doesn't seem desirable to me due to it making the whole thing even less readable.
Jan