x86/asm changes for v3.1
Subsystems (3)
Linus,
Please pull the latest x86-asm-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git x86-asm-for-linus
out-of-topic modifications in x86-asm-for-linus:
------------------------------------------------arch/um/sys-i386/Makefile # a738669: x86: Unify rwsem assembly implemearch/um/sys-x86_64/Makefile # a738669: x86: Unify rwsem assembly impleme
Thanks,
Ingo
------------------>
Borislav Petkov (5):
x86, asm: Thin down SAVE/RESTORE_* asm macros x86, asm: Flip SAVE_ARGS arguments logic x86, asm: Flip RESTORE_ARGS arguments logic x86, asm: Cleanup thunk_64.S
x86, asm: Fix binutils 2.16 issue with __USER32_CS
Jan Beulich (3):
x86: Unify rwlock assembly implementation
x86: Unify rwsem assembly implementation
x86: Fix write lock scalability 64-bit issue
arch/um/sys-i386/Makefile | 2 +-
arch/um/sys-x86_64/Makefile | 2 +-
arch/x86/ia32/ia32entry.S | 10 +-
arch/x86/include/asm/asm.h | 5 +-
arch/x86/include/asm/calling.h | 130 +++++++++++++--------------------
arch/x86/include/asm/frame.h | 11 ++-
arch/x86/include/asm/rwlock.h | 43 +++++++++++-
arch/x86/include/asm/segment.h | 2 +-
arch/x86/include/asm/spinlock.h | 37 ++++++----
arch/x86/include/asm/spinlock_types.h | 6 +-
arch/x86/kernel/entry_64.S | 6 +-
arch/x86/lib/Makefile | 7 +-
arch/x86/lib/rwlock.S | 44 +++++++++++
arch/x86/lib/rwlock_64.S | 38 ----------
arch/x86/lib/{rwsem_64.S => rwsem.S} | 75 +++++++++++++++----
arch/x86/lib/semaphore_32.S | 124 -------------------------------
arch/x86/lib/thunk_64.S | 45 +++++-------
17 files changed, 263 insertions(+), 324 deletions(-)
create mode 100644 arch/x86/lib/rwlock.S
delete mode 100644 arch/x86/lib/rwlock_64.S
rename arch/x86/lib/{rwsem_64.S => rwsem.S} (55%)
delete mode 100644 arch/x86/lib/semaphore_32.Sarch/um/sys-i386/Makefile +1 -1
@@ -8,7 +8,7 @@ obj-y = bug.o bugs.o checksum.o delay.o fault.o ksyms.o ldt.o ptrace.o \ obj-$(CONFIG_BINFMT_ELF) += elfcore.o -subarch-obj-y = lib/semaphore_32.o lib/string_32.o +subarch-obj-y = lib/rwsem.o lib/string_32.o subarch-obj-$(CONFIG_HIGHMEM) += mm/highmem_32.o subarch-obj-$(CONFIG_MODULES) += kernel/module.o
arch/um/sys-x86_64/Makefile +1 -1
@@ -9,7 +9,7 @@ obj-y = bug.o bugs.o delay.o fault.o ldt.o mem.o ptrace.o ptrace_user.o \ sysrq.o ksyms.o tls.o subarch-obj-y = lib/csum-partial_64.o lib/memcpy_64.o lib/thunk_64.o \ - lib/rwsem_64.o + lib/rwsem.o subarch-obj-$(CONFIG_MODULES) += kernel/module.o ldt-y = ../sys-i386/ldt.o
arch/x86/ia32/ia32entry.S +5 -5
@@ -143,7 +143,7 @@ ENTRY(ia32_sysenter_target) CFI_REL_OFFSET rip,0 pushq_cfi %rax cld - SAVE_ARGS 0,0,1 + SAVE_ARGS 0,1,0 /* no need to do an access_ok check here because rbp has been 32bit zero extended */ 1: movl (%rbp),%ebp
@@ -173,7 +173,7 @@ sysexit_from_sys_call: andl $~0x200,EFLAGS-R11(%rsp) movl RIP-R11(%rsp),%edx /* User %eip */ CFI_REGISTER rip,rdx - RESTORE_ARGS 1,24,1,1,1,1 + RESTORE_ARGS 0,24,0,0,0,0 xorq %r8,%r8 xorq %r9,%r9 xorq %r10,%r10
@@ -289,7 +289,7 @@ ENTRY(ia32_cstar_target) * disabled irqs and here we enable it straight after entry: */ ENABLE_INTERRUPTS(CLBR_NONE) - SAVE_ARGS 8,1,1 + SAVE_ARGS 8,0,0 movl %eax,%eax /* zero extension */ movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp)
@@ -328,7 +328,7 @@ cstar_dispatch: jnz sysretl_audit sysretl_from_sys_call: andl $~TS_COMPAT,TI_status(%r10) - RESTORE_ARGS 1,-ARG_SKIP,1,1,1 + RESTORE_ARGS 0,-ARG_SKIP,0,0,0 movl RIP-ARGOFFSET(%rsp),%ecx CFI_REGISTER rip,rcx movl EFLAGS-ARGOFFSET(%rsp),%r11d
@@ -419,7 +419,7 @@ ENTRY(ia32_syscall) cld /* note the registers are not zero extended to the sf. this could be a problem. */ - SAVE_ARGS 0,0,1 + SAVE_ARGS 0,1,0 GET_THREAD_INFO(%r10) orl $TS_COMPAT,TI_status(%r10) testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%r10)
arch/x86/include/asm/asm.h +4 -1
@@ -3,9 +3,11 @@#ifdef __ASSEMBLY__# define __ASM_FORM (x) x +# define __ASM_FORM_COMMA (x) x,# define __ASM_EX_SEC .section __ex_table, "a"#else # define __ASM_FORM (x) " " #x " " +# define __ASM_FORM_COMMA (x) " " #x ","# define __ASM_EX_SEC " .section __ex_table,\"a\"\n"#endif
@@ -15,7 +17,8 @@# define __ASM_SEL (a,b) __ASM_FORM(b)#endif -#define __ASM_SIZE (inst) __ASM_SEL(inst##l, inst##q) +#define __ASM_SIZE (inst, ...) __ASM_SEL(inst##l##__VA_ARGS__, \ + inst##q##__VA_ARGS__)#define __ASM_REG (reg) __ASM_SEL(e##reg, r##reg)#define _ASM_PTR __ASM_SEL(.long, .quad)
arch/x86/include/asm/calling.h +52 -78
170 lines not on this page · raw message
arch/x86/include/asm/frame.h +6 -5
@@ -1,5 +1,6 @@#ifdef __ASSEMBLY__ +#include <asm/asm.h> #include <asm/dwarf2.h> /* The annotation hides the frame from the unwinder and makes it look
@@ -7,13 +8,13 @@frame pointer later */ #ifdef CONFIG_FRAME_POINTER .macro FRAME - pushl_cfi %ebp - CFI_REL_OFFSET ebp,0 - movl %esp,%ebp + __ASM_SIZE(push,_cfi) %__ASM_REG(bp) + CFI_REL_OFFSET __ASM_REG(bp), 0 + __ASM_SIZE(mov) %__ASM_REG(sp), %__ASM_REG(bp) .endm .macro ENDFRAME - popl_cfi %ebp - CFI_RESTORE ebp + __ASM_SIZE(pop,_cfi) %__ASM_REG(bp) + CFI_RESTORE __ASM_REG(bp) .endm #else .macro FRAME
arch/x86/include/asm/rwlock.h +42 -1
49 lines not on this page · raw message
arch/x86/include/asm/segment.h +1 -1
@@ -162,7 +162,7 @@#define GDT_ENTRY_DEFAULT_USER32_CS 4#define GDT_ENTRY_DEFAULT_USER_DS 5#define GDT_ENTRY_DEFAULT_USER_CS 6 -#define __USER32_CS (GDT_ENTRY_DEFAULT_USER32_CS * 8 + 3) +#define __USER32_CS (GDT_ENTRY_DEFAULT_USER32_CS*8+3)#define __USER32_DS __USER_DS#define GDT_ENTRY_TSS 8/* needs two entries */
arch/x86/include/asm/spinlock.h +22 -15
@@ -2,7 +2,6 @@#define _ASM_X86_SPINLOCK_H#include <asm/atomic.h> -#include <asm/rwlock.h> #include <asm/page.h> #include <asm/processor.h> #include <linux/compiler.h>
@@ -234,7 +233,7 @@ static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)*/ static inline int arch_read_can_lock (arch_rwlock_t *lock) { -return (int )(lock)->lock >0 ; +return lock->lock >0 ; }/**
@@ -243,12 +242,12 @@ static inline int arch_read_can_lock(arch_rwlock_t *lock)*/ static inline int arch_write_can_lock (arch_rwlock_t *lock) { -return (lock)->lock ==RW_LOCK_BIAS ; +return lock->write ==WRITE_LOCK_CMP ; }static inline void arch_read_lock (arch_rwlock_t *rw) { - asmvolatile (LOCK_PREFIX " subl $1,(%0)\n\t" + asm volatile(LOCK_PREFIX READ_LOCK_SIZE(dec) " (%0)\n\t""jns 1f\n" "call __read_lock_failed\n\t" "1:\n"
@@ -257,47 +256,55 @@ static inline void arch_read_lock(arch_rwlock_t *rw)static inline void arch_write_lock (arch_rwlock_t *rw) { - asmvolatile (LOCK_PREFIX " subl %1,(%0)\n\t" + asm volatile(LOCK_PREFIX WRITE_LOCK_SUB(%1) "(%0)\n\t""jz 1f\n" "call __write_lock_failed\n\t" "1:\n" - ::LOCK_PTR_REG (rw), "i" (RW_LOCK_BIAS) : "memory"); + ::LOCK_PTR_REG (&rw->write), "i" (RW_LOCK_BIAS) + : "memory"); }static inline int arch_read_trylock (arch_rwlock_t *lock) { -atomic_t *count = (atomic_t *)lock; +READ_LOCK_ATOMIC (t) *count = (READ_LOCK_ATOMIC (t) *)lock; -if (atomic_dec_return (count) >=0 ) +if (READ_LOCK_ATOMIC (dec_return)(count) >=0 )return 1 ; -atomic_inc (count); +READ_LOCK_ATOMIC (inc)(count);return 0 ; }static inline int arch_write_trylock (arch_rwlock_t *lock) { -atomic_t *count = (atomic_t *)lock; +atomic_t *count = (atomic_t *)&lock->write; -if (atomic_sub_and_test (RW_LOCK_BIAS , count)) +if (atomic_sub_and_test (WRITE_LOCK_CMP , count))return 1 ; -atomic_add (RW_LOCK_BIAS , count); +atomic_add (WRITE_LOCK_CMP , count);return 0 ; }static inline void arch_read_unlock (arch_rwlock_t *rw) { - asmvolatile (LOCK_PREFIX "incl %0" :"+m" (rw->lock) : :"memory" ); + asm volatile(LOCK_PREFIX READ_LOCK_SIZE(inc) " %0" + :"+m" (rw->lock) : :"memory" ); }static inline void arch_write_unlock (arch_rwlock_t *rw) { - asmvolatile (LOCK_PREFIX "addl %1, %0" - :"+m" (rw->lock) :"i" (RW_LOCK_BIAS ) :"memory" ); + asm volatile(LOCK_PREFIX WRITE_LOCK_ADD(%1) "%0" + :"+m" (rw->write) :"i" (RW_LOCK_BIAS ) :"memory" ); }#define arch_read_lock_flags (lock, flags) arch_read_lock(lock)#define arch_write_lock_flags (lock, flags) arch_write_lock(lock) +#undef READ_LOCK_SIZE +#undef READ_LOCK_ATOMIC +#undef WRITE_LOCK_ADD +#undef WRITE_LOCK_SUB +#undef WRITE_LOCK_CMP +#define arch_spin_relax (lock) cpu_relax()#define arch_read_relax (lock) cpu_relax()#define arch_write_relax (lock) cpu_relax()
arch/x86/include/asm/spinlock_types.h +1 -5
@@ -11,10 +11,6 @@ typedef struct arch_spinlock {#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } -typedef struct { -unsigned int lock; -}arch_rwlock_t ; - -#define __ARCH_RW_LOCK_UNLOCKED { RW_LOCK_BIAS } +#include <asm/rwlock.h> #endif /* _ASM_X86_SPINLOCK_TYPES_H */
arch/x86/kernel/entry_64.S +3 -3
@@ -473,7 +473,7 @@ ENTRY(system_call_after_swapgs) * and short: */ ENABLE_INTERRUPTS(CLBR_NONE) - SAVE_ARGS 8,1 + SAVE_ARGS 8,0 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) movq %rcx,RIP-ARGOFFSET(%rsp) CFI_REL_OFFSET rip,RIP-ARGOFFSET
@@ -508,7 +508,7 @@ sysret_check: TRACE_IRQS_ON movq RIP-ARGOFFSET(%rsp),%rcx CFI_REGISTER rip,rcx - RESTORE_ARGS 0,-ARG_SKIP,1 + RESTORE_ARGS 1,-ARG_SKIP,0 /*CFI_REGISTER rflags,r11*/ movq PER_CPU_VAR(old_rsp), %rsp USERGS_SYSRET64
@@ -858,7 +858,7 @@ retint_restore_args: /* return to kernel space */ */ TRACE_IRQS_IRETQ restore_args: - RESTORE_ARGS 0,8,0 + RESTORE_ARGS 1,8,1 irq_return: INTERRUPT_RETURN
arch/x86/lib/Makefile +4 -3
@@ -20,6 +20,8 @@ lib-y := delay.o lib-y += thunk_$(BITS).o lib-y += usercopy_$(BITS).o getuser.o putuser.o lib-y += memcpy_$(BITS).o +lib-$(CONFIG_SMP) += rwlock.o +lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o obj-y += msr.o msr-reg.o msr-reg-export.o
@@ -29,7 +31,7 @@ ifeq ($(CONFIG_X86_32),y) lib-y += atomic64_cx8_32.o lib-y += checksum_32.o lib-y += strstr_32.o - lib-y += semaphore_32.o string_32.o + lib-y += string_32.o lib-y += cmpxchg.o ifneq ($(CONFIG_X86_CMPXCHG64),y) lib-y += cmpxchg8b_emu.o atomic64_386_32.o
@@ -40,7 +42,6 @@ else lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o lib-y += thunk_64.o clear_page_64.o copy_page_64.o lib-y += memmove_64.o memset_64.o - lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o - lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o + lib-y += copy_user_64.o copy_user_nocache_64.o lib-y += cmpxchg16b_emu.o endif
arch/x86/lib/rwlock.S new file +44 -0
44 lines not on this page · raw message
arch/x86/lib/rwlock_64.S deleted +0 -38
@@ -1,38 +0,0 @@ -/* Slow paths of read/write spinlocks. */ - -#include <linux/linkage.h> -#include <asm/rwlock.h> -#include <asm/alternative-asm.h> -#include <asm/dwarf2.h> - -/* rdi: pointer to rwlock_t */ -ENTRY(__write_lock_failed) - CFI_STARTPROC - LOCK_PREFIX - addl $RW_LOCK_BIAS,(%rdi) -1: rep - nop - cmpl $RW_LOCK_BIAS,(%rdi) - jne 1b - LOCK_PREFIX - subl $RW_LOCK_BIAS,(%rdi) - jnz __write_lock_failed - ret - CFI_ENDPROC -END(__write_lock_failed) - -/* rdi: pointer to rwlock_t */ -ENTRY(__read_lock_failed) - CFI_STARTPROC - LOCK_PREFIX - incl (%rdi) -1: rep - nop - cmpl $1,(%rdi) - js 1b - LOCK_PREFIX - decl (%rdi) - js __read_lock_failed - ret - CFI_ENDPROC -END(__read_lock_failed)
arch/x86/lib/rwsem.S renamed from arch/x86/lib/rwsem_64.S +59 -16
114 lines not on this page · raw message
arch/x86/lib/semaphore_32.S deleted +0 -124
124 lines not on this page · raw message
arch/x86/lib/thunk_64.S +18 -27
68 lines not on this page · raw message