locking changes for v4.10
Subsystems (33)
Linus,
Please pull the latest locking-core-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking-core-for-linus
# HEAD: 11f254dbb3a2e3f0d8552d0dd37f4faa432b6b16 x86/paravirt: Fix bool return type for PVOP_CALL()
The tree got pretty big in this development cycle, but the net effect is pretty good:
115 files changed, 673 insertions(+), 1522 deletions(-)
The main changes were:
- Rework and generalize the mutex code to remove per arch mutex primitives. (Peter Zijlstra)
- Add vCPU preemption support: add an interface to query the preemption status of vCPUs and use it in locking primitives - this optimizes paravirt performance. (Pan Xinhui, Juergen Gross, Christian Borntraeger)
- Introduce cpu_relax_yield() and remov cpu_relax_lowlatency() to clean up and improve the s390 lock yielding machinery and its core kernel impact. (Christian Borntraeger)
- Micro-optimize mutexes some more. (Waiman Long)
- Reluctantly add the to-be-deprecated mutex_trylock_recursive() interface on a temporary basis, to give the DRM code more time to get rid of its locking hacks. Any other users will be NAK-ed on sight. (We turned off the deprecation warning for the time being to not pollute the build log.) (Peter Zijlstra)
- Improve the rtmutex code a bit, in light of recent long lived bugs/races. (Thomas Gleixner)
- Misc fixes, cleanups.
Thanks,
Ingo
------------------>
Christian Borntraeger (7):
locking/core: Introduce cpu_relax_yield() locking/core, stop_machine: Yield the CPU during stop machine() locking/core, s390: Make cpu_relax() a barrier again locking/core: Remove cpu_relax_lowlatency() users locking/core, arch: Remove cpu_relax_lowlatency() locking/core: Provide common cpu_relax_yield() definition locking/spinlocks, s390: Implement vcpu_is_preempted(cpu)
Ingo Molnar (2):
locking/drm: Fix i915_gem_shrinker_lock() locking
locking/mutex: Don't mark mutex_trylock_recursive() as deprecated, temporarily
Juergen Gross (1):
x86/xen: Support the vCPU preemption check
Pan Xinhui (9):
sched/core: Introduce the vcpu_is_preempted(cpu) interface
locking/core, powerpc: Implement vcpu_is_preempted(cpu)
locking/core, x86/paravirt: Implement vcpu_is_preempted(cpu) for KVM and Xen guests
kvm: Introduce kvm_write_guest_offset_cached()
x86/kvm: Support the vCPU preemption check
x86/kvm: Support the vCPU preemption check
Documentation/virtual/kvm: Support the vCPU preemption check
locking/osq: Break out of spin-wait busy waiting loop for a preempted vCPU in osq_lock() locking/mutex: Break out of expensive busy-loop on {mutex,rwsem}_spin_on_owner() when owner vCPU is preempted
Peter Zijlstra (11):
locking/drm: Kill mutex trickery
locking/mutex: Rework mutex::owner
locking/mutex: Kill arch specific code locking/mutex: Allow MUTEX_SPIN_ON_OWNER when DEBUG_MUTEXES locking/mutex: Add lock handoff to avoid starvation locking/mutex: Restructure wait loop locking/mutex, drm: Introduce mutex_trylock_recursive() x86/paravirt: Optimize native pv_lock_ops.vcpu_is_preempted() locking/ww_mutex: Use relaxed atomics x86/paravirt: Fix native_patch()
x86/paravirt: Fix bool return type for PVOP_CALL()
Tahsin Erdogan (1):
locking/lockdep: Remove unused parameter from the add_lock_to_list() function
Thomas Gleixner (2):
locking/rtmutex: Get rid of RT_MUTEX_OWNER_MASKALL
locking/rtmutex: Explain locking rules for rt_mutex_proxy_unlock()/init_proxy_locked()
Waiman Long (3):
locking/mutex: Simplify some ww_mutex code in __mutex_lock_common() locking/mutex: Enable optimistic spinning of woken waiter sched/wake_q: Rename WAKE_Q to DEFINE_WAKE_Q
Documentation/virtual/kvm/msr.txt | 9 +- arch/alpha/include/asm/mutex.h | 9 - arch/alpha/include/asm/processor.h | 1 - arch/arc/include/asm/mutex.h | 18 - arch/arc/include/asm/processor.h | 3 - arch/arm/include/asm/mutex.h | 21 -- arch/arm/include/asm/processor.h | 2 - arch/arm64/include/asm/Kbuild | 1 - arch/arm64/include/asm/processor.h | 2 - arch/avr32/include/asm/mutex.h | 9 - arch/avr32/include/asm/processor.h | 1 - arch/blackfin/include/asm/Kbuild | 1 - arch/blackfin/include/asm/processor.h | 1 - arch/c6x/include/asm/mutex.h | 6 - arch/c6x/include/asm/processor.h | 1 - arch/cris/include/asm/mutex.h | 9 - arch/cris/include/asm/processor.h | 1 - arch/frv/include/asm/mutex.h | 9 - arch/frv/include/asm/processor.h | 1 - arch/h8300/include/asm/mutex.h | 9 - arch/h8300/include/asm/processor.h | 1 - arch/hexagon/include/asm/mutex.h | 8 - arch/hexagon/include/asm/processor.h | 1 - arch/ia64/include/asm/mutex.h | 90 ----- arch/ia64/include/asm/processor.h | 1 - arch/m32r/include/asm/mutex.h | 9 - arch/m32r/include/asm/processor.h | 1 - arch/m68k/include/asm/Kbuild | 1 - arch/m68k/include/asm/processor.h | 1 - arch/metag/include/asm/Kbuild | 1 - arch/metag/include/asm/processor.h | 1 - arch/microblaze/include/asm/mutex.h | 1 - arch/microblaze/include/asm/processor.h | 1 - arch/mips/include/asm/Kbuild | 1 - arch/mips/include/asm/processor.h | 1 - arch/mn10300/include/asm/mutex.h | 16 - arch/mn10300/include/asm/processor.h | 1 - arch/nios2/include/asm/mutex.h | 1 - arch/nios2/include/asm/processor.h | 1 - arch/openrisc/include/asm/mutex.h | 27 -- arch/openrisc/include/asm/processor.h | 1 - arch/parisc/include/asm/Kbuild | 1 - arch/parisc/include/asm/processor.h | 1 - arch/powerpc/include/asm/mutex.h | 132 ------- arch/powerpc/include/asm/processor.h | 2 - arch/powerpc/include/asm/spinlock.h | 8 + arch/s390/include/asm/mutex.h | 9 - arch/s390/include/asm/processor.h | 5 +- arch/s390/include/asm/spinlock.h | 8 + arch/s390/kernel/processor.c | 4 +- arch/s390/kernel/smp.c | 9 +- arch/s390/lib/spinlock.c | 25 +- arch/score/include/asm/mutex.h | 6 - arch/score/include/asm/processor.h | 1 - arch/sh/include/asm/mutex-llsc.h | 109 ------ arch/sh/include/asm/mutex.h | 12 - arch/sh/include/asm/processor.h | 1 - arch/sparc/include/asm/Kbuild | 1 - arch/sparc/include/asm/processor_32.h | 1 - arch/sparc/include/asm/processor_64.h | 1 - arch/tile/include/asm/Kbuild | 1 - arch/tile/include/asm/processor.h | 2 - arch/um/include/asm/Kbuild | 1 - arch/unicore32/include/asm/mutex.h | 20 -- arch/unicore32/include/asm/processor.h | 1 - arch/x86/include/asm/mutex.h | 5 - arch/x86/include/asm/mutex_32.h | 110 ------ arch/x86/include/asm/mutex_64.h | 127 ------- arch/x86/include/asm/paravirt.h | 5 + arch/x86/include/asm/paravirt_types.h | 16 +- arch/x86/include/asm/processor.h | 2 - arch/x86/include/asm/qspinlock.h | 6 + arch/x86/include/uapi/asm/kvm_para.h | 4 +- arch/x86/kernel/kvm.c | 13 + arch/x86/kernel/paravirt-spinlocks.c | 14 +- arch/x86/kernel/paravirt_patch_32.c | 12 + arch/x86/kernel/paravirt_patch_64.c | 12 + arch/x86/kvm/x86.c | 16 + arch/x86/um/asm/processor.h | 1 - arch/x86/xen/spinlock.c | 2 + arch/xtensa/include/asm/mutex.h | 9 - arch/xtensa/include/asm/processor.h | 1 - drivers/gpu/drm/i915/i915_gem_request.c | 2 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 30 +- drivers/gpu/drm/msm/msm_gem_shrinker.c | 27 +- drivers/vhost/net.c | 4 +- include/asm-generic/mutex-dec.h | 88 ----- include/asm-generic/mutex-null.h | 19 - include/asm-generic/mutex-xchg.h | 120 ------- include/asm-generic/mutex.h | 9 - include/linux/kvm_host.h | 2 + include/linux/mutex-debug.h | 24 -- include/linux/mutex.h | 77 +++- include/linux/sched.h | 20 +- include/linux/ww_mutex.h | 2 +- ipc/mqueue.c | 4 +- ipc/msg.c | 8 +- kernel/Kconfig.locks | 2 +- kernel/futex.c | 8 +- kernel/locking/lockdep.c | 10 +- kernel/locking/mcs_spinlock.h | 4 +- kernel/locking/mutex-debug.c | 13 - kernel/locking/mutex-debug.h | 10 - kernel/locking/mutex.c | 588 ++++++++++++++++++------------- kernel/locking/mutex.h | 26 -- kernel/locking/osq_lock.c | 15 +- kernel/locking/qrwlock.c | 6 +- kernel/locking/rtmutex.c | 18 +- kernel/locking/rtmutex_common.h | 3 +- kernel/locking/rwsem-xadd.c | 28 +- kernel/sched/core.c | 2 +- kernel/stop_machine.c | 2 +- lib/lockref.c | 2 +- scripts/checkpatch.pl | 6 + virt/kvm/kvm_main.c | 20 +- 115 files changed, 673 insertions(+), 1522 deletions(-) delete mode 100644 arch/alpha/include/asm/mutex.h delete mode 100644 arch/arc/include/asm/mutex.h delete mode 100644 arch/arm/include/asm/mutex.h delete mode 100644 arch/avr32/include/asm/mutex.h delete mode 100644 arch/c6x/include/asm/mutex.h delete mode 100644 arch/cris/include/asm/mutex.h delete mode 100644 arch/frv/include/asm/mutex.h delete mode 100644 arch/h8300/include/asm/mutex.h delete mode 100644 arch/hexagon/include/asm/mutex.h delete mode 100644 arch/ia64/include/asm/mutex.h delete mode 100644 arch/m32r/include/asm/mutex.h delete mode 100644 arch/microblaze/include/asm/mutex.h delete mode 100644 arch/mn10300/include/asm/mutex.h delete mode 100644 arch/nios2/include/asm/mutex.h delete mode 100644 arch/openrisc/include/asm/mutex.h delete mode 100644 arch/powerpc/include/asm/mutex.h delete mode 100644 arch/s390/include/asm/mutex.h delete mode 100644 arch/score/include/asm/mutex.h delete mode 100644 arch/sh/include/asm/mutex-llsc.h delete mode 100644 arch/sh/include/asm/mutex.h delete mode 100644 arch/unicore32/include/asm/mutex.h delete mode 100644 arch/x86/include/asm/mutex.h delete mode 100644 arch/x86/include/asm/mutex_32.h delete mode 100644 arch/x86/include/asm/mutex_64.h delete mode 100644 arch/xtensa/include/asm/mutex.h delete mode 100644 include/asm-generic/mutex-dec.h delete mode 100644 include/asm-generic/mutex-null.h delete mode 100644 include/asm-generic/mutex-xchg.h delete mode 100644 include/asm-generic/mutex.h delete mode 100644 include/linux/mutex-debug.h
Documentation/virtual/kvm/msr.txt +8 -1
@@ -208,7 +208,9 @@ MSR_KVM_STEAL_TIME: 0x4b564d03 __u64 steal; __u32 version; __u32 flags; - __u32 pad[12]; + __u8 preempted; + __u8 u8_pad[3]; + __u32 pad[11]; } whose data will be filled in by the hypervisor periodically. Only one
@@ -232,6 +234,11 @@ MSR_KVM_STEAL_TIME: 0x4b564d03 nanoseconds. Time during which the vcpu is idle, will not be reported as steal time. + preempted: indicate the vCPU who owns this struct is running or + not. Non-zero values mean the vCPU has been preempted. Zero + means the vCPU is not preempted. NOTE, it is always zero if the + the hypervisor doesn't support this field. + MSR_KVM_EOI_EN: 0x4b564d04 data: Bit 0 is 1 when PV end of interrupt is enabled on the vcpu; 0 when disabled. Bit 1 is reserved and must be zero. When PV end of
arch/alpha/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/alpha/include/asm/processor.h +0 -1
@@ -58,7 +58,6 @@ unsigned long get_wchan(struct task_struct *p); ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#define ARCH_HAS_PREFETCH #define ARCH_HAS_PREFETCHW
arch/arc/include/asm/mutex.h deleted +0 -18
@@ -1,18 +0,0 @@ -/* -* Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) -* -* This program is free software; you can redistribute it and/or modify -* it under the terms of the GNU General Public License version 2 as -* published by the Free Software Foundation. -*/ - -/* -* xchg() based mutex fast path maintains a state of 0 or 1, as opposed to -* atomic dec based which can "count" any number of lock contenders. -* This ideally needs to be fixed in core, but for now switching to dec ver. -*/ -#if defined(CONFIG_SMP ) && (CONFIG_NR_CPUS >2 ) -#include <asm-generic/mutex-dec.h> -#else -#include <asm-generic/mutex-xchg.h> -#endif
arch/arc/include/asm/processor.h +0 -3
@@ -60,15 +60,12 @@ struct task_struct;#ifndef CONFIG_EZNPS_MTM_EXT #define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#else #define cpu_relax () \ __asm__ __volatile__ (".word %0" : : "i"(CTOP_INST_SCHD_RW) : "memory") -#define cpu_relax_lowlatency () barrier() -#endif #define copy_segments (tsk, mm) do { } while (0)
arch/arm/include/asm/mutex.h deleted +0 -21
@@ -1,21 +0,0 @@ -/* -* arch/arm/include/asm/mutex.h -* -* ARM optimized mutex locking primitives -* -* Please look into asm-generic/mutex-xchg.h for a formal definition. -*/ -#ifndef _ASM_MUTEX_H -#define _ASM_MUTEX_H -/* -* On pre-ARMv6 hardware this results in a swp-based implementation, -* which is the most efficient. For ARMv6+, we have exclusive memory -* accessors and use atomic_dec to avoid the extra xchg operations -* on the locking slowpaths. -*/ -#if __LINUX_ARM_ARCH__ <6 -#include <asm-generic/mutex-xchg.h> -#else -#include <asm-generic/mutex-dec.h> -#endif -#endif /* _ASM_MUTEX_H */
arch/arm/include/asm/processor.h +0 -2
@@ -82,8 +82,6 @@ unsigned long get_wchan(struct task_struct *p);#define cpu_relax () barrier()#endif -#define cpu_relax_lowlatency () cpu_relax() -#define task_pt_regs (p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
arch/arm64/include/asm/Kbuild +0 -1
@@ -24,7 +24,6 @@ generic-y += mm-arch-hooks.h generic-y += mman.h generic-y += msgbuf.h generic-y += msi.h -generic-y += mutex.h generic-y += poll.h generic-y += preempt.h generic-y += resource.h
arch/arm64/include/asm/processor.h +0 -2
@@ -149,8 +149,6 @@ static inline void cpu_relax(void) asmvolatile ("yield" :::"memory" ); } -#define cpu_relax_lowlatency () cpu_relax() -/* Thread switching */ extern struct task_struct *cpu_switch_to (struct task_struct *prev,struct task_struct *next);
arch/avr32/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/avr32/include/asm/processor.h +0 -1
@@ -92,7 +92,6 @@ extern struct avr32_cpuinfo boot_cpu_data;#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3))#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#define cpu_sync_pipeline () asm volatile("sub pc, -2" : : : "memory")struct cpu_context {
arch/blackfin/include/asm/Kbuild +0 -1
@@ -24,7 +24,6 @@ generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += mman.h generic-y += msgbuf.h -generic-y += mutex.h generic-y += param.h generic-y += percpu.h generic-y += pgalloc.h
arch/blackfin/include/asm/processor.h +0 -1
@@ -92,7 +92,6 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk) == current ? rdusp() : (tsk)->thread.usp)#define cpu_relax () smp_mb() -#define cpu_relax_lowlatency () cpu_relax()/* Get the Silicon Revision of the chip */ static inline uint32_t __pure bfin_revid (void )
arch/c6x/include/asm/mutex.h deleted +0 -6
@@ -1,6 +0,0 @@ -#ifndef _ASM_C6X_MUTEX_H -#define _ASM_C6X_MUTEX_H - -#include <asm-generic/mutex-null.h> - -#endif /* _ASM_C6X_MUTEX_H */
arch/c6x/include/asm/processor.h +0 -1
@@ -121,7 +121,6 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (task) (task_pt_regs(task)->sp)#define cpu_relax () do { } while (0) -#define cpu_relax_lowlatency () cpu_relax()extern const struct seq_operations cpuinfo_op;
arch/cris/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/cris/include/asm/processor.h +0 -1
@@ -63,7 +63,6 @@ static inline void release_thread(struct task_struct *dead_task)#define init_stack (init_thread_union.stack)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()void default_idle (void );
arch/frv/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/frv/include/asm/processor.h +0 -1
@@ -107,7 +107,6 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.frame0->sp)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()/* data cache prefetch */ #define ARCH_HAS_PREFETCH
arch/h8300/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/h8300/include/asm/processor.h +0 -1
@@ -127,7 +127,6 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk) == current ? rdusp() : (tsk)->thread.usp)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#define HARD_RESET_NOW () ({ \ local_irq_disable(); \
arch/hexagon/include/asm/mutex.h deleted +0 -8
@@ -1,8 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ -#include <asm-generic/mutex-xchg.h>
arch/hexagon/include/asm/processor.h +0 -1
@@ -56,7 +56,6 @@ struct thread_struct { }#define cpu_relax () __vmyield() -#define cpu_relax_lowlatency () cpu_relax()/* * Decides where the kernel will search for a free chunk of vm space during
arch/ia64/include/asm/mutex.h deleted +0 -90
90 lines not on this page · raw message
arch/ia64/include/asm/processor.h +0 -1
@@ -547,7 +547,6 @@ ia64_eoi (void) }#define cpu_relax () ia64_hint(ia64_hint_pause) -#define cpu_relax_lowlatency () cpu_relax()static inline int ia64_get_irr (unsigned int vector)
arch/m32r/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/m32r/include/asm/processor.h +0 -1
@@ -133,6 +133,5 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.sp)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#endif /* _ASM_M32R_PROCESSOR_H */
arch/m68k/include/asm/Kbuild +0 -1
@@ -20,7 +20,6 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += mman.h -generic-y += mutex.h generic-y += percpu.h generic-y += preempt.h generic-y += resource.h
arch/m68k/include/asm/processor.h +0 -1
@@ -156,6 +156,5 @@ unsigned long get_wchan(struct task_struct *p);#define task_pt_regs (tsk) ((struct pt_regs *) ((tsk)->thread.esp0))#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#endif
arch/metag/include/asm/Kbuild +0 -1
@@ -27,7 +27,6 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += msgbuf.h -generic-y += mutex.h generic-y += param.h generic-y += pci.h generic-y += percpu.h
arch/metag/include/asm/processor.h +0 -1
@@ -152,7 +152,6 @@ unsigned long get_wchan(struct task_struct *p);#define user_stack_pointer (regs) ((regs)->ctx.AX[0].U0)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()extern void setup_priv (void );
arch/microblaze/include/asm/mutex.h deleted +0 -1
@@ -1 +0,0 @@ -#include <asm-generic/mutex-dec.h>
arch/microblaze/include/asm/processor.h +0 -1
@@ -22,7 +22,6 @@extern const struct seq_operations cpuinfo_op;# define cpu_relax () barrier() -# define cpu_relax_lowlatency () cpu_relax()#define task_pt_regs (tsk) \ (((struct pt_regs *)(THREAD_SIZE + task_stack_page(tsk))) - 1)
arch/mips/include/asm/Kbuild +0 -1
@@ -9,7 +9,6 @@ generic-y += irq_work.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h -generic-y += mutex.h generic-y += parport.h generic-y += percpu.h generic-y += preempt.h
arch/mips/include/asm/processor.h +0 -1
@@ -389,7 +389,6 @@ unsigned long get_wchan(struct task_struct *p);#define KSTK_STATUS (tsk) (task_pt_regs(tsk)->cp0_status)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()/* * Return_address is a replacement for __builtin_return_address(count)
arch/mn10300/include/asm/mutex.h deleted +0 -16
@@ -1,16 +0,0 @@ -/* MN10300 Mutex fastpath -* -* Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. -* Written by David Howells ([email protected]) -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public Licence -* as published by the Free Software Foundation; either version -* 2 of the Licence, or (at your option) any later version. -* -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ -#include <asm-generic/mutex-null.h>
arch/mn10300/include/asm/processor.h +0 -1
@@ -69,7 +69,6 @@ extern void print_cpu_info(struct mn10300_cpuinfo *);extern void dodgy_tsc (void );#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()/* * User space process size: 1.75GB (default).
arch/nios2/include/asm/mutex.h deleted +0 -1
@@ -1 +0,0 @@ -#include <asm-generic/mutex-dec.h>
arch/nios2/include/asm/processor.h +0 -1
@@ -88,7 +88,6 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.kregs->sp)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#endif /* __ASSEMBLY__ */
arch/openrisc/include/asm/mutex.h deleted +0 -27
@@ -1,27 +0,0 @@ -/* -* OpenRISC Linux -* -* Linux architectural port borrowing liberally from similar works of -* others. All original copyrights apply as per the original source -* declaration. -* -* OpenRISC implementation: -* Copyright (C) 2003 Matjaz Breskvar <[email protected]> -* Copyright (C) 2010-2011 Jonas Bonn <[email protected]> -* et al. -* -* This program is free software; you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation; either version 2 of the License, or -* (at your option) any later version. -*/ - -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/openrisc/include/asm/processor.h +0 -1
@@ -92,7 +92,6 @@ extern unsigned long thread_saved_pc(struct task_struct *t);#define init_stack (init_thread_union.stack)#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#endif /* __ASSEMBLY__ */ #endif /* __ASM_OPENRISC_PROCESSOR_H */
arch/parisc/include/asm/Kbuild +0 -1
@@ -16,7 +16,6 @@ generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h -generic-y += mutex.h generic-y += param.h generic-y += percpu.h generic-y += poll.h
arch/parisc/include/asm/processor.h +0 -1
@@ -309,7 +309,6 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) ((tsk)->thread.regs.gr[30])#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()/* * parisc_requires_coherency() is used to identify the combined VIPT/PIPT
arch/powerpc/include/asm/mutex.h deleted +0 -132
132 lines not on this page · raw message
arch/powerpc/include/asm/processor.h +0 -2
@@ -404,8 +404,6 @@ static inline unsigned long __pack_fe01(unsigned int fpmode)#define cpu_relax () barrier()#endif -#define cpu_relax_lowlatency () cpu_relax() -/* Check that a certain kernel stack pointer is valid in task_struct p */ int validate_sp (unsigned long sp,struct task_struct *p,unsigned long nbytes);
arch/powerpc/include/asm/spinlock.h +8 -0
@@ -52,6 +52,14 @@#define SYNC_IO #endif +#ifdef CONFIG_PPC_PSERIES +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted (int cpu) +{ +return !!(be32_to_cpu (lppaca_of (cpu).yield_count) &1 ); +} +#endif + static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock) {return lock.slock ==0 ;
arch/s390/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/s390/include/asm/processor.h +3 -2
@@ -234,9 +234,10 @@ static inline unsigned short stap(void)/* * Give up the time slice of the virtual PU. */ -void cpu_relax (void ); +#define cpu_relax_yield cpu_relax_yield +void cpu_relax_yield (void ); -#define cpu_relax_lowlatency () barrier() +#define cpu_relax () barrier()#define ECAG_CACHE_ATTRIBUTE 0#define ECAG_CPU_ATTRIBUTE 1
arch/s390/include/asm/spinlock.h +8 -0
@@ -23,6 +23,14 @@ _raw_compare_and_swap(unsigned int *lock, unsigned int old, unsigned int new)return __sync_bool_compare_and_swap (lock, old, new); } +#ifndef CONFIG_SMP +static inline bool arch_vcpu_is_preempted (int cpu) {return false; } +#else +bool arch_vcpu_is_preempted (int cpu); +#endif + +#define vcpu_is_preempted arch_vcpu_is_preempted +/* * Simple spin lock operations. There are two variants, one clears IRQ's * on the local processor, one does not.
arch/s390/kernel/processor.c +2 -2
@@ -53,7 +53,7 @@ void s390_update_cpu_mhz(void)on_each_cpu (update_cpu_mhz,NULL ,0 ); } -void notrace cpu_relax(void) +void notrace cpu_relax_yield(void) {if (!smp_cpu_mtid &&MACHINE_HAS_DIAG44 ) {diag_stat_inc (DIAG_STAT_X044 );
@@ -61,7 +61,7 @@ void notrace cpu_relax(void) }barrier (); } -EXPORT_SYMBOL (cpu_relax); +EXPORT_SYMBOL (cpu_relax_yield);/* * cpu_init - initializes state that is per-CPU.
arch/s390/kernel/smp.c +7 -2
@@ -368,10 +368,15 @@ int smp_find_processor_id(u16 address)return -1 ; } -int smp_vcpu_scheduled (int cpu) +bool arch_vcpu_is_preempted (int cpu) { -return pcpu_running (pcpu_devices + cpu); +if (test_cpu_flag_of (CIF_ENABLED_WAIT , cpu)) +return false; +if (pcpu_running (pcpu_devices + cpu)) +return false; +return true; } +EXPORT_SYMBOL (arch_vcpu_is_preempted);void smp_yield_cpu (int cpu) {
arch/s390/lib/spinlock.c +8 -17
@@ -37,15 +37,6 @@ static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old) asm(".insn rsy,0xeb0000000022,%0,0,%1" : :"d" (old),"Q" (*lock)); } -static inline int cpu_is_preempted (int cpu) -{ -if (test_cpu_flag_of (CIF_ENABLED_WAIT , cpu)) -return 0 ; -if (smp_vcpu_scheduled (cpu)) -return 0 ; -return 1 ; -} -void arch_spin_lock_wait (arch_spinlock_t *lp) {unsigned int cpu =SPINLOCK_LOCKVAL ;
@@ -62,7 +53,7 @@ void arch_spin_lock_wait(arch_spinlock_t *lp)continue ; }/* First iteration: check if the lock owner is running. */ -if (first_diag &&cpu_is_preempted (~owner)) { +if (first_diag &&arch_vcpu_is_preempted (~owner)) {smp_yield_cpu (~owner); first_diag =0 ;continue ;
@@ -81,7 +72,7 @@ void arch_spin_lock_wait(arch_spinlock_t *lp)* yield the CPU unconditionally. For LPAR rely on the * sense running status. */ -if (!MACHINE_IS_LPAR ||cpu_is_preempted (~owner)) { +if (!MACHINE_IS_LPAR ||arch_vcpu_is_preempted (~owner)) {smp_yield_cpu (~owner); first_diag =0 ; }
@@ -108,7 +99,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)continue ; }/* Check if the lock owner is running. */ -if (first_diag &&cpu_is_preempted (~owner)) { +if (first_diag &&arch_vcpu_is_preempted (~owner)) {smp_yield_cpu (~owner); first_diag =0 ;continue ;
@@ -127,7 +118,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)* yield the CPU unconditionally. For LPAR rely on the * sense running status. */ -if (!MACHINE_IS_LPAR ||cpu_is_preempted (~owner)) { +if (!MACHINE_IS_LPAR ||arch_vcpu_is_preempted (~owner)) {smp_yield_cpu (~owner); first_diag =0 ; }
@@ -165,7 +156,7 @@ void _raw_read_lock_wait(arch_rwlock_t *rw) owner =0 ;while (1 ) {if (count-- <=0 ) { -if (owner &&cpu_is_preempted (~owner)) +if (owner &&arch_vcpu_is_preempted (~owner))smp_yield_cpu (~owner); count = spin_retry; }
@@ -211,7 +202,7 @@ void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev) owner =0 ;while (1 ) {if (count-- <=0 ) { -if (owner &&cpu_is_preempted (~owner)) +if (owner &&arch_vcpu_is_preempted (~owner))smp_yield_cpu (~owner); count = spin_retry; }
@@ -241,7 +232,7 @@ void _raw_write_lock_wait(arch_rwlock_t *rw) owner =0 ;while (1 ) {if (count-- <=0 ) { -if (owner &&cpu_is_preempted (~owner)) +if (owner &&arch_vcpu_is_preempted (~owner))smp_yield_cpu (~owner); count = spin_retry; }
@@ -285,7 +276,7 @@ void arch_lock_relax(unsigned int cpu) {if (!cpu)return ; -if (MACHINE_IS_LPAR && !cpu_is_preempted (~cpu)) +if (MACHINE_IS_LPAR && !arch_vcpu_is_preempted (~cpu))return ;smp_yield_cpu (~cpu); }
arch/score/include/asm/mutex.h deleted +0 -6
@@ -1,6 +0,0 @@ -#ifndef _ASM_SCORE_MUTEX_H -#define _ASM_SCORE_MUTEX_H - -#include <asm-generic/mutex-dec.h> - -#endif /* _ASM_SCORE_MUTEX_H */
arch/score/include/asm/processor.h +0 -1
@@ -24,7 +24,6 @@ extern unsigned long get_wchan(struct task_struct *p);#define current_text_addr () ({ __label__ _l; _l: &&_l; })#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#define release_thread (thread) do {} while (0)/*
arch/sh/include/asm/mutex-llsc.h deleted +0 -109
109 lines not on this page · raw message
arch/sh/include/asm/mutex.h deleted +0 -12
@@ -1,12 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ -#if defined(CONFIG_CPU_SH4A ) -#include <asm/mutex-llsc.h> -#else -#include <asm-generic/mutex-dec.h> -#endif
arch/sh/include/asm/processor.h +0 -1
@@ -97,7 +97,6 @@ extern struct sh_cpuinfo cpu_data[];#define cpu_sleep () __asm__ __volatile__ ("sleep" : : : "memory")#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()void default_idle (void );void stop_this_cpu (void *);
arch/sparc/include/asm/Kbuild +0 -1
@@ -15,7 +15,6 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += module.h -generic-y += mutex.h generic-y += preempt.h generic-y += rwsem.h generic-y += serial.h
arch/sparc/include/asm/processor_32.h +0 -1
@@ -119,7 +119,6 @@ extern struct task_struct *last_task_used_math;int do_mathemu (struct pt_regs *regs,struct task_struct *fpt);#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()extern void (*sparc_idle)(void );
arch/sparc/include/asm/processor_64.h +0 -1
@@ -216,7 +216,6 @@ unsigned long get_wchan(struct task_struct *task); "nop\n\t" \ ".previous" \ ::: "memory") -#define cpu_relax_lowlatency () cpu_relax()/* Prefetch support. This is tuned for UltraSPARC-III and later. * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
arch/tile/include/asm/Kbuild +0 -1
@@ -21,7 +21,6 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += msgbuf.h -generic-y += mutex.h generic-y += param.h generic-y += parport.h generic-y += poll.h
arch/tile/include/asm/processor.h +0 -2
@@ -264,8 +264,6 @@ static inline void cpu_relax(void)barrier (); } -#define cpu_relax_lowlatency () cpu_relax() -/* Info on this processor (see fs/proc/cpuinfo.c) */ struct seq_operations ;extern const struct seq_operations cpuinfo_op;
arch/um/include/asm/Kbuild +0 -1
@@ -17,7 +17,6 @@ generic-y += irq_work.h generic-y += kdebug.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h -generic-y += mutex.h generic-y += param.h generic-y += pci.h generic-y += percpu.h
arch/unicore32/include/asm/mutex.h deleted +0 -20
@@ -1,20 +0,0 @@ -/* -* linux/arch/unicore32/include/asm/mutex.h -* -* Code specific to PKUnity SoC and UniCore ISA -* -* Copyright (C) 2001-2010 GUAN Xue-tao -* -* This program is free software; you can redistribute it and/or modify -* it under the terms of the GNU General Public License version 2 as -* published by the Free Software Foundation. -* -* UniCore optimized mutex locking primitives -* -* Please look into asm-generic/mutex-xchg.h for a formal definition. -*/ -#ifndef __UNICORE_MUTEX_H__ -#define __UNICORE_MUTEX_H__ - -# include <asm-generic/mutex-xchg.h> -#endif
arch/unicore32/include/asm/processor.h +0 -1
@@ -71,7 +71,6 @@ extern void release_thread(struct task_struct *);unsigned long get_wchan (struct task_struct *p);#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()#define task_pt_regs (p) \ ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
arch/x86/include/asm/mutex.h deleted +0 -5
@@ -1,5 +0,0 @@ -#ifdef CONFIG_X86_32 -# include <asm/mutex_32.h> -#else -# include <asm/mutex_64.h> -#endif
arch/x86/include/asm/mutex_32.h deleted +0 -110
110 lines not on this page · raw message
arch/x86/include/asm/mutex_64.h deleted +0 -127
127 lines not on this page · raw message
arch/x86/include/asm/paravirt.h +5 -0
@@ -678,6 +678,11 @@ static __always_inline void pv_kick(int cpu)PVOP_VCALL1 (pv_lock_ops.kick, cpu); } +static __always_inline bool pv_vcpu_is_preempted(int cpu) +{ +return PVOP_CALLEE1 (bool, pv_lock_ops.vcpu_is_preempted, cpu); +} +#endif /* SMP && PARAVIRT_SPINLOCKS */ #ifdef CONFIG_X86_32
arch/x86/include/asm/paravirt_types.h +15 -1
@@ -310,6 +310,8 @@ struct pv_lock_ops {void (*wait)(u8 *ptr,u8 val);void (*kick)(int cpu); + +struct paravirt_callee_save vcpu_is_preempted; };/* This contains all the paravirt structures: we get a convenient
@@ -508,6 +510,18 @@ int paravirt_disable_iospace(void);#define PVOP_TEST_NULL (op) ((void)op)#endif +#define PVOP_RETMASK (rettype) \ + ({ unsigned long __mask = ~0UL; \ + switch (sizeof(rettype)) { \ + case 1: __mask = 0xffUL; break; \ + case 2: __mask = 0xffffUL; break; \ + case 4: __mask = 0xffffffffUL; break; \ + default: break; \ + } \ + __mask; \ + }) + +#define ____PVOP_CALL (rettype, op, clbr, call_clbr, extra_clbr, \ pre, post, ...) \ ({ \
@@ -535,7 +549,7 @@ int paravirt_disable_iospace(void);paravirt_clobber (clbr), \ ##__VA_ARGS__ \ : "memory", "cc" extra_clbr); \ - __ret = (rettype )__eax; \ + __ret = (rettype)(__eax &PVOP_RETMASK (rettype)); \ } \ __ret; \ })
arch/x86/include/asm/processor.h +0 -2
@@ -588,8 +588,6 @@ static __always_inline void cpu_relax(void)rep_nop (); } -#define cpu_relax_lowlatency () cpu_relax() -/* Stop speculative execution and prefetching of modified code. */ static inline void sync_core (void ) {
arch/x86/include/asm/qspinlock.h +6 -0
@@ -32,6 +32,12 @@ static inline void queued_spin_unlock(struct qspinlock *lock) {pv_queued_spin_unlock (lock); } + +#define vcpu_is_preempted vcpu_is_preempted +static inline bool vcpu_is_preempted (int cpu) +{ +return pv_vcpu_is_preempted (cpu); +}#else static inline void queued_spin_unlock (struct qspinlock *lock) {
arch/x86/include/uapi/asm/kvm_para.h +3 -1
@@ -45,7 +45,9 @@ struct kvm_steal_time {__u64 steal;__u32 version;__u32 flags; -__u32 pad[12 ]; +__u8 preempted; +__u8 u8_pad[3 ]; +__u32 pad[11 ]; };#define KVM_STEAL_ALIGNMENT_BITS 5
arch/x86/kernel/kvm.c +13 -0
@@ -592,6 +592,14 @@ static void kvm_wait(u8 *ptr, u8 val)local_irq_restore (flags); } +__visible bool __kvm_vcpu_is_preempted(int cpu) +{ +struct kvm_steal_time *src = &per_cpu (steal_time, cpu); + +return !!src->preempted; +} +PV_CALLEE_SAVE_REGS_THUNK (__kvm_vcpu_is_preempted); +/* * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present. */
@@ -608,6 +616,11 @@ void __init kvm_spinlock_init(void) pv_lock_ops.queued_spin_unlock =PV_CALLEE_SAVE (__pv_queued_spin_unlock); pv_lock_ops.wait = kvm_wait; pv_lock_ops.kick = kvm_kick_cpu; + +if (kvm_para_has_feature (KVM_FEATURE_STEAL_TIME )) { + pv_lock_ops.vcpu_is_preempted = +PV_CALLEE_SAVE (__kvm_vcpu_is_preempted); + } }static __init int kvm_spinlock_init_jump (void )
arch/x86/kernel/paravirt-spinlocks.c +13 -1
@@ -12,7 +12,6 @@ __visible void __native_queued_spin_unlock(struct qspinlock *lock) {native_queued_spin_unlock (lock); } -PV_CALLEE_SAVE_REGS_THUNK (__native_queued_spin_unlock);bool pv_is_native_spin_unlock (void )
@@ -21,12 +20,25 @@ bool pv_is_native_spin_unlock(void) __raw_callee_save___native_queued_spin_unlock; } +__visible bool __native_vcpu_is_preempted(int cpu) +{ +return false; +} +PV_CALLEE_SAVE_REGS_THUNK (__native_vcpu_is_preempted); + +bool pv_is_native_vcpu_is_preempted (void ) +{ +return pv_lock_ops.vcpu_is_preempted.func == + __raw_callee_save___native_vcpu_is_preempted; +} +struct pv_lock_ops pv_lock_ops = { #ifdef CONFIG_SMP .queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, .queued_spin_unlock =PV_CALLEE_SAVE (__native_queued_spin_unlock), .wait = paravirt_nop, .kick = paravirt_nop, + .vcpu_is_preempted =PV_CALLEE_SAVE (__native_vcpu_is_preempted), #endif /* SMP */ };EXPORT_SYMBOL (pv_lock_ops);
arch/x86/kernel/paravirt_patch_32.c +12 -0
@@ -12,6 +12,7 @@ DEF_NATIVE(pv_cpu_ops, clts, "clts");#if defined(CONFIG_PARAVIRT_SPINLOCKS )DEF_NATIVE (pv_lock_ops, queued_spin_unlock,"movb $0, (%eax)" ); +DEF_NATIVE (pv_lock_ops, vcpu_is_preempted,"xor %eax, %eax" );#endif unsigned paravirt_patch_ident_32 (void *insnbuf,unsigned len)
@@ -27,6 +28,7 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len) }extern bool pv_is_native_spin_unlock (void ); +extern bool pv_is_native_vcpu_is_preempted (void );unsigned native_patch (u8 type,u16 clobbers,void *ibuf,unsigned long addr,unsigned len)
@@ -56,9 +58,19 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf, end = end_pv_lock_ops_queued_spin_unlock; goto patch_site; } + goto patch_default; + +case PARAVIRT_PATCH (pv_lock_ops.vcpu_is_preempted): +if (pv_is_native_vcpu_is_preempted ()) { + start = start_pv_lock_ops_vcpu_is_preempted; + end = end_pv_lock_ops_vcpu_is_preempted; + goto patch_site; + } + goto patch_default;#endif default : +patch_default: ret =paravirt_patch_default (type, clobbers, ibuf, addr, len);break ;
arch/x86/kernel/paravirt_patch_64.c +12 -0
@@ -21,6 +21,7 @@ DEF_NATIVE(, mov64, "mov %rdi, %rax");#if defined(CONFIG_PARAVIRT_SPINLOCKS )DEF_NATIVE (pv_lock_ops, queued_spin_unlock,"movb $0, (%rdi)" ); +DEF_NATIVE (pv_lock_ops, vcpu_is_preempted,"xor %rax, %rax" );#endif unsigned paravirt_patch_ident_32 (void *insnbuf,unsigned len)
@@ -36,6 +37,7 @@ unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len) }extern bool pv_is_native_spin_unlock (void ); +extern bool pv_is_native_vcpu_is_preempted (void );unsigned native_patch (u8 type,u16 clobbers,void *ibuf,unsigned long addr,unsigned len)
@@ -68,9 +70,19 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf, end = end_pv_lock_ops_queued_spin_unlock; goto patch_site; } + goto patch_default; + +case PARAVIRT_PATCH (pv_lock_ops.vcpu_is_preempted): +if (pv_is_native_vcpu_is_preempted ()) { + start = start_pv_lock_ops_vcpu_is_preempted; + end = end_pv_lock_ops_vcpu_is_preempted; + goto patch_site; + } + goto patch_default;#endif default : +patch_default: ret =paravirt_patch_default (type, clobbers, ibuf, addr, len);break ;
arch/x86/kvm/x86.c +16 -0
@@ -2071,6 +2071,8 @@ static void record_steal_time(struct kvm_vcpu *vcpu) &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))return ; + vcpu->arch.st.steal.preempted =0 ; +if (vcpu->arch.st.steal.version &1 ) vcpu->arch.st.steal.version +=1 ;/* first time write, random junk */
@@ -2826,8 +2828,22 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)kvm_make_request (KVM_REQ_STEAL_UPDATE , vcpu); } +static void kvm_steal_time_set_preempted (struct kvm_vcpu *vcpu) +{ +if (!(vcpu->arch.st.msr_val &KVM_MSR_ENABLED )) +return ; + + vcpu->arch.st.steal.preempted =1 ; + +kvm_write_guest_offset_cached (vcpu->kvm, &vcpu->arch.st.stime, + &vcpu->arch.st.steal.preempted, + offsetof(struct kvm_steal_time , preempted), +sizeof (vcpu->arch.st.steal.preempted)); +} +void kvm_arch_vcpu_put (struct kvm_vcpu *vcpu) { +kvm_steal_time_set_preempted (vcpu); kvm_x86_ops->vcpu_put (vcpu);kvm_put_guest_fpu (vcpu); vcpu->arch.last_host_tsc =rdtsc ();
arch/x86/um/asm/processor.h +0 -1
@@ -26,7 +26,6 @@ static inline void rep_nop(void) }#define cpu_relax () rep_nop() -#define cpu_relax_lowlatency () cpu_relax()#define task_pt_regs (t) (&(t)->thread.regs)
arch/x86/xen/spinlock.c +2 -0
@@ -114,6 +114,7 @@ void xen_uninit_lock_cpu(int cpu)per_cpu (irq_name, cpu) =NULL ; } +PV_CALLEE_SAVE_REGS_THUNK (xen_vcpu_stolen);/* * Our init of PV spinlocks is split in two init functions due to us
@@ -137,6 +138,7 @@ void __init xen_init_spinlocks(void) pv_lock_ops.queued_spin_unlock =PV_CALLEE_SAVE (__pv_queued_spin_unlock); pv_lock_ops.wait = xen_qlock_wait; pv_lock_ops.kick = xen_qlock_kick; + pv_lock_ops.vcpu_is_preempted =PV_CALLEE_SAVE (xen_vcpu_stolen); }/*
arch/xtensa/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -/* -* Pull in the generic implementation for the mutex fastpath. -* -* TODO: implement optimized primitives instead, or leave the generic -* implementation in place, or pick the atomic_xchg() based generic -* implementation. (see asm-generic/mutex-xchg.h for details) -*/ - -#include <asm-generic/mutex-dec.h>
arch/xtensa/include/asm/processor.h +0 -1
@@ -206,7 +206,6 @@ extern unsigned long get_wchan(struct task_struct *p);#define KSTK_ESP (tsk) (task_pt_regs(tsk)->areg[1])#define cpu_relax () barrier() -#define cpu_relax_lowlatency () cpu_relax()/* Special register access. */
drivers/gpu/drm/i915/i915_gem_request.c +1 -1
@@ -723,7 +723,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req,if (busywait_stop (timeout_us, cpu))break ; -cpu_relax_lowlatency (); +cpu_relax (); }while (!need_resched ());return false;
drivers/gpu/drm/i915/i915_gem_shrinker.c +11 -19
@@ -35,19 +35,6 @@#include "i915_drv.h" #include "i915_trace.h" -static bool mutex_is_locked_by (struct mutex *mutex,struct task_struct *task) -{ -if (!mutex_is_locked (mutex)) -return false; - -#if defined(CONFIG_DEBUG_MUTEXES ) || defined(CONFIG_MUTEX_SPIN_ON_OWNER ) -return mutex->owner == task; -#else -/* Since UP may be pre-empted, we cannot assume that we own the lock */ -return false; -#endif -} -static bool any_vma_pinned (struct drm_i915_gem_object *obj) {struct i915_vma *vma;
@@ -240,15 +227,20 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)static bool i915_gem_shrinker_lock (struct drm_device *dev,bool *unlock) { -if (!mutex_trylock (&dev->struct_mutex)) { -if (!mutex_is_locked_by (&dev->struct_mutex, current)) -return false; +switch (mutex_trylock_recursive (&dev->struct_mutex)) { +case MUTEX_TRYLOCK_FAILED : +return false; - *unlock = false; - }else +case MUTEX_TRYLOCK_SUCCESS : *unlock = true; +return true; -return true; +case MUTEX_TRYLOCK_RECURSIVE : + *unlock = false; +return true; + } + +BUG (); } static unsigned long
drivers/gpu/drm/msm/msm_gem_shrinker.c +9 -18
@@ -18,33 +18,24 @@#include "msm_drv.h" #include "msm_gem.h" -static bool mutex_is_locked_by (struct mutex *mutex,struct task_struct *task) +static bool msm_gem_shrinker_lock (struct drm_device *dev,bool *unlock) { -if (!mutex_is_locked (mutex)) +switch (mutex_trylock_recursive (&dev->struct_mutex)) { +case MUTEX_TRYLOCK_FAILED :return false; -#if defined(CONFIG_SMP ) || defined(CONFIG_DEBUG_MUTEXES ) -return mutex->owner == task; -#else -/* Since UP may be pre-empted, we cannot assume that we own the lock */ -return false; -#endif -} +case MUTEX_TRYLOCK_SUCCESS : + *unlock = true; +return true; -static bool msm_gem_shrinker_lock (struct drm_device *dev,bool *unlock) -{ -if (!mutex_trylock (&dev->struct_mutex)) { -if (!mutex_is_locked_by (&dev->struct_mutex, current)) -return false; +case MUTEX_TRYLOCK_RECURSIVE : *unlock = false; - }else { - *unlock = true; +return true; } -return true; +BUG (); } -static unsigned long msm_gem_shrinker_count (struct shrinker *shrinker,struct shrink_control *sc) {
drivers/vhost/net.c +2 -2
@@ -342,7 +342,7 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net, endtime =busy_clock () + vq->busyloop_timeout;while (vhost_can_busy_poll (vq->dev, endtime) &&vhost_vq_avail_empty (vq->dev, vq)) -cpu_relax_lowlatency (); +cpu_relax ();preempt_enable (); r =vhost_get_vq_desc (vq, vq->iov,ARRAY_SIZE (vq->iov), out_num, in_num,NULL ,NULL );
@@ -533,7 +533,7 @@ static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)while (vhost_can_busy_poll (&net->dev, endtime) && !sk_has_rx_data (sk) &&vhost_vq_avail_empty (&net->dev, vq)) -cpu_relax_lowlatency (); +cpu_relax ();preempt_enable ();
include/asm-generic/mutex-dec.h deleted +0 -88
88 lines not on this page · raw message
include/asm-generic/mutex-null.h deleted +0 -19
@@ -1,19 +0,0 @@ -/* -* include/asm-generic/mutex-null.h -* -* Generic implementation of the mutex fastpath, based on NOP :-) -* -* This is used by the mutex-debugging infrastructure, but it can also -* be used by architectures that (for whatever reason) want to use the -* spinlock based slowpath. -*/ -#ifndef _ASM_GENERIC_MUTEX_NULL_H -#define _ASM_GENERIC_MUTEX_NULL_H - -#define __mutex_fastpath_lock (count, fail_fn) fail_fn(count) -#define __mutex_fastpath_lock_retval (count) (-1) -#define __mutex_fastpath_unlock (count, fail_fn) fail_fn(count) -#define __mutex_fastpath_trylock (count, fail_fn) fail_fn(count) -#define __mutex_slowpath_needs_to_unlock () 1 - -#endif
include/asm-generic/mutex-xchg.h deleted +0 -120
120 lines not on this page · raw message
include/asm-generic/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@ -#ifndef __ASM_GENERIC_MUTEX_H -#define __ASM_GENERIC_MUTEX_H -/* -* Pull in the generic implementation for the mutex fastpath, -* which is a reasonable default on many architectures. -*/ - -#include <asm-generic/mutex-dec.h> -#endif /* __ASM_GENERIC_MUTEX_H */
include/linux/kvm_host.h +2 -0
@@ -645,6 +645,8 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, unsigned long len);int kvm_write_guest_cached (struct kvm *kvm,struct gfn_to_hva_cache *ghc,void *data,unsigned long len); +int kvm_write_guest_offset_cached (struct kvm *kvm,struct gfn_to_hva_cache *ghc, +void *data,int offset,unsigned long len);int kvm_gfn_to_hva_cache_init (struct kvm *kvm,struct gfn_to_hva_cache *ghc,gpa_t gpa,unsigned long len);int kvm_clear_guest_page (struct kvm *kvm,gfn_t gfn,int offset,int len);
include/linux/mutex-debug.h deleted +0 -24
@@ -1,24 +0,0 @@ -#ifndef __LINUX_MUTEX_DEBUG_H -#define __LINUX_MUTEX_DEBUG_H - -#include <linux/linkage.h> -#include <linux/lockdep.h> -#include <linux/debug_locks.h> - -/* -* Mutexes - debugging helpers: -*/ - -#define __DEBUG_MUTEX_INITIALIZER (lockname) \ - , .magic = &lockname - -#define mutex_init (mutex) \ -do { \ - static struct lock_class_key __key; \ - \ - __mutex_init((mutex), #mutex, &__key); \ -} while (0) - -extern void mutex_destroy (struct mutex *lock); - -#endif
include/linux/mutex.h +61 -16
130 lines not on this page · raw message
include/linux/sched.h +18 -2
@@ -989,7 +989,7 @@ enum cpu_idle_type {* already in a wake queue, the wakeup will happen soon and the second * waker can just skip it. * -* The WAKE_Q macro declares and initializes the list head. +* The DEFINE_WAKE_Q macro declares and initializes the list head. * wake_up_q() does NOT reinitialize the list; it's expected to be * called near the end of a function, where the fact that the queue is * not used again will be easy to see by inspection.
@@ -1009,7 +1009,7 @@ struct wake_q_head {#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) -#define WAKE_Q (name) \ +#define DEFINE_WAKE_Q (name) \ struct wake_q_head name = { WAKE_Q_TAIL, &name.first } extern void wake_q_add(struct wake_q_head *head,
@@ -2444,6 +2444,10 @@ static inline void calc_load_enter_idle(void) { }static inline void calc_load_exit_idle (void ) { }#endif /* CONFIG_NO_HZ_COMMON */ +#ifndef cpu_relax_yield +#define cpu_relax_yield () cpu_relax() +#endif +/* * Do not use outside of architecture code which knows its limitations. *
@@ -3508,6 +3512,18 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)#endif /* CONFIG_SMP */ +/* +* In order to reduce various lock holder preemption latencies provide an +* interface to see if a vCPU is currently running or not. +* +* This allows us to terminate optimistic spin loops and block, analogous to +* the native optimistic spin heuristic of testing if the lock owner task is +* running or not. +*/ +#ifndef vcpu_is_preempted +# define vcpu_is_preempted (cpu) false +#endif +extern long sched_setaffinity (pid_t pid,const struct cpumask *new_mask);extern long sched_getaffinity (pid_t pid,struct cpumask *mask);
include/linux/ww_mutex.h +1 -1
@@ -120,7 +120,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, struct ww_class *ww_class) { ctx->task = current; - ctx->stamp =atomic_long_inc_return (&ww_class->stamp); + ctx->stamp =atomic_long_inc_return_relaxed (&ww_class->stamp); ctx->acquired =0 ;#ifdef CONFIG_DEBUG_MUTEXES ctx->ww_class = ww_class;
ipc/mqueue.c +2 -2
@@ -967,7 +967,7 @@ SYSCALL_DEFINE5(mq_timedsend, mqd_t, mqdes, const char __user *, u_msg_ptr,struct timespec ts;struct posix_msg_tree_node *new_leaf =NULL ;int ret =0 ; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);if (u_abs_timeout) {int res =prepare_timeout (u_abs_timeout, &expires, &ts);
@@ -1151,7 +1151,7 @@ SYSCALL_DEFINE5(mq_timedreceive, mqd_t, mqdes, char __user *, u_msg_ptr, msg_ptr = wait.msg; } }else { -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q); msg_ptr =msg_get (info);
ipc/msg.c +4 -4
@@ -235,7 +235,7 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) {struct msg_msg *msg, *t; struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm); -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);expunge_all (msq, -EIDRM , &wake_q);ss_wakeup (msq, &wake_q, true);
@@ -397,7 +397,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, goto out_up;case IPC_SET : { -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);if (msqid64.msg_qbytes > ns->msg_ctlmnb && !capable (CAP_SYS_RESOURCE )) {
@@ -634,7 +634,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,struct msg_msg *msg;int err;struct ipc_namespace *ns; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q); ns = current->nsproxy->ipc_ns;
@@ -850,7 +850,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgflstruct msg_queue *msq;struct ipc_namespace *ns;struct msg_msg *msg, *copy =NULL ; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q); ns = current->nsproxy->ipc_ns;
kernel/Kconfig.locks +1 -1
@@ -225,7 +225,7 @@ config ARCH_SUPPORTS_ATOMIC_RMW config MUTEX_SPIN_ON_OWNER def_bool y - depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW + depends on SMP && ARCH_SUPPORTS_ATOMIC_RMW config RWSEM_SPIN_ON_OWNER def_bool y
kernel/futex.c +4 -4
@@ -1298,7 +1298,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,struct task_struct *new_owner;struct futex_pi_state *pi_state = this->pi_state;u32 uninitialized_var (curval ), newval; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);bool deboost;int ret =0 ;
@@ -1415,7 +1415,7 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)struct futex_q *this, *next;union futex_key key =FUTEX_KEY_INIT ;int ret; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);if (!bitset)return -EINVAL ;
@@ -1469,7 +1469,7 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,struct futex_hash_bucket *hb1, *hb2;struct futex_q *this, *next;int ret, op_ret; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q); retry: ret =get_futex_key (uaddr1, flags &FLAGS_SHARED , &key1,VERIFY_READ );
@@ -1708,7 +1708,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags,struct futex_pi_state *pi_state =NULL ;struct futex_hash_bucket *hb1, *hb2;struct futex_q *this, *next; -WAKE_Q (wake_q); +DEFINE_WAKE_Q (wake_q);if (requeue_pi) {/*
kernel/locking/lockdep.c +5 -5
@@ -840,9 +840,9 @@ static struct lock_list *alloc_list_entry(void)/* * Add a new dependency to the head of the list: */ -static int add_lock_to_list (struct lock_class *class,struct lock_class *this, -struct list_head *head,unsigned long ip, -int distance,struct stack_trace *trace) +static int add_lock_to_list (struct lock_class *this,struct list_head *head, +unsigned long ip,int distance, +struct stack_trace *trace) {struct lock_list *entry;/*
@@ -1868,14 +1868,14 @@ check_prev_add(struct task_struct *curr, struct held_lock *prev,* Ok, all validations passed, add the new lock * to the previous lock's dependency list: */ - ret =add_lock_to_list (hlock_class (prev),hlock_class (next), + ret =add_lock_to_list (hlock_class (next), &hlock_class (prev)->locks_after, next->acquire_ip, distance, &trace);if (!ret)return 0 ; - ret =add_lock_to_list (hlock_class (next),hlock_class (prev), + ret =add_lock_to_list (hlock_class (prev), &hlock_class (next)->locks_before, next->acquire_ip, distance, &trace);if (!ret)
kernel/locking/mcs_spinlock.h +2 -2
@@ -28,7 +28,7 @@ struct mcs_spinlock {#define arch_mcs_spin_lock_contended (l) \ do { \ while (!(smp_load_acquire(l))) \ - cpu_relax_lowlatency(); \ + cpu_relax(); \ } while (0)#endif
@@ -108,7 +108,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)return ;/* Wait until the next pointer is set */ while (!(next =READ_ONCE (node->next))) -cpu_relax_lowlatency (); +cpu_relax (); }/* Pass lock to next waiter. */
kernel/locking/mutex-debug.c +0 -13
@@ -73,21 +73,8 @@ void debug_mutex_unlock(struct mutex *lock) {if (likely (debug_locks)) {DEBUG_LOCKS_WARN_ON (lock->magic != lock); - -if (!lock->owner) -DEBUG_LOCKS_WARN_ON (!lock->owner); -else -DEBUG_LOCKS_WARN_ON (lock->owner != current); -DEBUG_LOCKS_WARN_ON (!lock->wait_list.prev && !lock->wait_list.next); } - -/* -* __mutex_slowpath_needs_to_unlock() is explicitly 0 for debug -* mutexes so that we can do it here after we've verified state. -*/ -mutex_clear_owner (lock); -atomic_set (&lock->count,1 ); } void debug_mutex_init(struct mutex *lock, const char *name,
kernel/locking/mutex-debug.h +0 -10
@@ -27,16 +27,6 @@ extern void debug_mutex_unlock(struct mutex *lock);extern void debug_mutex_init (struct mutex *lock,const char *name,struct lock_class_key *key); -static inline void mutex_set_owner (struct mutex *lock) -{ -WRITE_ONCE (lock->owner, current); -} - -static inline void mutex_clear_owner (struct mutex *lock) -{ -WRITE_ONCE (lock->owner,NULL ); -} -#define spin_lock_mutex (lock, flags) \ do { \ struct mutex *l = container_of(lock, struct mutex, wait_lock); \
kernel/locking/mutex.c +340 -248
791 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Part of the diff is not shown here. View the raw message.