Skip to contentSkip to conversation

locking/mutex: Rewrite basic mutex

PATCH In review Peter Zijlstra 2 participants 14 messages Sep 5, 2016 – Sep 14, 2016 2 messages not in the archive

Subsystems (25)
DRM drivers Arm (32-bit) x86 PowerPC +21 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

cover locking/mutex: Rewrite basic mutex

Hi,

New version, with improved Changelogs and bits. This version has survived aim7 loads on my 2 socket machine, I've yet to try the 4 socket. But over-all it seem to behave as expected.

That is, all except the last patch, which I just wrote and hasn't even seen a compiler up close.

---
 arch/alpha/include/asm/mutex.h           |   9 -
 arch/arc/include/asm/mutex.h             |  18 -
 arch/arm/include/asm/mutex.h             |  21 --
 arch/arm64/include/asm/Kbuild            |   1 -
 arch/avr32/include/asm/mutex.h           |   9 -
 arch/blackfin/include/asm/Kbuild         |   1 -
 arch/c6x/include/asm/mutex.h             |   6 -
 arch/cris/include/asm/mutex.h            |   9 -
 arch/frv/include/asm/mutex.h             |   9 -
 arch/h8300/include/asm/mutex.h           |   9 -
 arch/hexagon/include/asm/mutex.h         |   8 -
 arch/ia64/include/asm/mutex.h            |  90 -----
 arch/m32r/include/asm/mutex.h            |   9 -
 arch/m68k/include/asm/Kbuild             |   1 -
 arch/metag/include/asm/Kbuild            |   1 -
 arch/microblaze/include/asm/mutex.h      |   1 -
 arch/mips/include/asm/Kbuild             |   1 -
 arch/mn10300/include/asm/mutex.h         |  16 -
 arch/nios2/include/asm/mutex.h           |   1 -
 arch/openrisc/include/asm/mutex.h        |  27 --
 arch/parisc/include/asm/Kbuild           |   1 -
 arch/powerpc/include/asm/mutex.h         | 132 -------
 arch/s390/include/asm/mutex.h            |   9 -
 arch/score/include/asm/mutex.h           |   6 -
 arch/sh/include/asm/atomic-llsc.h        |   2 +-
 arch/sh/include/asm/mutex-llsc.h         | 109 ------
 arch/sh/include/asm/mutex.h              |  12 -
 arch/sparc/include/asm/Kbuild            |   1 -
 arch/tile/include/asm/Kbuild             |   1 -
 arch/um/include/asm/Kbuild               |   1 -
 arch/unicore32/include/asm/mutex.h       |  20 --
 arch/x86/include/asm/mutex.h             |   5 -
 arch/x86/include/asm/mutex_32.h          | 110 ------
 arch/x86/include/asm/mutex_64.h          | 127 -------
 arch/xtensa/include/asm/mutex.h          |   9 -
 drivers/gpu/drm/i915/i915_gem_shrinker.c |  26 +-
 drivers/gpu/drm/msm/msm_gem_shrinker.c   |  23 +-
 include/asm-generic/mutex-dec.h          |  88 -----
 include/asm-generic/mutex-null.h         |  19 -
 include/asm-generic/mutex-xchg.h         | 120 -------
 include/asm-generic/mutex.h              |   9 -
 include/linux/mutex-debug.h              |  24 --
 include/linux/mutex.h                    |  46 ++-
 kernel/Kconfig.locks                     |   2 +-
 kernel/locking/mutex-debug.c             |  13 -
 kernel/locking/mutex-debug.h             |  10 -
 kernel/locking/mutex.c                   | 589 ++++++++++++++++++-------------
 kernel/locking/mutex.h                   |  26 --
 kernel/sched/core.c                      |   2 +-
 49 files changed, 382 insertions(+), 1407 deletions(-)
author maintainer
#1 lore raw

PATCH 1/10 locking/drm: Kill mutex trickery

Poking at lock internals is not cool. Since I'm going to change the implementation this will break, take it out.

Cc Daniel Vetter <[email protected]>
Cc Chris Wilson <[email protected]>
Cc Rob Clark <[email protected]>
---
 drivers/gpu/drm/i915/i915_gem_shrinker.c |   26 +++-----------------------
 drivers/gpu/drm/msm/msm_gem_shrinker.c   |   23 +++--------------------
 2 files changed, 6 insertions(+), 43 deletions(-)
drivers/gpu/drm/i915/i915_gem_shrinker.c +3 -23
@@ -35,19 +35,6 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 
-static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
-{
-	if (!mutex_is_locked(mutex))
-		return false;
-
-#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_MUTEX_SPIN_ON_OWNER)
-	return mutex->owner == task;
-#else
-	/* Since UP may be pre-empted, we cannot assume that we own the lock */
-	return false;
-#endif
-}
-
 static int num_vma_bound(struct drm_i915_gem_object *obj)
 {
 	struct i915_vma *vma;
@@ -238,17 +225,10 @@ unsigned long i915_gem_shrink_all(struct
 
 static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
 {
-	if (!mutex_trylock(&dev->struct_mutex)) {
-		if (!mutex_is_locked_by(&dev->struct_mutex, current))
-			return false;
-
-		if (to_i915(dev)->mm.shrinker_no_lock_stealing)
-			return false;
-
-		*unlock = false;
-	} else
-		*unlock = true;
+	if (!mutex_trylock(&dev->struct_mutex))
+		return false;
 
+	*unlock = true;
 	return true;
 }
 
drivers/gpu/drm/msm/msm_gem_shrinker.c +3 -20
@@ -18,29 +18,12 @@
 #include "msm_drv.h"
 #include "msm_gem.h"
 
-static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
-{
-	if (!mutex_is_locked(mutex))
-		return false;
-
-#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
-	return mutex->owner == task;
-#else
-	/* Since UP may be pre-empted, we cannot assume that we own the lock */
-	return false;
-#endif
-}
-
 static bool msm_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
 {
-	if (!mutex_trylock(&dev->struct_mutex)) {
-		if (!mutex_is_locked_by(&dev->struct_mutex, current))
-			return false;
-		*unlock = false;
-	} else {
-		*unlock = true;
-	}
+	if (!mutex_trylock(&dev->struct_mutex))
+		return false;
 
+	*unlock = true;
 	return true;
 }
 
author maintainer
#2 lore raw

PATCH 2/10 sh/atomic: Fix ATOMIC_FETCH_OP()

In reply to Peter Zijlstra

We cannot use the "z" constraint twice, since its a single register (r0). Change the one not used by movli.l/movco.l to "r".

---
 arch/sh/include/asm/atomic-llsc.h |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/sh/include/asm/atomic-llsc.h +1 -1
@@ -60,7 +60,7 @@ static inline int atomic_fetch_##op(int
 "	movco.l	%0, @%3					\n"		\
 "	bf	1b					\n"		\
 "	synco						\n"		\
-	: "=&z" (temp), "=&z" (res)					\
+	: "=&z" (temp), "=&r" (res)					\
 	: "r" (i), "r" (&v->counter)					\
 	: "t");								\
 									\
author maintainer
#3 lore raw

PATCH 3/10 locking/mutex: Rework mutex::owner

In reply to Peter Zijlstra

The current mutex implementation has an atomic lock word and a non-atomic owner field.

This disparity leads to a number of issues with the current mutex code as it means that we can have a locked mutex without an explicit owner (because the owner field has not been set, or already cleared).

This leads to a number of weird corner cases, esp. between the optimistic spinning and debug code. Where the optimistic spinning code needs the owner field updated inside the lock region, the debug code is more relaxed because the whole lock is serialized by the wait_lock.

Also, the spinning code itself has a few corner cases where we need to deal with a held lock without an owner field.

Furthermore, it becomes even more of a problem when trying to fix starvation cases in the current code. We end up stacking special case on special case.

To solve this rework the basic mutex implementation to be a single atomic word that contains the owner and uses the low bits for extra state.

This matches how PI futexes and rt_mutex already work. By having the owner an integral part of the lock state a lot of the problems dissapear and we get a better option to deal with starvation cases, direct owner handoff.

Changing the basic mutex does however invalidate all the arch specific mutex code; this patch leaves that unused in-place, a later patch will remove that.

---
 include/linux/mutex-debug.h  |   24 --
 include/linux/mutex.h        |   46 +++--
 kernel/locking/mutex-debug.c |   13 -
 kernel/locking/mutex-debug.h |   10 -
 kernel/locking/mutex.c       |  371 ++++++++++++++++++-------------------------
 kernel/locking/mutex.h       |   26 ---
 kernel/sched/core.c          |    2 
 7 files changed, 187 insertions(+), 305 deletions(-)
include/linux/mutex-debug.h deleted +0 -24
@@ -1,24 +0,0 @@
-#ifndef __LINUX_MUTEX_DEBUG_H
-#define __LINUX_MUTEX_DEBUG_H
-
-#include <linux/linkage.h>
-#include <linux/lockdep.h>
-#include <linux/debug_locks.h>
-
-/*
- * Mutexes - debugging helpers:
- */
-
-#define __DEBUG_MUTEX_INITIALIZER(lockname)				\
-	, .magic = &lockname
-
-#define mutex_init(mutex)						\
-do {									\
-	static struct lock_class_key __key;				\
-									\
-	__mutex_init((mutex), #mutex, &__key);				\
-} while (0)
-
-extern void mutex_destroy(struct mutex *lock);
-
-#endif
include/linux/mutex.h +30 -16

95 lines not on this page · raw message

kernel/locking/mutex-debug.c +0 -13
@@ -73,21 +73,8 @@ void debug_mutex_unlock(struct mutex *lo
 {
 	if (likely(debug_locks)) {
 		DEBUG_LOCKS_WARN_ON(lock->magic != lock);
-
-		if (!lock->owner)
-			DEBUG_LOCKS_WARN_ON(!lock->owner);
-		else
-			DEBUG_LOCKS_WARN_ON(lock->owner != current);
-
 		DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
 	}
-
-	/*
-	 * __mutex_slowpath_needs_to_unlock() is explicitly 0 for debug
-	 * mutexes so that we can do it here after we've verified state.
-	 */
-	mutex_clear_owner(lock);
-	atomic_set(&lock->count, 1);
 }
 
 void debug_mutex_init(struct mutex *lock, const char *name,
kernel/locking/mutex-debug.h +0 -10
@@ -27,16 +27,6 @@ extern void debug_mutex_unlock(struct mu
 extern void debug_mutex_init(struct mutex *lock, const char *name,
 			     struct lock_class_key *key);
 
-static inline void mutex_set_owner(struct mutex *lock)
-{
-	WRITE_ONCE(lock->owner, current);
-}
-
-static inline void mutex_clear_owner(struct mutex *lock)
-{
-	WRITE_ONCE(lock->owner, NULL);
-}
-
 #define spin_lock_mutex(lock, flags)			\
 	do {						\
 		struct mutex *l = container_of(lock, struct mutex, wait_lock); \
kernel/locking/mutex.c +156 -215

601 lines not on this page · raw message

kernel/locking/mutex.h +0 -26
@@ -16,32 +16,6 @@
 #define mutex_remove_waiter(lock, waiter, task) \
 		__list_del((waiter)->list.prev, (waiter)->list.next)
 
-#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
-/*
- * The mutex owner can get read and written to locklessly.
- * We should use WRITE_ONCE when writing the owner value to
- * avoid store tearing, otherwise, a thread could potentially
- * read a partially written and incomplete owner value.
- */
-static inline void mutex_set_owner(struct mutex *lock)
-{
-	WRITE_ONCE(lock->owner, current);
-}
-
-static inline void mutex_clear_owner(struct mutex *lock)
-{
-	WRITE_ONCE(lock->owner, NULL);
-}
-#else
-static inline void mutex_set_owner(struct mutex *lock)
-{
-}
-
-static inline void mutex_clear_owner(struct mutex *lock)
-{
-}
-#endif
-
 #define debug_mutex_wake_waiter(lock, waiter)		do { } while (0)
 #define debug_mutex_free_waiter(waiter)			do { } while (0)
 #define debug_mutex_add_waiter(lock, waiter, ti)	do { } while (0)
kernel/sched/core.c +1 -1
@@ -75,11 +75,11 @@
 #include <linux/compiler.h>
 #include <linux/frame.h>
 #include <linux/prefetch.h>
+#include <linux/mutex.h>
 
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
 #include <asm/irq_regs.h>
-#include <asm/mutex.h>
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #endif
author maintainer
#4 lore raw

PATCH 4/10 locking/mutex: Kill arch specific code

In reply to Peter Zijlstra

Its all generic atomic_long_t stuff now.

---
 arch/alpha/include/asm/mutex.h      |    9 --
 arch/arc/include/asm/mutex.h        |   18 ----
 arch/arm/include/asm/mutex.h        |   21 -----
 arch/arm64/include/asm/Kbuild       |    1 
 arch/avr32/include/asm/mutex.h      |    9 --
 arch/blackfin/include/asm/Kbuild    |    1 
 arch/c6x/include/asm/mutex.h        |    6 -
 arch/cris/include/asm/mutex.h       |    9 --
 arch/frv/include/asm/mutex.h        |    9 --
 arch/h8300/include/asm/mutex.h      |    9 --
 arch/hexagon/include/asm/mutex.h    |    8 --
 arch/ia64/include/asm/mutex.h       |   90 ------------------------
 arch/m32r/include/asm/mutex.h       |    9 --
 arch/m68k/include/asm/Kbuild        |    1 
 arch/metag/include/asm/Kbuild       |    1 
 arch/microblaze/include/asm/mutex.h |    1 
 arch/mips/include/asm/Kbuild        |    1 
 arch/mn10300/include/asm/mutex.h    |   16 ----
 arch/nios2/include/asm/mutex.h      |    1 
 arch/openrisc/include/asm/mutex.h   |   27 -------
 arch/parisc/include/asm/Kbuild      |    1 
 arch/powerpc/include/asm/mutex.h    |  132 ------------------------------------
 arch/s390/include/asm/mutex.h       |    9 --
 arch/score/include/asm/mutex.h      |    6 -
 arch/sh/include/asm/mutex-llsc.h    |  109 -----------------------------
 arch/sh/include/asm/mutex.h         |   12 ---
 arch/sparc/include/asm/Kbuild       |    1 
 arch/tile/include/asm/Kbuild        |    1 
 arch/um/include/asm/Kbuild          |    1 
 arch/unicore32/include/asm/mutex.h  |   20 -----
 arch/x86/include/asm/mutex.h        |    5 -
 arch/x86/include/asm/mutex_32.h     |  110 ------------------------------
 arch/x86/include/asm/mutex_64.h     |  127 ----------------------------------
 arch/xtensa/include/asm/mutex.h     |    9 --
 include/asm-generic/mutex-dec.h     |   88 ------------------------
 include/asm-generic/mutex-null.h    |   19 -----
 include/asm-generic/mutex-xchg.h    |  120 --------------------------------
 include/asm-generic/mutex.h         |    9 --
 38 files changed, 1026 deletions(-)
arch/alpha/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/arc/include/asm/mutex.h deleted +0 -18
@@ -1,18 +0,0 @@
-/*
- * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-/*
- * xchg() based mutex fast path maintains a state of 0 or 1, as opposed to
- * atomic dec based which can "count" any number of lock contenders.
- * This ideally needs to be fixed in core, but for now switching to dec ver.
- */
-#if defined(CONFIG_SMP) && (CONFIG_NR_CPUS > 2)
-#include <asm-generic/mutex-dec.h>
-#else
-#include <asm-generic/mutex-xchg.h>
-#endif
arch/arm/include/asm/mutex.h deleted +0 -21
@@ -1,21 +0,0 @@
-/*
- * arch/arm/include/asm/mutex.h
- *
- * ARM optimized mutex locking primitives
- *
- * Please look into asm-generic/mutex-xchg.h for a formal definition.
- */
-#ifndef _ASM_MUTEX_H
-#define _ASM_MUTEX_H
-/*
- * On pre-ARMv6 hardware this results in a swp-based implementation,
- * which is the most efficient. For ARMv6+, we have exclusive memory
- * accessors and use atomic_dec to avoid the extra xchg operations
- * on the locking slowpaths.
- */
-#if __LINUX_ARM_ARCH__ < 6
-#include <asm-generic/mutex-xchg.h>
-#else
-#include <asm-generic/mutex-dec.h>
-#endif
-#endif	/* _ASM_MUTEX_H */
arch/arm64/include/asm/Kbuild +0 -1
@@ -26,7 +26,6 @@ generic-y += mm-arch-hooks.h
 generic-y += mman.h
 generic-y += msgbuf.h
 generic-y += msi.h
-generic-y += mutex.h
 generic-y += pci.h
 generic-y += poll.h
 generic-y += preempt.h
arch/avr32/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/blackfin/include/asm/Kbuild +0 -1
@@ -24,7 +24,6 @@ generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
 generic-y += mman.h
 generic-y += msgbuf.h
-generic-y += mutex.h
 generic-y += param.h
 generic-y += percpu.h
 generic-y += pgalloc.h
arch/c6x/include/asm/mutex.h deleted +0 -6
@@ -1,6 +0,0 @@
-#ifndef _ASM_C6X_MUTEX_H
-#define _ASM_C6X_MUTEX_H
-
-#include <asm-generic/mutex-null.h>
-
-#endif /* _ASM_C6X_MUTEX_H */
arch/cris/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/frv/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/h8300/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/hexagon/include/asm/mutex.h deleted +0 -8
@@ -1,8 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-#include <asm-generic/mutex-xchg.h>
arch/ia64/include/asm/mutex.h deleted +0 -90

90 lines not on this page · raw message

arch/m32r/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/m68k/include/asm/Kbuild +0 -1
@@ -20,7 +20,6 @@ generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
 generic-y += mman.h
-generic-y += mutex.h
 generic-y += percpu.h
 generic-y += preempt.h
 generic-y += resource.h
arch/metag/include/asm/Kbuild +0 -1
@@ -27,7 +27,6 @@ generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
 generic-y += msgbuf.h
-generic-y += mutex.h
 generic-y += param.h
 generic-y += pci.h
 generic-y += percpu.h
arch/microblaze/include/asm/mutex.h deleted +0 -1
@@ -1 +0,0 @@
-#include <asm-generic/mutex-dec.h>
arch/mips/include/asm/Kbuild +0 -1
@@ -9,7 +9,6 @@ generic-y += irq_work.h
 generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
-generic-y += mutex.h
 generic-y += parport.h
 generic-y += percpu.h
 generic-y += preempt.h
arch/mn10300/include/asm/mutex.h deleted +0 -16
@@ -1,16 +0,0 @@
-/* MN10300 Mutex fastpath
- *
- * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
- * Written by David Howells ([email protected])
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
- * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
- *
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-#include <asm-generic/mutex-null.h>
arch/nios2/include/asm/mutex.h deleted +0 -1
@@ -1 +0,0 @@
-#include <asm-generic/mutex-dec.h>
arch/openrisc/include/asm/mutex.h deleted +0 -27
@@ -1,27 +0,0 @@
-/*
- * OpenRISC Linux
- *
- * Linux architectural port borrowing liberally from similar works of
- * others.  All original copyrights apply as per the original source
- * declaration.
- *
- * OpenRISC implementation:
- * Copyright (C) 2003 Matjaz Breskvar <[email protected]>
- * Copyright (C) 2010-2011 Jonas Bonn <[email protected]>
- * et al.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/parisc/include/asm/Kbuild +0 -1
@@ -16,7 +16,6 @@ generic-y += local.h
 generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
-generic-y += mutex.h
 generic-y += param.h
 generic-y += percpu.h
 generic-y += poll.h
arch/powerpc/include/asm/mutex.h deleted +0 -132

132 lines not on this page · raw message

arch/s390/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
arch/score/include/asm/mutex.h deleted +0 -6
@@ -1,6 +0,0 @@
-#ifndef _ASM_SCORE_MUTEX_H
-#define _ASM_SCORE_MUTEX_H
-
-#include <asm-generic/mutex-dec.h>
-
-#endif /* _ASM_SCORE_MUTEX_H */
arch/sh/include/asm/mutex-llsc.h deleted +0 -109

109 lines not on this page · raw message

arch/sh/include/asm/mutex.h deleted +0 -12
@@ -1,12 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-#if defined(CONFIG_CPU_SH4A)
-#include <asm/mutex-llsc.h>
-#else
-#include <asm-generic/mutex-dec.h>
-#endif
arch/sparc/include/asm/Kbuild +0 -1
@@ -14,7 +14,6 @@ generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
 generic-y += module.h
-generic-y += mutex.h
 generic-y += preempt.h
 generic-y += rwsem.h
 generic-y += serial.h
arch/tile/include/asm/Kbuild +0 -1
@@ -21,7 +21,6 @@ generic-y += local64.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
 generic-y += msgbuf.h
-generic-y += mutex.h
 generic-y += param.h
 generic-y += parport.h
 generic-y += poll.h
arch/um/include/asm/Kbuild +0 -1
@@ -17,7 +17,6 @@ generic-y += irq_work.h
 generic-y += kdebug.h
 generic-y += mcs_spinlock.h
 generic-y += mm-arch-hooks.h
-generic-y += mutex.h
 generic-y += param.h
 generic-y += pci.h
 generic-y += percpu.h
arch/unicore32/include/asm/mutex.h deleted +0 -20
@@ -1,20 +0,0 @@
-/*
- * linux/arch/unicore32/include/asm/mutex.h
- *
- * Code specific to PKUnity SoC and UniCore ISA
- *
- * Copyright (C) 2001-2010 GUAN Xue-tao
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * UniCore optimized mutex locking primitives
- *
- * Please look into asm-generic/mutex-xchg.h for a formal definition.
- */
-#ifndef __UNICORE_MUTEX_H__
-#define __UNICORE_MUTEX_H__
-
-# include <asm-generic/mutex-xchg.h>
-#endif
arch/x86/include/asm/mutex.h deleted +0 -5
@@ -1,5 +0,0 @@
-#ifdef CONFIG_X86_32
-# include <asm/mutex_32.h>
-#else
-# include <asm/mutex_64.h>
-#endif
arch/x86/include/asm/mutex_32.h deleted +0 -110

110 lines not on this page · raw message

arch/x86/include/asm/mutex_64.h deleted +0 -127

127 lines not on this page · raw message

arch/xtensa/include/asm/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-/*
- * Pull in the generic implementation for the mutex fastpath.
- *
- * TODO: implement optimized primitives instead, or leave the generic
- * implementation in place, or pick the atomic_xchg() based generic
- * implementation. (see asm-generic/mutex-xchg.h for details)
- */
-
-#include <asm-generic/mutex-dec.h>
include/asm-generic/mutex-dec.h deleted +0 -88

88 lines not on this page · raw message

include/asm-generic/mutex-null.h deleted +0 -19
@@ -1,19 +0,0 @@
-/*
- * include/asm-generic/mutex-null.h
- *
- * Generic implementation of the mutex fastpath, based on NOP :-)
- *
- * This is used by the mutex-debugging infrastructure, but it can also
- * be used by architectures that (for whatever reason) want to use the
- * spinlock based slowpath.
- */
-#ifndef _ASM_GENERIC_MUTEX_NULL_H
-#define _ASM_GENERIC_MUTEX_NULL_H
-
-#define __mutex_fastpath_lock(count, fail_fn)		fail_fn(count)
-#define __mutex_fastpath_lock_retval(count)		(-1)
-#define __mutex_fastpath_unlock(count, fail_fn)		fail_fn(count)
-#define __mutex_fastpath_trylock(count, fail_fn)	fail_fn(count)
-#define __mutex_slowpath_needs_to_unlock()		1
-
-#endif
include/asm-generic/mutex-xchg.h deleted +0 -120

120 lines not on this page · raw message

include/asm-generic/mutex.h deleted +0 -9
@@ -1,9 +0,0 @@
-#ifndef __ASM_GENERIC_MUTEX_H
-#define __ASM_GENERIC_MUTEX_H
-/*
- * Pull in the generic implementation for the mutex fastpath,
- * which is a reasonable default on many architectures.
- */
-
-#include <asm-generic/mutex-dec.h>
-#endif /* __ASM_GENERIC_MUTEX_H */
author maintainer
#5 lore raw

PATCH 5/10 locking/mutex: Allow MUTEX_SPIN_ON_OWNER when DEBUG_MUTEXES

In reply to Peter Zijlstra

Now that mutex::count and mutex::owner are the same field, we can allow SPIN_ON_OWNER while DEBUG_MUTEX.

---
 kernel/Kconfig.locks |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/Kconfig.locks +1 -1
@@ -225,7 +225,7 @@ config ARCH_SUPPORTS_ATOMIC_RMW
 
 config MUTEX_SPIN_ON_OWNER
 	def_bool y
-	depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW
+	depends on SMP && ARCH_SUPPORTS_ATOMIC_RMW
 
 config RWSEM_SPIN_ON_OWNER
        def_bool y
author maintainer
#6 lore raw

PATCH 6/10 locking/mutex: Add lock handoff to avoid starvation

In reply to Peter Zijlstra

Implement lock handoff to avoid lock starvation.

Lock starvation is possible because mutex_lock() allows lock stealing, where a running (or optimistic spinning) task beats the woken waiter to the acquire.

Lock stealing is an important performance optimization because waiting for a waiter to wake up and get runtime can take a significant time, during which everyboy would stall on the lock.

The down-side is of course that it allows for starvation.

This patch has the waiter requesting a handoff if it fails to acquire the lock upon waking. This re-introduces some of the wait time, because once we do a handoff we have to wait for the waiter to wake up again.

A future patch will add a round of optimistic spinning to attempt to alleviate this penalty, but if that turns out to not be enough, we can add a counter and only request handoff after multiple failed wakeups.

There are a few tricky implementation details:

- accepting a handoff must only be done in the wait-loop. Since the handoff condition is owner == current, it can easily cause recursive locking trouble.

- accepting the handoff must be careful to provide the ACQUIRE semantics.

- having the HANDOFF bit set on unlock requires care, we must not clear the owner.

- we must be careful to not leave HANDOFF set after we've acquired the lock. An alternative approach is to combine the bit set and a trylock in one cmpxchg loop, which ensures we either steal/acquire the lock or unlock must observe and clear HANDOFF.

---
 kernel/locking/mutex.c |  139 +++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 117 insertions(+), 22 deletions(-)
kernel/locking/mutex.c +117 -22
@@ -54,8 +54,10 @@ EXPORT_SYMBOL(__mutex_init);
  * bits to store extra state.
  *
  * Bit0 indicates a non-empty waiter list; unlock must issue a wakeup.
+ * Bit1 indicates unlock needs to hand the lock to the top-waiter
  */
 #define MUTEX_FLAG_WAITERS	0x01
+#define MUTEX_FLAG_HANDOFF	0x02
 
 #define MUTEX_FLAGS		0x03
 
@@ -71,8 +73,13 @@ static inline unsigned long __owner_flag
 
 /*
  * Actual trylock that will work on any unlocked state.
+ *
+ * When setting the owner field, we must preserve the low flag bits.
+ *
+ * Be careful with @handoff, only set that in a wait-loop (where you set
+ * HANDOFF) to avoid recursive lock attempts.
  */
-static inline bool __mutex_trylock(struct mutex *lock)
+static inline bool __mutex_trylock(struct mutex *lock, const bool handoff)
 {
 	unsigned long owner, curr = (unsigned long)current;
 
@@ -80,8 +87,24 @@ static inline bool __mutex_trylock(struc
 	for (;;) { /* must loop, can race against a flag */
 		unsigned long old;
 
-		if (__owner_task(owner))
+		if (__owner_task(owner)) {
+			if (handoff && unlikely(__owner_task(owner) == current)) {
+				/*
+				 * Provide ACQUIRE semantics for the lock-handoff.
+				 *
+				 * We cannot easily use load-acquire here, since
+				 * the actual load is a failed cmpxchg, which
+				 * doesn't imply any barriers.
+				 *
+				 * Also, this is a fairly unlikely scenario, and
+				 * this contains the cost.
+				 */
+				smp_mb(); /* ACQUIRE */
+				return true;
+			}
+
 			return false;
+		}
 
 		old = atomic_long_cmpxchg_acquire(&lock->owner, owner,
 						  curr | __owner_flags(owner));
@@ -134,6 +157,39 @@ static inline void __mutex_clear_flag(st
 	atomic_long_andnot(flag, &lock->owner);
 }
 
+static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
+{
+	return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
+}
+
+/*
+ * Give up ownership to a specific task, when @task = NULL, this is equivalent
+ * to a regular unlock. Clears HANDOFF, preserves WAITERS. Provides RELEASE
+ * semantics like a regular unlock, the __mutex_trylock() provides matching
+ * ACQUIRE semantics for the handoff.
+ */
+static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
+{
+	unsigned long owner = atomic_long_read(&lock->owner);
+
+	for (;;) {
+		unsigned long old, new;
+
+#ifdef CONFIG_DEBUG_MUTEXES
+		DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
+#endif
+
+		new = (owner & MUTEX_FLAG_WAITERS);
+		new |= (unsigned long)task;
+
+		old = atomic_long_cmpxchg_release(&lock->owner, owner, new);
+		if (old == owner)
+			break;
+
+		owner = old;
+	}
+}
+
 #ifndef CONFIG_DEBUG_LOCK_ALLOC
 /*
  * We split the mutex lock/unlock logic into separate fastpath and
@@ -398,7 +454,7 @@ static bool mutex_optimistic_spin(struct
 			break;
 
 		/* Try to acquire the mutex if it is unlocked. */
-		if (__mutex_trylock(lock)) {
+		if (__mutex_trylock(lock, false)) {
 			osq_unlock(&lock->osq);
 			return true;
 		}
@@ -523,6 +579,7 @@ __mutex_lock_common(struct mutex *lock,
 	struct task_struct *task = current;
 	struct mutex_waiter waiter;
 	unsigned long flags;
+	bool first = false;
 	int ret;
 
 	if (use_ww_ctx) {
@@ -534,7 +591,8 @@ __mutex_lock_common(struct mutex *lock,
 	preempt_disable();
 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
 
-	if (__mutex_trylock(lock) || mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
+	if (__mutex_trylock(lock, false) ||
+	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
 		/* got the lock, yay! */
 		lock_acquired(&lock->dep_map, ip);
 		if (use_ww_ctx) {
@@ -551,7 +609,7 @@ __mutex_lock_common(struct mutex *lock,
 	/*
 	 * After waiting to acquire the wait_lock, try again.
 	 */
-	if (__mutex_trylock(lock))
+	if (__mutex_trylock(lock, false))
 		goto skip_wait;
 
 	debug_mutex_lock_common(lock, &waiter);
@@ -561,15 +619,15 @@ __mutex_lock_common(struct mutex *lock,
 	list_add_tail(&waiter.list, &lock->wait_list);
 	waiter.task = task;
 
-	if (list_first_entry(&lock->wait_list, struct mutex_waiter, list) == &waiter)
+	if (__mutex_waiter_is_first(lock, &waiter))
 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
 
+	if (__mutex_trylock(lock, false))
+		goto remove_waiter;
+
 	lock_contended(&lock->dep_map, ip);
 
 	for (;;) {
-		if (__mutex_trylock(lock))
-			break;
-
 		/*
 		 * got a signal? (This code gets eliminated in the
 		 * TASK_UNINTERRUPTIBLE case.)
@@ -586,17 +644,26 @@ __mutex_lock_common(struct mutex *lock,
 		}
 
 		__set_task_state(task, state);
-
-		/* didn't get the lock, go to sleep: */
 		spin_unlock_mutex(&lock->wait_lock, flags);
 		schedule_preempt_disabled();
 		spin_lock_mutex(&lock->wait_lock, flags);
+
+		if (__mutex_waiter_is_first(lock, &waiter)) {
+			first = true;
+			__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
+		}
+
+		if (__mutex_trylock(lock, true))
+			break;
 	}
 	__set_task_state(task, TASK_RUNNING);
 
+remove_waiter:
 	mutex_remove_waiter(lock, &waiter, task);
 	if (likely(list_empty(&lock->wait_list)))
-		__mutex_clear_flag(lock, MUTEX_FLAG_WAITERS);
+		__mutex_clear_flag(lock, MUTEX_FLAGS);
+	else if (first && (atomic_long_read(&lock->owner) & MUTEX_FLAG_HANDOFF))
+		__mutex_clear_flag(lock, MUTEX_FLAG_HANDOFF);
 
 	debug_mutex_free_waiter(&waiter);
 
@@ -724,33 +791,61 @@ EXPORT_SYMBOL_GPL(__ww_mutex_lock_interr
  */
 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
 {
+	struct task_struct *next = NULL;
 	unsigned long owner, flags;
 	WAKE_Q(wake_q);
 
 	mutex_release(&lock->dep_map, 1, ip);
 
 	/*
-	 * Release the lock before (potentially) taking the spinlock
-	 * such that other contenders can get on with things ASAP.
+	 * Release the lock before (potentially) taking the spinlock such that
+	 * other contenders can get on with things ASAP.
+	 *
+	 * Except when HANDOFF, in that case we must not clear the owner field,
+	 * but instead set it to the top waiter.
 	 */
-	owner = atomic_long_fetch_and_release(MUTEX_FLAGS, &lock->owner);
-	if (!owner)
-		return;
+	owner = atomic_long_read(&lock->owner);
+	for (;;) {
+		unsigned long old;
+
+#ifdef CONFIG_DEBUG_MUTEXES
+		DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
+#endif
+
+		if (owner & MUTEX_FLAG_HANDOFF)
+			break;
+
+		old = atomic_long_cmpxchg_release(&lock->owner, owner,
+						  __owner_flags(owner));
+		if (old == owner) {
+			if (owner & MUTEX_FLAG_WAITERS)
+				break;
+
+			return;
+		}
+
+		owner = old;
+	}
 
 	spin_lock_mutex(&lock->wait_lock, flags);
 	debug_mutex_unlock(lock);
-
 	if (!list_empty(&lock->wait_list)) {
 		/* get the first entry from the wait-list: */
 		struct mutex_waiter *waiter =
-				list_entry(lock->wait_list.next,
-					   struct mutex_waiter, list);
+			list_first_entry(&lock->wait_list,
+					 struct mutex_waiter, list);
+
+		next = waiter->task;
 
 		debug_mutex_wake_waiter(lock, waiter);
-		wake_q_add(&wake_q, waiter->task);
+		wake_q_add(&wake_q, next);
 	}
 
+	if (owner & MUTEX_FLAG_HANDOFF)
+		__mutex_handoff(lock, next);
+
 	spin_unlock_mutex(&lock->wait_lock, flags);
+
 	wake_up_q(&wake_q);
 }
 
@@ -853,7 +948,7 @@ __ww_mutex_lock_interruptible_slowpath(s
  */
 int __sched mutex_trylock(struct mutex *lock)
 {
-	bool locked = __mutex_trylock(lock);
+	bool locked = __mutex_trylock(lock, false);
 
 	if (locked)
 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
author maintainer
#7 lore raw

PATCH 7/10 mutex: Restructure wait loop

In reply to Peter Zijlstra

Doesn't really matter yet, but pull the HANDOFF and trylock out from under the wait_lock.

The intention is to add an optimistic spin loop here, which requires we do not hold the wait_lock, so shuffle code around in preparation.

Also clarify the purpose of taking the wait_lock in the wait loop, its tempting to want to avoid it altogether, but the cancellation cases need to to avoid loosing wakeups.

---
 kernel/locking/mutex.c |   18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)
kernel/locking/mutex.c +14 -4
@@ -627,10 +627,12 @@ __mutex_lock_common(struct mutex *lock,
 
 	lock_contended(&lock->dep_map, ip);
 
+	set_task_state(task, state);
 	for (;;) {
 		/*
-		 * got a signal? (This code gets eliminated in the
-		 * TASK_UNINTERRUPTIBLE case.)
+		 * Check for signals and wound conditions while holding
+		 * wait_lock. This ensures the lock cancellation is ordered
+		 * against mutex_unlock() and wake-ups do not go missing.
 		 */
 		if (unlikely(signal_pending_state(state, task))) {
 			ret = -EINTR;
@@ -643,20 +645,27 @@ __mutex_lock_common(struct mutex *lock,
 				goto err;
 		}
 
-		__set_task_state(task, state);
 		spin_unlock_mutex(&lock->wait_lock, flags);
 		schedule_preempt_disabled();
-		spin_lock_mutex(&lock->wait_lock, flags);
 
 		if (__mutex_waiter_is_first(lock, &waiter)) {
 			first = true;
 			__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
 		}
 
+		set_task_state(task, state);
+		/*
+		 * Here we order against unlock; we must either see it change
+		 * state back to RUNNING and fall through the next schedule(),
+		 * or we must see its unlock and acquire.
+		 */
 		if (__mutex_trylock(lock, true))
 			break;
+
+		spin_lock_mutex(&lock->wait_lock, flags);
 	}
 	__set_task_state(task, TASK_RUNNING);
+	spin_lock_mutex(&lock->wait_lock, flags);
 
 remove_waiter:
 	mutex_remove_waiter(lock, &waiter, task);
@@ -681,6 +690,7 @@ __mutex_lock_common(struct mutex *lock,
 	return 0;
 
 err:
+	__set_task_state(task, TASK_RUNNING);
 	mutex_remove_waiter(lock, &waiter, task);
 	spin_unlock_mutex(&lock->wait_lock, flags);
 	debug_mutex_free_waiter(&waiter);

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

author maintainer
#10 lore raw

Re: [PATCH -v3 07/10] mutex: Restructure wait loop

Quoted 10 lines · Waiman Long
Quoted 6 lines

I think you need to remove the HANDOFF flag here in the error path too. Maybe you should fix that in patch 6.

It is also possible that __mutex_handoff() has been called at this point. So this task may now be the owner of the lock. Maybe you should do a final trylock here.

Alternatively, we can move the trylock in the main loop after the spin_lock_mutex(). In this case, we don't need an extra trylock in the error path.

I did something like so..

kernel/locking/mutex.c +3 -7
@@ -629,12 +629,12 @@ __mutex_lock_common(struct mutex *lock,
 	if (__mutex_waiter_is_first(lock, &waiter))
 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
 
-	if (__mutex_trylock(lock, false))
-		goto remove_waiter;
-
 	lock_contended(&lock->dep_map, ip);
 
 	for (;;) {
+		if (__mutex_trylock(lock, first))
+			break;
+
 		/*
 		 * got a signal? (This code gets eliminated in the
 		 * TASK_UNINTERRUPTIBLE case.)
@@ -659,13 +659,9 @@ __mutex_lock_common(struct mutex *lock,
 			first = true;
 			__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
 		}
-
-		if (__mutex_trylock(lock, first))
-			break;
 	}
 	__set_task_state(task, TASK_RUNNING);
 
-remove_waiter:
 	mutex_remove_waiter(lock, &waiter, task);
 	if (likely(list_empty(&lock->wait_list)))
 		__mutex_clear_flag(lock, MUTEX_FLAGS);
author maintainer
#11 lore raw

PATCH 8/10 locking/mutex: Simplify some ww_mutex code in __mutex_lock_common()

In reply to Peter Zijlstra

This patch removes some of the redundant ww_mutex code in __mutex_lock_common().

Cc Ingo Molnar <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Tim Chen <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Imre Deak <[email protected]>
Cc Jason Low <[email protected]>
Cc "Paul E. McKenney" <[email protected]>
Cc Ding Tianhong <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Will Deacon <[email protected]>
---
 kernel/locking/mutex.c |   13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)
kernel/locking/mutex.c +4 -9
@@ -580,10 +580,11 @@ __mutex_lock_common(struct mutex *lock,
 	struct mutex_waiter waiter;
 	unsigned long flags;
 	bool first = false;
+	struct ww_mutex *ww;
 	int ret;
 
 	if (use_ww_ctx) {
-		struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
+		ww = container_of(lock, struct ww_mutex, base);
 		if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
 			return -EALREADY;
 	}
@@ -595,12 +596,8 @@ __mutex_lock_common(struct mutex *lock,
 	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
 		/* got the lock, yay! */
 		lock_acquired(&lock->dep_map, ip);
-		if (use_ww_ctx) {
-			struct ww_mutex *ww;
-			ww = container_of(lock, struct ww_mutex, base);
-
+		if (use_ww_ctx)
 			ww_mutex_set_context_fastpath(ww, ww_ctx);
-		}
 		preempt_enable();
 		return 0;
 	}
@@ -680,10 +677,8 @@ __mutex_lock_common(struct mutex *lock,
 	/* got the lock - cleanup and rejoice! */
 	lock_acquired(&lock->dep_map, ip);
 
-	if (use_ww_ctx) {
-		struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
+	if (use_ww_ctx)
 		ww_mutex_set_context_slowpath(ww, ww_ctx);
-	}
 
 	spin_unlock_mutex(&lock->wait_lock, flags);
 	preempt_enable();
author maintainer
#12 lore raw

PATCH 9/10 locking/mutex: Enable optimistic spinning of woken waiter

In reply to Peter Zijlstra

This patch makes the waiter that sets the HANDOFF flag start spinning instead of sleeping until the handoff is complete or the owner sleeps. Otherwise, the handoff will cause the optimistic spinners to abort spinning as the handed-off owner may not be running.

Cc Ingo Molnar <[email protected]>
Cc Linus Torvalds <[email protected]>
Cc Tim Chen <[email protected]>
Cc Thomas Gleixner <[email protected]>
Cc Imre Deak <[email protected]>
Cc Jason Low <[email protected]>
Cc "Paul E. McKenney" <[email protected]>
Cc Ding Tianhong <[email protected]>
Cc Davidlohr Bueso <[email protected]>
Cc Will Deacon <[email protected]>
---
 kernel/locking/mutex.c |   76 ++++++++++++++++++++++++++++++++++---------------
 1 file changed, 54 insertions(+), 22 deletions(-)
kernel/locking/mutex.c +54 -22
@@ -409,24 +409,39 @@ static inline int mutex_can_spin_on_owne
  *
  * Returns true when the lock was taken, otherwise false, indicating
  * that we need to jump to the slowpath and sleep.
+ *
+ * The waiter flag is set to true if the spinner is a waiter in the wait
+ * queue. The waiter-spinner will spin on the lock directly and concurrently
+ * with the spinner at the head of the OSQ, if present, until the owner is
+ * changed to itself.
  */
 static bool mutex_optimistic_spin(struct mutex *lock,
-				  struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
+				  struct ww_acquire_ctx *ww_ctx,
+				  const bool use_ww_ctx, const bool waiter)
 {
 	struct task_struct *task = current;
 
-	if (!mutex_can_spin_on_owner(lock))
-		goto done;
+	if (!waiter) {
+		/*
+		 * The purpose of the mutex_can_spin_on_owner() function is
+		 * to eliminate the overhead of osq_lock() and osq_unlock()
+		 * in case spinning isn't possible. As a waiter-spinner
+		 * is not going to take OSQ lock anyway, there is no need
+		 * to call mutex_can_spin_on_owner().
+		 */
+		if (!mutex_can_spin_on_owner(lock))
+			goto fail;
 
-	/*
-	 * In order to avoid a stampede of mutex spinners trying to
-	 * acquire the mutex all at once, the spinners need to take a
-	 * MCS (queued) lock first before spinning on the owner field.
-	 */
-	if (!osq_lock(&lock->osq))
-		goto done;
+		/*
+		 * In order to avoid a stampede of mutex spinners trying to
+		 * acquire the mutex all at once, the spinners need to take a
+		 * MCS (queued) lock first before spinning on the owner field.
+		 */
+		if (!osq_lock(&lock->osq))
+			goto fail;
+	}
 
-	while (true) {
+	for (;;) {
 		struct task_struct *owner;
 
 		if (use_ww_ctx && ww_ctx->acquired > 0) {
@@ -442,7 +457,7 @@ static bool mutex_optimistic_spin(struct
 			 * performed the optimistic spinning cannot be done.
 			 */
 			if (READ_ONCE(ww->ctx))
-				break;
+				goto fail_unlock;
 		}
 
 		/*
@@ -450,15 +465,20 @@ static bool mutex_optimistic_spin(struct
 		 * release the lock or go to sleep.
 		 */
 		owner = __mutex_owner(lock);
-		if (owner && !mutex_spin_on_owner(lock, owner))
-			break;
+		if (owner) {
+			if (waiter && owner == task) {
+				smp_mb(); /* ACQUIRE */
+				break;
+			}
 
-		/* Try to acquire the mutex if it is unlocked. */
-		if (__mutex_trylock(lock, false)) {
-			osq_unlock(&lock->osq);
-			return true;
+			if (!mutex_spin_on_owner(lock, owner))
+				goto fail_unlock;
 		}
 
+		/* Try to acquire the mutex if it is unlocked. */
+		if (__mutex_trylock(lock, waiter))
+			break;
+
 		/*
 		 * The cpu_relax() call is a compiler barrier which forces
 		 * everything in this loop to be re-loaded. We don't need
@@ -468,8 +488,17 @@ static bool mutex_optimistic_spin(struct
 		cpu_relax_lowlatency();
 	}
 
-	osq_unlock(&lock->osq);
-done:
+	if (!waiter)
+		osq_unlock(&lock->osq);
+
+	return true;
+
+
+fail_unlock:
+	if (!waiter)
+		osq_unlock(&lock->osq);
+
+fail:
 	/*
 	 * If we fell out of the spin path because of need_resched(),
 	 * reschedule now, before we try-lock the mutex. This avoids getting
@@ -488,7 +517,8 @@ static bool mutex_optimistic_spin(struct
 }
 #else
 static bool mutex_optimistic_spin(struct mutex *lock,
-				  struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
+				  struct ww_acquire_ctx *ww_ctx,
+				  const bool use_ww_ctx, const bool waiter)
 {
 	return false;
 }
@@ -593,7 +623,7 @@ __mutex_lock_common(struct mutex *lock,
 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
 
 	if (__mutex_trylock(lock, false) ||
-	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx)) {
+	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, false)) {
 		/* got the lock, yay! */
 		lock_acquired(&lock->dep_map, ip);
 		if (use_ww_ctx)
@@ -648,6 +678,8 @@ __mutex_lock_common(struct mutex *lock,
 		if (__mutex_waiter_is_first(lock, &waiter)) {
 			first = true;
 			__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
+			if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, true))
+				break;
 		}
 
 		set_task_state(task, state);
author maintainer
#13 lore raw

RFC PATCH 10/10 locking/mutex: Implement alternative HANDOFF

In reply to Peter Zijlstra

As mentioned in a previous patch, its possible to implement the handoff logic differently, avoiding the issue where we 'leak' a HADOFF flag.

This patch does so, just to show what it looks like; I'm not at all convinced this is worth it.

---
 kernel/locking/mutex.c |   34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)
kernel/locking/mutex.c +29 -5
@@ -71,6 +71,32 @@ static inline unsigned long __owner_flag
 	return owner & MUTEX_FLAGS;
 }
 
+static inline bool __mutex_req_handoff(struct mutex *lock)
+{
+	unsigned long owner, curr = (unsigned long)current;
+
+	owner = atomic_long_read(&lock->owner);
+	for (;;) {
+		unsigned long old, new;
+
+		if (owner & MUTEX_FLAG_HANDOFF)
+			return false;
+
+		if (__owner_task(owner))
+			new = owner | MUTEX_FLAG_HANDOFF;
+		else
+			new = curr | __owner_flags(owner);
+
+		old = atomic_long_cmpxchg_acquire(&lock->owner, owner, new);
+		if (old == owner)
+			break;
+
+		owner = old;
+	}
+
+	return !__owner_task(owner);
+}
+
 /*
  * Actual trylock that will work on any unlocked state.
  *
@@ -609,7 +635,6 @@ __mutex_lock_common(struct mutex *lock,
 	struct task_struct *task = current;
 	struct mutex_waiter waiter;
 	unsigned long flags;
-	bool first = false;
 	struct ww_mutex *ww;
 	int ret;
 
@@ -676,8 +701,9 @@ __mutex_lock_common(struct mutex *lock,
 		schedule_preempt_disabled();
 
 		if (__mutex_waiter_is_first(lock, &waiter)) {
-			first = true;
-			__mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
+			if (__mutex_req_handoff(lock))
+				break;
+
 			if (mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, true))
 				break;
 		}
@@ -700,8 +726,6 @@ __mutex_lock_common(struct mutex *lock,
 	mutex_remove_waiter(lock, &waiter, task);
 	if (likely(list_empty(&lock->wait_list)))
 		__mutex_clear_flag(lock, MUTEX_FLAGS);
-	else if (first && (atomic_long_read(&lock->owner) & MUTEX_FLAG_HANDOFF))
-		__mutex_clear_flag(lock, MUTEX_FLAG_HANDOFF);
 
 	debug_mutex_free_waiter(&waiter);
 
#14 lore raw

Re: [RFC][PATCH -v3 10/10] locking/mutex: Implement alternative HANDOFF

Quoted 9 lines · Peter Zijlstra

As mentioned in a previous patch, its possible to implement the handoff logic differently, avoiding the issue where we 'leak' a HADOFF flag.

This patch does so, just to show what it looks like; I'm not at all convinced this is worth it.

I have another way of catching the uncleared handoff flag. See the following code to see if you think that will work.

kernel/locking/mutex.c +7 -1
index 9492494..362ff83 100644
@@ -85,7 +85,13 @@ static inline bool __mutex_trylock(struct mutex *lock, const
 
        owner = atomic_long_read(&lock->owner);
        for (;;) { /* must loop, can race against a flag */
-               unsigned long old;
+               unsigned long old, flags = __owner_flags(owner);
+
+               /*
+                * We don't need to keep the HANDOFF flag for the waiter.
+                */
+               if (handoff)
+                       flags &= ~MUTEX_FLAG_HANDOFF;
 
                if (__owner_task(owner)) {
                        if (handoff && unlikely(__owner_task(owner) == 

current))
@@ -107,7 +113,7 @@ static inline bool __mutex_trylock(struct mutex *lock, const

                }

                old = atomic_long_cmpxchg_acquire(&lock->owner, owner,
-                                                 curr | __owner_flags(owner));
+                                                 curr | flags);
                if (old == owner)
                        return true;

@@ -688,7 +694,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned
* state back to RUNNING and fall through the next
schedule(),
* or we must see its unlock and acquire.
*/
- if (__mutex_trylock(lock, true))
+ if (__mutex_trylock(lock, first))

                        break;

                spin_lock_mutex(&lock->wait_lock, flags);
@@ -700,8 +706,6 @@ remove_waiter:
        mutex_remove_waiter(lock, &waiter, task);
        if (likely(list_empty(&lock->wait_list)))
                __mutex_clear_flag(lock, MUTEX_FLAGS);

- else if (first && (atomic_long_read(&lock->owner) & MUTEX_FLAG_HANDOFF))
- __mutex_clear_flag(lock, MUTEX_FLAG_HANDOFF);

        debug_mutex_free_waiter(&waiter);

Cheers,
Longman

author maintainer
#15 lore raw

Re: [RFC][PATCH -v3 10/10] locking/mutex: Implement alternative HANDOFF

Quoted 13 lines · Waiman Long

I have another way of catching the uncleared handoff flag. See the following code to see if you think that will work.

kernel/locking/mutex.c +0 -0
index 9492494..362ff83 100644
@@ -85,7 +85,13 @@ static inline bool __mutex_trylock(struct mutex *lock,

const

owner = atomic_long_read(&lock->owner); for (;;) { /* must loop, can race against a flag */

kernel/locking/mutex.c +0 -1
-               unsigned long old;

kernel/locking/mutex.c static inline bool __mutex_trylock(struct mutex *lock, const

Quoted 10 lines
kernel/locking/mutex.c +7 -0
+               unsigned long old, flags = __owner_flags(owner);
+
+               /*
+                * We don't need to keep the HANDOFF flag for the waiter.
+                */
+               if (handoff)
+                       flags &= ~MUTEX_FLAG_HANDOFF;
 
                if (__owner_task(owner)) {
                        if (handoff && unlikely(__owner_task(owner) == current))

I placed this condition below the __owner_task() branch.

kernel/locking/mutex.c __mutex_lock_common(struct mutex *lock, long state, unsigned · line 700

Quoted 9 lines
kernel/locking/mutex.c +1 -1
@@ -688,7 +694,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned
                 * state back to RUNNING and fall through the next schedule(),
                 * or we must see its unlock and acquire.
                 */
-               if (__mutex_trylock(lock, true))
+               if (__mutex_trylock(lock, first))
                        break;
 
                spin_lock_mutex(&lock->wait_lock, flags);

Hmm, yes. I think that works.

We (the first waiter) set the flag, we clear it on try-lock, or unlock clears it when it hands the thing off.

Much simpler. Thanks!