Skip to contentSkip to conversation

locking/spinlock_debug: Change it to a mostly fair lock

PATCH Superseded Waiman Long 2 participants 5 messages Feb 1, 2017 – Feb 2, 2017

Subsystems (6)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Waiman Long

author reviewer
#0 lore raw

v2 cover locking/spinlock_debug: Change it to a mostly fair lock

v1->v2:
- Pack lockup and break_lock into a single 4-byte slot so as not to in increase spinlock size when GENERIC_LOCKBREAK is on. Hopefully that will be enough to fix a frame size too large warning in 0-day build.
- Add a new patch to disable GENERIC_LOCKBREAK when DEBUG_LOCK_ALLOC is on.

The current debug spinlock implementation is a TATAS unfair lock. This can occasionally lead to system lockup with a debug kernel because of the unfairness of the lock rather than inherent locking problem.

This patch set changes the debug spinlock implementation to a mostly fair spinlock based on the MCS lock similar to what is done in qspinlock.

Waiman Long (3):
locking/spinlock_debug: Reduce lockup suspected message clutter locking/spinlock_debug: Reduce lock cacheline contention locking/spinlock: Disable GENERIC_LOCKBREAK when DEBUG_LOCK_ALLOC is
on

 arch/m32r/Kconfig               |  2 +-
 arch/parisc/Kconfig             |  2 +-
 arch/powerpc/Kconfig            |  2 +-
 arch/s390/Kconfig               |  2 +-
 arch/sh/Kconfig                 |  2 +-
 arch/sparc/Kconfig              |  2 +-
 include/linux/spinlock_types.h  |  9 +++--
 kernel/locking/spinlock_debug.c | 73 ++++++++++++++++++++++++++++++-----------
 8 files changed, 65 insertions(+), 29 deletions(-)
Signature · 1 line
1.8.3.1
author reviewer
#1 lore raw

PATCH v2 1/3 locking/spinlock_debug: Reduce lockup suspected message clutter

When the debug spinlock code detects a lockup, it will print out an error messages as well as the backtraces of all the CPUs. However, if more than one CPUs are waiting on that lock, multiple lockup messages will be printed leading to garbled output.

To reduce clutter in the console log, now only one of the lock waiters will be allowed to print out the CPU backtraces.

Since break_lock, like lockup, can only have a value of 0 or 1, its size is now reduced so that on a 64-bit architecture, the size of the raw_spinlock structure won't increase whether CONFIG_GENERIC_LOCKBREAK is defined or not.

---
 include/linux/spinlock_types.h  |  4 +++-
 kernel/locking/spinlock_debug.c | 26 +++++++++++++++++++++-----
 2 files changed, 24 insertions(+), 6 deletions(-)
include/linux/spinlock_types.h +3 -1
index 73548eb..99f28bd 100644
@@ -20,9 +20,10 @@
 typedef struct raw_spinlock {
 	arch_spinlock_t raw_lock;
 #ifdef CONFIG_GENERIC_LOCKBREAK
-	unsigned int break_lock;
+	unsigned short break_lock;
 #endif
 #ifdef CONFIG_DEBUG_SPINLOCK
+	unsigned short lockup;
 	unsigned int magic, owner_cpu;
 	void *owner;
 #endif
@@ -43,6 +44,7 @@
 
 #ifdef CONFIG_DEBUG_SPINLOCK
 # define SPIN_DEBUG_INIT(lockname)		\
+	.lockup = 0,				\
 	.magic = SPINLOCK_MAGIC,		\
 	.owner_cpu = -1,			\
 	.owner = SPINLOCK_OWNER_INIT,
kernel/locking/spinlock_debug.c +21 -5
index 0374a59..0f880a8 100644
@@ -27,6 +27,7 @@ void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
 	lock->magic = SPINLOCK_MAGIC;
 	lock->owner = SPINLOCK_OWNER_INIT;
 	lock->owner_cpu = -1;
+	lock->lockup = 0;
 }
 
 EXPORT_SYMBOL(__raw_spin_lock_init);
@@ -101,6 +102,24 @@ static inline void debug_spin_unlock(raw_spinlock_t *lock)
 							lock, "wrong CPU");
 	lock->owner = SPINLOCK_OWNER_INIT;
 	lock->owner_cpu = -1;
+	lock->lockup = 0;
+}
+
+static inline void __spin_lockup(raw_spinlock_t *lock)
+{
+	/*
+	 * lockup suspected:
+	 *
+	 * Only one of the lock waiters will be allowed to print the lockup
+	 * message in order to avoid an avalanche of lockup and backtrace
+	 * messages from different lock waiters of the same lock.
+	 */
+	if (!xchg(&lock->lockup, 1)) {
+		spin_dump(lock, "lockup suspected");
+#ifdef CONFIG_SMP
+		trigger_all_cpu_backtrace();
+#endif
+	}
 }
 
 static void __spin_lock_debug(raw_spinlock_t *lock)
@@ -113,11 +132,8 @@ static void __spin_lock_debug(raw_spinlock_t *lock)
 			return;
 		__delay(1);
 	}
-	/* lockup suspected: */
-	spin_dump(lock, "lockup suspected");
-#ifdef CONFIG_SMP
-	trigger_all_cpu_backtrace();
-#endif
+
+	__spin_lockup(lock);
 
 	/*
 	 * The trylock above was causing a livelock.  Give the lower level arch
Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH v2 1/3] locking/spinlock_debug: Reduce lockup suspected message clutter

url: https://github.com/0day-ci/linux/commits/Waiman-Long/locking-spinlock_debug-Change-it-to-a-mostly-fair-lock/20170202-052215 config: arm-omap2plus_defconfig (attached as .config) compiler…

Hi Waiman,

[auto build test ERROR on tip/locking/core]
[also build test ERROR on v4.10-rc6 next-20170201]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url: https://github.com/0day-ci/linux/commits/Waiman-Long/locking-spinlock_debug-Change-it-to-a-mostly-fair-lock/20170202-052215
config: arm-omap2plus_defconfig (attached as .config)
compiler: arm-linux-gnueabi-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
make.cross ARCH=arm

All errors (new ones prefixed by >>):

kernel/built-in.o: In function `do_raw_spin_lock':

>> arch/arm/include/asm/cmpxchg.h:109: undefined reference to `__bad_xchg'
>> kernel/built-in.o:(.debug_addr+0x28c6c): undefined reference to `__bad_xchg'

vim +109 arch/arm/include/asm/cmpxchg.h

9f97da78 David Howells 2012-03-28   93  		asm volatile("@	__xchg1\n"
9f97da78 David Howells 2012-03-28   94  		"	swpb	%0, %1, [%2]"
9f97da78 David Howells 2012-03-28   95  			: "=&r" (ret)
9f97da78 David Howells 2012-03-28   96  			: "r" (x), "r" (ptr)
9f97da78 David Howells 2012-03-28   97  			: "memory", "cc");
9f97da78 David Howells 2012-03-28   98  		break;
9f97da78 David Howells 2012-03-28   99  	case 4:
9f97da78 David Howells 2012-03-28  100  		asm volatile("@	__xchg4\n"
9f97da78 David Howells 2012-03-28  101  		"	swp	%0, %1, [%2]"
9f97da78 David Howells 2012-03-28  102  			: "=&r" (ret)
9f97da78 David Howells 2012-03-28  103  			: "r" (x), "r" (ptr)
9f97da78 David Howells 2012-03-28  104  			: "memory", "cc");
9f97da78 David Howells 2012-03-28  105  		break;
9f97da78 David Howells 2012-03-28  106  #endif
9f97da78 David Howells 2012-03-28  107  	default:
31cd08c3 Russell King  2015-05-19  108  		/* Cause a link-time error, the xchg() size is not supported */
9f97da78 David Howells 2012-03-28 @109  		__bad_xchg(ptr, size), ret = 0;
9f97da78 David Howells 2012-03-28  110  		break;
9f97da78 David Howells 2012-03-28  111  	}
9f97da78 David Howells 2012-03-28  112  
9f97da78 David Howells 2012-03-28  113  	return ret;
9f97da78 David Howells 2012-03-28  114  }
9f97da78 David Howells 2012-03-28  115  
0ca326de Will Deacon   2015-08-06  116  #define xchg_relaxed(ptr, x) ({						\
e001bbae Russell King  2015-05-26  117  	(__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr),		\

:::::: The code at line 109 was first introduced by commit
:::::: 9f97da78bf018206fb623cd351d454af2f105fe0 Disintegrate asm/system.h for ARM

:::::: TO: David Howells <[email protected]> :::::: CC: David Howells <[email protected]>

---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation

author reviewer
#3 lore raw

PATCH v2 2/3 locking/spinlock_debug: Reduce lock cacheline contention

In reply to Waiman Long

The debug spinlock code is a basic TATAS unfair lock irrespective of what the underlying architecture specific spinlock implementation is. As a result, it is sometimes possible to trigger a false positive "lockup suspected" warning with all the cpu backtraces.

This patch re-implements the debug spinlock as a fair MCS lock. This reduces the chance of false positive warning messages. At the same time, it also improves performance by reducing lock cacheline contention.

Because there is a trylock before entering the MCS queue, this new debug spinlock code also perform pretty well in a virtual machine even if its vCPUSs are over-committed.

On a 4-socket 32-core 64-thread system, the performance of a locking microbenchmark (locking rate and standard deviation) on a 4.9.6 based debug kernel with and without the patch was as follows:

32 locking threads:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------
  w/o patch     263.1 Mop/s         1.39s
  with patch    917.6 Mop/s         0.07s

64 locking threads:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------
  w/o patch     368.3 Mop/s         6.88s
  with patch    733.0 Mop/s         0.09s

On a 2-socket 24-core 48-thread system, the performance of the same locking microbenchmark (# of locking threads = # of vCPUs) on a KVM guest are as follows:

24 vCPUs:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------

w/o patch 746.4 Mop/s 1.07s
with patch 1323.6 Mop/s 0.20s

48 vCPUs:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------
  w/o patch    1077.8 Mop/s         3.34s
  with patch   1090.4 Mop/s         0.29s

72 vCPUs:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------

w/o patch 944.5 Mop/s 3.96s
with patch 1176.7 Mop/s 0.44s

96 vCPUs:

  Kernel       Locking Rate    SD (execution time)
  ------       ------------    -------------------

w/o patch 878.0 Mop/s 5.19s
with patch 1017.0 Mop/s 0.83s

---
 include/linux/spinlock_types.h  |  5 ++--
 kernel/locking/spinlock_debug.c | 53 +++++++++++++++++++++++++++--------------
 2 files changed, 38 insertions(+), 20 deletions(-)
include/linux/spinlock_types.h +3 -2
index 99f28bd..562af2d 100644
@@ -25,7 +25,7 @@
 #ifdef CONFIG_DEBUG_SPINLOCK
 	unsigned short lockup;
 	unsigned int magic, owner_cpu;
-	void *owner;
+	void *owner, *tail;
 #endif
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map dep_map;
@@ -47,7 +47,8 @@
 	.lockup = 0,				\
 	.magic = SPINLOCK_MAGIC,		\
 	.owner_cpu = -1,			\
-	.owner = SPINLOCK_OWNER_INIT,
+	.owner = SPINLOCK_OWNER_INIT,		\
+	.tail = NULL,
 #else
 # define SPIN_DEBUG_INIT(lockname)
 #endif
kernel/locking/spinlock_debug.c +35 -18

91 lines not on this page · raw message

Signature · 1 line
1.8.3.1
author reviewer
#4 lore raw

PATCH v2 3/3 locking/spinlock: Disable GENERIC_LOCKBREAK when DEBUG_LOCK_ALLOC is on

In reply to Waiman Long

The break_lock variable defined when GENERIC_LOCKBREAK is on is used only in kernel/locking/spinlock.c when

defined(CONFIG_GENERIC_LOCKBREAK) && !defined(CONFIG_DEBUG_LOCK_ALLOC).

As a result, there is no point in enabling GENERIC_LOCKBREAK to define one more variable in the spinlock structure that is not going to be used when DEBUG_LOCK_ALLOC is also on. This patch disables GENERIC_LOCKBREAK under this circumstance.

---
 arch/m32r/Kconfig    | 2 +-
 arch/parisc/Kconfig  | 2 +-
 arch/powerpc/Kconfig | 2 +-
 arch/s390/Kconfig    | 2 +-
 arch/sh/Kconfig      | 2 +-
 arch/sparc/Kconfig   | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)
arch/m32r/Kconfig +1 -1
index d227a69..c0922e0 100644
@@ -242,7 +242,7 @@ config IRAM_SIZE
 config GENERIC_LOCKBREAK
 	bool
 	default y
-	depends on SMP && PREEMPT
+	depends on SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config RWSEM_GENERIC_SPINLOCK
 	bool
arch/parisc/Kconfig +1 -1
index 3a71f38..bd8dbb9 100644
@@ -56,7 +56,7 @@ config STACK_GROWSUP
 config GENERIC_LOCKBREAK
 	bool
 	default y
-	depends on SMP && PREEMPT
+	depends on SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config RWSEM_GENERIC_SPINLOCK
 	def_bool y
arch/powerpc/Kconfig +1 -1
index a8ee573..feae0a3 100644
@@ -60,7 +60,7 @@ config RWSEM_XCHGADD_ALGORITHM
 config GENERIC_LOCKBREAK
 	bool
 	default y
-	depends on SMP && PREEMPT
+	depends on SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config ARCH_HAS_ILOG2_U32
 	bool
arch/s390/Kconfig +1 -1
index c6722112..a2a33a6 100644
@@ -38,7 +38,7 @@ config ARCH_DMA_ADDR_T_64BIT
 	def_bool y
 
 config GENERIC_LOCKBREAK
-	def_bool y if SMP && PREEMPT
+	def_bool y if SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config PGSTE
 	def_bool y if KVM
arch/sh/Kconfig +1 -1
index ee08695..e2eb35c 100644
@@ -105,7 +105,7 @@ config GENERIC_CALIBRATE_DELAY
 
 config GENERIC_LOCKBREAK
 	def_bool y
-	depends on SMP && PREEMPT
+	depends on SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config ARCH_SUSPEND_POSSIBLE
 	def_bool n
arch/sparc/Kconfig +1 -1
index cf4034c..695a31a 100644
@@ -286,7 +286,7 @@ config US3_MC
 config GENERIC_LOCKBREAK
 	bool
 	default y
-	depends on SPARC64 && SMP && PREEMPT
+	depends on SPARC64 && SMP && PREEMPT && !DEBUG_LOCK_ALLOC
 
 config NUMA
 	bool "NUMA support"
Signature · 1 line
1.8.3.1