locking/lockdep: Fix debug_locks off performance problem
Subsystems (2)
Waiman Long
reviewerPATCH 1/2 locking/lockdep: Fix debug_locks off performance problem
It was found that when debug_locks was turned off because of a problem found by the lockdep code, the system performance could drop quite significantly when the lock_stat code was also configured into the kernel. For instance, parallel kernel build time on a 4-socket x86-64 server nearly doubled.
Further analysis into the cause of the slowdown traced back to the frequent call to debug_locks_off() from the __lock_acquired() function probably due to some inconsistent lockdep states with debug_locks off. The debug_locks_off() function did an unconditional atomic xchg to write a 0 value into debug_locks which had already been set to 0. This led to severe cacheline contention in the cacheline that held debug_locks. As debug_locks is being referenced in quite a few different places in the kernel, this greatly slow down the system performance.
To prevent that trashing of debug_locks cacheline, lock_acquired() and lock_contended() now checks the state of debug_locks before proceeding. The debug_locks_off() function is also modified to check debug_locks before calling __debug_locks_off().
--- kernel/locking/lockdep.c | 4 ++-- lib/debug_locks.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
kernel/locking/lockdep.c +2 -2
@@ -4066,7 +4066,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip) {unsigned long flags; -if (unlikely (!lock_stat)) +if (unlikely (!lock_stat || !debug_locks))return ;if (unlikely (current->lockdep_recursion))
@@ -4086,7 +4086,7 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip) {unsigned long flags; -if (unlikely (!lock_stat)) +if (unlikely (!lock_stat || !debug_locks))return ;if (unlikely (current->lockdep_recursion))
lib/debug_locks.c +1 -1
@@ -37,7 +37,7 @@*/ int debug_locks_off (void ) { -if (__debug_locks_off ()) { +if (debug_locks &&__debug_locks_off ()) {if (!debug_locks_silent) {console_verbose ();return 1 ;
Signature · 1 line
1.8.3.1
Waiman Long
reviewerPATCH 2/2 locking/lockdep: Make global debug_locks* variables read-mostly
Make the frequently used lockdep global variable debug_locks read-mostly. As debug_locks_silent is sometime used together with debug_locks, it is also made read-mostly so that they can be close together.
With false cacheline sharing, cacheline contention problem can happen depending on what get put into the same cacheline as debug_locks.
--- include/linux/debug_locks.h | 4 ++-- lib/debug_locks.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/debug_locks.h +2 -2
@@ -8,8 +8,8 @@struct task_struct ; -extern int debug_locks; -extern int debug_locks_silent; +extern int debug_locks__read_mostly ; +extern int debug_locks_silent__read_mostly ;static inline int __debug_locks_off (void )
lib/debug_locks.c +2 -2
@@ -21,7 +21,7 @@* that would just muddy the log. So we report the first one and * shut up after that. */ -int debug_locks =1 ; +int debug_locks__read_mostly =1 ;EXPORT_SYMBOL_GPL (debug_locks);/*
@@ -29,7 +29,7 @@* 'silent failure': nothing is printed to the console when * a locking bug is detected. */ -int debug_locks_silent; +int debug_locks_silent__read_mostly ;EXPORT_SYMBOL_GPL (debug_locks_silent);/*
Signature · 1 line
1.8.3.1
tip-bot for Waiman Long
merge bot[tip:locking/core] locking/lockdep: Make global debug_locks* variables read-mostly
locking/lockdep: Make global debug_locks* variables read-mostly
Commit-ID: 01a14bda11add9dcd4a59200f13834d634559935 Gitweb: https://git.kernel.org/tip/01a14bda11add9dcd4a59200f13834d634559935 Author: Waiman Long <[email protected]> AuthorDate: Thu, 18 Oct 2018 21:45:18 -0400 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 19 Oct 2018 07:53:18 +0200
locking/lockdep: Make global debug_locks* variables read-mostly
Make the frequently used lockdep global variable debug_locks read-mostly. As debug_locks_silent is sometime used together with debug_locks, it is also made read-mostly so that they can be close together.
With false cacheline sharing, cacheline contention problem can happen depending on what get put into the same cacheline as debug_locks.
--- include/linux/debug_locks.h | 4 ++-- lib/debug_locks.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/debug_locks.h +2 -2
@@ -8,8 +8,8 @@struct task_struct ; -extern int debug_locks; -extern int debug_locks_silent; +extern int debug_locks__read_mostly ; +extern int debug_locks_silent__read_mostly ;static inline int __debug_locks_off (void )
lib/debug_locks.c +2 -2
@@ -21,7 +21,7 @@* that would just muddy the log. So we report the first one and * shut up after that. */ -int debug_locks =1 ; +int debug_locks__read_mostly =1 ;EXPORT_SYMBOL_GPL (debug_locks);/*
@@ -29,7 +29,7 @@ EXPORT_SYMBOL_GPL(debug_locks);* 'silent failure': nothing is printed to the console when * a locking bug is detected. */ -int debug_locks_silent; +int debug_locks_silent__read_mostly ;EXPORT_SYMBOL_GPL (debug_locks_silent);/*
tip-bot for Waiman Long
merge botlocking/lockdep: Fix debug_locks off performance problem
Commit-ID: 9506a7425b094d2f1d9c877ed5a78f416669269b Gitweb: https://git.kernel.org/tip/9506a7425b094d2f1d9c877ed5a78f416669269b Author: Waiman Long <[email protected]> AuthorDate: Thu, 18 Oct 2018 21:45:17 -0400 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 19 Oct 2018 07:53:17 +0200
locking/lockdep: Fix debug_locks off performance problem
It was found that when debug_locks was turned off because of a problem found by the lockdep code, the system performance could drop quite significantly when the lock_stat code was also configured into the kernel. For instance, parallel kernel build time on a 4-socket x86-64 server nearly doubled.
Further analysis into the cause of the slowdown traced back to the frequent call to debug_locks_off() from the __lock_acquired() function probably due to some inconsistent lockdep states with debug_locks off. The debug_locks_off() function did an unconditional atomic xchg to write a 0 value into debug_locks which had already been set to 0. This led to severe cacheline contention in the cacheline that held debug_locks. As debug_locks is being referenced in quite a few different places in the kernel, this greatly slow down the system performance.
To prevent that trashing of debug_locks cacheline, lock_acquired() and lock_contended() now checks the state of debug_locks before proceeding. The debug_locks_off() function is also modified to check debug_locks before calling __debug_locks_off().
--- kernel/locking/lockdep.c | 4 ++-- lib/debug_locks.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-)
kernel/locking/lockdep.c +2 -2
@@ -4066,7 +4066,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip) {unsigned long flags; -if (unlikely (!lock_stat)) +if (unlikely (!lock_stat || !debug_locks))return ;if (unlikely (current->lockdep_recursion))
@@ -4086,7 +4086,7 @@ void lock_acquired(struct lockdep_map *lock, unsigned long ip) {unsigned long flags; -if (unlikely (!lock_stat)) +if (unlikely (!lock_stat || !debug_locks))return ;if (unlikely (current->lockdep_recursion))
lib/debug_locks.c +1 -1
@@ -37,7 +37,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent);*/ int debug_locks_off (void ) { -if (__debug_locks_off ()) { +if (debug_locks &&__debug_locks_off ()) {if (!debug_locks_silent) {console_verbose ();return 1 ;