lock stat kills lock meter for -rt
cover lock stat kills lock meter for -rt
Hello,
I'm back with another annoying announcement and post of my "lock stat" patches for Ingo's 2.6.19-rt14 patch. I want review, comments and eventually inclusion into the -rt.
Changes in this release:
- forward ported to 2,6.19-rt14
- rt_mutex_slowtrylock() path now works with lock stat after an initialization check. Apparently there's a try-lock some where before my lock stat stuff is initialized and it hard crashes the machine on boot. This is fixed now.
- Addes a new field to track adaptive spins in the rtmutex as a future feature.
bill
PATCH 2/5 lock stat kills lock meter for -rt
.c files that have been changed, rtmutex.c, rt.c
============================================================
rt.c +133 -30
@@ -66,6 +66,7 @@#include <linux/plist.h> #include <linux/fs.h> #include <linux/futex.h> +#include <linux/lock_stat.h> #include "rtmutex_common.h"
@@ -75,6 +76,42 @@# include "rtmutex.h" #endif +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_LOCK (a) \ + rt_mutex_lock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_LOCK (a) \ + rt_mutex_lock(a); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE (a, b) \ + rt_mutex_lock_interruptible_with_ip(a, b, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE (a) \ + rt_mutex_lock_interruptible(a, b); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_MUTEX_TRYLOCK (a) \ + rt_mutex_trylock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_MUTEX_TRYLOCK (a) \ + rt_mutex_trylock(a); +#endif + +#ifdef CONFIG_LOCK_STAT +#define __LOCK_STAT_RT_SPIN_LOCK (a) \ + __rt_spin_lock_with_ip(a, \ + (unsigned long) __builtin_return_address(0)) +#else +#define __LOCK_STAT_RT_SPIN_LOCK (a) \ + __rt_spin_lock(a); +#endif +#ifdef CONFIG_PREEMPT_RT /* * Unlock these on crash:
@@ -88,7 +125,8 @@ void zap_rt_locks(void)/* * struct mutex functions */ -void _mutex_init (struct mutex *lock,char *name,struct lock_class_key *key) +void _mutex_init(struct mutex *lock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) {#ifdef CONFIG_DEBUG_LOCK_ALLOC /*
@@ -97,14 +135,15 @@ void _mutex_init(struct mutex *lock, chadebug_check_no_locks_freed ((void *)lock,sizeof (*lock));lockdep_init_map (&lock->dep_map, name, key,0 );#endif -__rt_mutex_init (&lock->lock, name); + __rt_mutex_init(&lock->lock, name __COMMA_LOCK_STAT_NOTE_VARS); }EXPORT_SYMBOL (_mutex_init); void __lockfunc _mutex_lock(struct mutex *lock) {mutex_acquire (&lock->dep_map,0 ,0 , _RET_IP_); -rt_mutex_lock (&lock->lock); + +__LOCK_STAT_RT_MUTEX_LOCK (&lock->lock); }EXPORT_SYMBOL (_mutex_lock);
@@ -124,14 +163,14 @@ void __lockfunc _mutex_lock_nested(struc void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) {mutex_acquire (&lock->dep_map, subclass,0 , _RET_IP_); -rt_mutex_lock (&lock->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&lock->lock); }EXPORT_SYMBOL (_mutex_lock_nested);#endif int __lockfunc _mutex_trylock(struct mutex *lock) { -int ret =rt_mutex_trylock (&lock->lock); +int ret =__LOCK_STAT_RT_MUTEX_TRYLOCK (&lock->lock);if (ret)mutex_acquire (&lock->dep_map,0 ,1 , _RET_IP_);
@@ -152,7 +191,7 @@ int __lockfunc rt_write_trylock(rwlock_t*/ int __lockfunc rt_write_trylock(rwlock_t *rwlock) { -int ret =rt_mutex_trylock (&rwlock->lock); +int ret =__LOCK_STAT_RT_MUTEX_TRYLOCK (&rwlock->lock);if (ret)rwlock_acquire (&rwlock->dep_map,0 ,1 , _RET_IP_);
@@ -179,7 +218,7 @@ int __lockfunc rt_read_trylock(rwlock_t }spin_unlock_irqrestore (&lock->wait_lock, flags); - ret =rt_mutex_trylock (lock); + ret =__LOCK_STAT_RT_MUTEX_TRYLOCK (lock);if (ret)rwlock_acquire_read (&rwlock->dep_map,0 ,1 , _RET_IP_);
@@ -190,7 +229,7 @@ void __lockfunc rt_write_lock(rwlock_t * void __lockfunc rt_write_lock(rwlock_t *rwlock) {rwlock_acquire (&rwlock->dep_map,0 ,0 , _RET_IP_); -__rt_spin_lock (&rwlock->lock); +__LOCK_STAT_RT_SPIN_LOCK (&rwlock->lock); }EXPORT_SYMBOL (rt_write_lock);
@@ -210,11 +249,44 @@ void __lockfunc rt_read_lock(rwlock_t *rreturn ; }spin_unlock_irqrestore (&lock->wait_lock, flags); -__rt_spin_lock (lock); +__LOCK_STAT_RT_SPIN_LOCK (lock); }EXPORT_SYMBOL (rt_read_lock); +#ifdef CONFIG_LOCK_STAT +void __lockfunc rt_write_lock_with_ip(rwlock_t *rwlock, unsigned long ip) +{ +rwlock_acquire (&rwlock->dep_map,0 ,0 , ip); +__rt_spin_lock_with_ip (&rwlock->lock, ip); +} +EXPORT_SYMBOL (rt_write_lock_with_ip); + +void __lockfunc rt_read_lock_with_ip(rwlock_t *rwlock, unsigned long ip) +{ +unsigned long flags; +struct rt_mutex *lock = &rwlock->lock; + +/* +* NOTE: we handle it as a write-lock: +*/ +rwlock_acquire (&rwlock->dep_map,0 ,0 , ip); +/* +* Read locks within the write lock succeed. +*/ +spin_lock_irqsave (&lock->wait_lock, flags); +if (rt_mutex_real_owner (lock) == current) { +spin_unlock_irqrestore (&lock->wait_lock, flags); + rwlock->read_depth++; +return ; + } +spin_unlock_irqrestore (&lock->wait_lock, flags); +__rt_spin_lock_with_ip (lock, ip); +} + +EXPORT_SYMBOL (rt_read_lock_with_ip); +#endif + void __lockfunc rt_write_unlock(rwlock_t *rwlock) {/* NOTE: we always pass in '1' for nested, for simplicity */
@@ -246,7 +318,12 @@ unsigned long __lockfunc rt_write_lock_iunsigned long __lockfunc rt_write_lock_irqsave (rwlock_t *rwlock) { +#ifdef CONFIG_LOCK_STAT +rt_write_lock_with_ip (rwlock, + (unsigned long )__builtin_return_address (0 )); +#else rt_write_lock (rwlock); +#endif return 0 ; }
@@ -254,13 +331,19 @@ unsigned long __lockfunc rt_read_lock_irunsigned long __lockfunc rt_read_lock_irqsave (rwlock_t *rwlock) { +#ifdef CONFIG_LOCK_STAT +rt_read_lock_with_ip (rwlock, + (unsigned long )__builtin_return_address (0 )); +#else rt_read_lock (rwlock); +#endif return 0 ; }EXPORT_SYMBOL (rt_read_lock_irqsave); -void __rt_rwlock_init (rwlock_t *rwlock,char *name,struct lock_class_key *key) +void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) {#ifdef CONFIG_DEBUG_LOCK_ALLOC /*
@@ -269,11 +352,30 @@ void __rt_rwlock_init(rwlock_t *rwlock,debug_check_no_locks_freed ((void *)rwlock,sizeof (*rwlock));lockdep_init_map (&rwlock->dep_map, name, key,0 );#endif -__rt_mutex_init (&rwlock->lock, name); + __rt_mutex_init(&rwlock->lock, name __COMMA_LOCK_STAT_NOTE_VARS); rwlock->read_depth =0 ; }EXPORT_SYMBOL (__rt_rwlock_init); +#ifdef CONFIG_LOCK_STAT +void __rt_rwlock_init_annotated (rwlock_t *rwlock,char *name, +struct lock_class_key *key, +LOCK_STAT_NOTE_PARAM_DECL , +struct lock_stat *lsobject) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC +/* +* Make sure we are not reinitializing a held lock: +*/ +debug_check_no_locks_freed ((void *)rwlock,sizeof (*rwlock)); +lockdep_init_map (&rwlock->dep_map, name, key,0 ); +#endif +__rt_mutex_init_annotated (&rwlock->lock, name,LOCK_STAT_NOTE_VARS , lsobject); + rwlock->read_depth =0 ; +} +EXPORT_SYMBOL (__rt_rwlock_init_annotated); +#endif +/* * rw_semaphores */
@@ -335,7 +437,7 @@ int fastcall rt_down_write_trylock(struc int fastcall rt_down_write_trylock(struct rw_semaphore *rwsem) { -int ret =rt_mutex_trylock (&rwsem->lock); +int ret =__LOCK_STAT_RT_MUTEX_TRYLOCK (&rwsem->lock);if (ret)rwsem_acquire (&rwsem->dep_map,0 ,1 , _RET_IP_);
@@ -346,14 +448,14 @@ void fastcall rt_down_write(struct rw_se void fastcall rt_down_write(struct rw_semaphore *rwsem) {rwsem_acquire (&rwsem->dep_map,0 ,0 , _RET_IP_); -rt_mutex_lock (&rwsem->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&rwsem->lock); }EXPORT_SYMBOL (rt_down_write); void fastcall rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) {rwsem_acquire (&rwsem->dep_map, subclass,0 , _RET_IP_); -rt_mutex_lock (&rwsem->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&rwsem->lock); }EXPORT_SYMBOL (rt_down_write_nested);
@@ -374,7 +476,7 @@ int fastcall rt_down_read_trylock(struct }spin_unlock_irqrestore (&rwsem->lock.wait_lock, flags); - ret =rt_mutex_trylock (&rwsem->lock); + ret =__LOCK_STAT_RT_MUTEX_TRYLOCK (&rwsem->lock);if (ret)rwsem_acquire (&rwsem->dep_map,0 ,1 , _RET_IP_);return ret;
@@ -398,7 +500,7 @@ static void __rt_down_read(struct rw_semreturn ; }spin_unlock_irqrestore (&rwsem->lock.wait_lock, flags); -rt_mutex_lock (&rwsem->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&rwsem->lock); } void fastcall rt_down_read(struct rw_semaphore *rwsem)
@@ -433,14 +535,15 @@ void fastcall rt_down_read_non_owner(str return; }spin_unlock_irqrestore (&rwsem->lock.wait_lock, flags); -rt_mutex_lock (&rwsem->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&rwsem->lock); }EXPORT_SYMBOL (rt_down_read_non_owner);#endif void fastcall __rt_rwsem_init(struct rw_semaphore *rwsem, char *name, -struct lock_class_key *key) + struct lock_class_key *key + __COMMA_LOCK_STAT_NOTE_PARAM_DECL) {#ifdef CONFIG_DEBUG_LOCK_ALLOC /*
@@ -449,7 +552,7 @@ void fastcall __rt_rwsem_init(struct rw_debug_check_no_locks_freed ((void *)rwsem,sizeof (*rwsem));lockdep_init_map (&rwsem->dep_map, name, key,0 );#endif -__rt_mutex_init (&rwsem->lock, name); + __rt_mutex_init(&rwsem->lock, name __COMMA_LOCK_STAT_NOTE_VARS); rwsem->read_depth =0 ; }EXPORT_SYMBOL (__rt_rwsem_init);
@@ -478,7 +581,7 @@ void fastcall rt_down(struct semaphore * void fastcall rt_down(struct semaphore *sem) { -rt_mutex_lock (&sem->lock); +__LOCK_STAT_RT_MUTEX_LOCK (&sem->lock);__down_complete (sem); }EXPORT_SYMBOL (rt_down);
@@ -487,7 +590,7 @@ int fastcall rt_down_interruptible(struc {int ret; - ret =rt_mutex_lock_interruptible (&sem->lock,0 ); + ret =__LOCK_STAT_RT_MUTEX_LOCK_INTERRUPTIBLE (&sem->lock,0 );if (ret)return ret;__down_complete (sem);
@@ -507,7 +610,7 @@ int fastcall rt_down_trylock(struct sema* embedded mutex internally. It would be quite complex to remove * these transient failures so lets try it the simple way first: */ -if (rt_mutex_trylock (&sem->lock)) { +if (__LOCK_STAT_RT_MUTEX_TRYLOCK (&sem->lock)) {__down_complete (sem);return 0 ; }
@@ -534,26 +637,26 @@ EXPORT_SYMBOL(rt_up); }EXPORT_SYMBOL (rt_up); -void fastcall __sema_init(struct semaphore *sem, int val, -char *name,char *file,int line) +void fastcall __sema_init(struct semaphore *sem, int val, char *name + __COMMA_LOCK_STAT_FN_DECL,char *_file,int _line) {atomic_set (&sem->count, val);switch (val) {case 0 : -__rt_mutex_init (&sem->lock, name); -rt_mutex_lock (&sem->lock); + __rt_mutex_init(&sem->lock, name __COMMA_LOCK_STAT_NOTE_VARS); +rt_mutex_lock_with_ip (&sem->lock, (unsigned long )__builtin_return_address (0 ));break ;default : -__rt_mutex_init (&sem->lock, name); + __rt_mutex_init(&sem->lock, name __COMMA_LOCK_STAT_NOTE_VARS);break ; } }EXPORT_SYMBOL (__sema_init); -void fastcall __init_MUTEX(struct semaphore *sem, char *name, char *file, -int line) +void fastcall __init_MUTEX(struct semaphore *sem, char *name + __COMMA_LOCK_STAT_FN_DECL,char *_file,int _line) { -__sema_init (sem,1 , name, file, line); + __sema_init(sem, 1, name __COMMA_LOCK_STAT_FN_VAR, _file, _line); }EXPORT_SYMBOL (__init_MUTEX);
============================================================
rtmutex.c +236 -27
497 lines not on this page · raw message