extend synchro-test module to test spinlocks too
Subsystems (2)
cover extend synchro-test module to test spinlocks too
I wanted a synthetic test to help me understand the performance of Rik's proposed spinlock proportional backoff patches, and the synchro-test in andrew's mmotm tree looked like an obvious candidate to be extended, so I ended up with the following couple patches.
I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?
Michel Lespinasse (2):
add spinlock test to synchro-test module
Document default load and interval values in synchro-test module
Documentation/synchro-test.txt | 4 +- kernel/synchro-test.c | 76 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 8 deletions(-)
Signature · 1 line
1.7.7.3
PATCH 1/2 add spinlock test to synchro-test module
--- kernel/synchro-test.c | 76 +++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 70 insertions(+), 6 deletions(-)
kernel/synchro-test.c +70 -6
@@ -41,6 +41,7 @@# define VALIDATE_OPERATORS #endif +static int numsp;static int nummx;static int numsm, seminit =4 ;static int numrd, numwr, numdg;
@@ -54,6 +55,9 @@ MODULE_LICENSE("GPL");module_param_named (v, verbose, int,0 );MODULE_PARM_DESC (verbose,"Verbosity" ); +module_param_named (sp, numsp, int,0 ); +MODULE_PARM_DESC (numsp,"Number of spinlock threads" ); +module_param_named (mx, nummx, int,0 );MODULE_PARM_DESC (nummx,"Number of mutex threads" );
@@ -85,6 +89,7 @@ module_param(do_sched, int, 0);MODULE_PARM_DESC (do_sched,"True if each thread should schedule regularly" );/* the semaphores under test */ +static spinlock_t ____cacheline_aligned spinlock;static struct mutex ____cacheline_aligned mutex;static struct semaphore ____cacheline_aligned sem;static struct rw_semaphore ____cacheline_aligned rwsem;
@@ -92,18 +97,21 @@ static struct rw_semaphore ____cacheline_aligned rwsem;static atomic_t ____cacheline_aligned do_stuff =ATOMIC_INIT (0 );#ifdef VALIDATE_OPERATORS +static atomic_t ____cacheline_aligned spinlocks =ATOMIC_INIT (0 );static atomic_t ____cacheline_aligned mutexes =ATOMIC_INIT (0 );static atomic_t ____cacheline_aligned semaphores =ATOMIC_INIT (0 );static atomic_t ____cacheline_aligned readers =ATOMIC_INIT (0 );static atomic_t ____cacheline_aligned writers =ATOMIC_INIT (0 );#endif +static unsigned int ____cacheline_aligned spinlocks_taken [MAX_THREADS ];static unsigned int ____cacheline_aligned mutexes_taken [MAX_THREADS ];static unsigned int ____cacheline_aligned semaphores_taken [MAX_THREADS ];static unsigned int ____cacheline_aligned reads_taken [MAX_THREADS ];static unsigned int ____cacheline_aligned writes_taken [MAX_THREADS ];static unsigned int ____cacheline_aligned downgrades_taken [MAX_THREADS ]; +static struct completion ____cacheline_aligned sp_comp[MAX_THREADS ];static struct completion ____cacheline_aligned mx_comp[MAX_THREADS ];static struct completion ____cacheline_aligned sm_comp[MAX_THREADS ];static struct completion ____cacheline_aligned rd_comp[MAX_THREADS ];
@@ -130,6 +138,23 @@ do { \#define CHECK (var, cond, val) do {} while(0)#endif +static inline void do_spin_lock (unsigned int N ) +{ +spin_lock (&spinlock); + +ACCOUNT (spinlocks,N ); +TRACK (spinlocks, inc); + CHECK(spinlocks, ==, 1); +} + +static inline void do_spin_unlock (unsigned int N ) +{ + CHECK(spinlocks, ==, 1); +TRACK (spinlocks, dec); + +spin_unlock (&spinlock); +} +static inline void do_mutex_lock (unsigned int N ) {mutex_lock (&mutex);
@@ -221,6 +246,27 @@ static inline void sched(void)schedule (); } +static int spinlocker (void *arg) +{ +unsigned int N = (unsigned long ) arg; + +set_user_nice (current,19 ); + +while (atomic_read (&do_stuff)) { +do_spin_lock (N ); +if (load) +udelay (load); +do_spin_unlock (N ); +sched (); +if (interval) +udelay (interval); + } + +if (verbose >=2 ) +printk ("%s: done\n" , current->comm); +complete_and_exit (&sp_comp[N ],0 ); +} +static int mutexer (void *arg) {unsigned int N = (unsigned long ) arg;
@@ -388,9 +434,11 @@ static unsigned int total(const char *what, unsigned int counts[], int num)static int __init do_tests (void ) {unsigned long loop; -unsigned int mutex_total, sem_total, rd_total, wr_total, dg_total; +unsigned int spinlock_total, mutex_total, sem_total; +unsigned int rd_total, wr_total, dg_total; - if (nummx < 0 || nummx > MAX_THREADS || + if (numsp < 0 || numsp > MAX_THREADS || + nummx < 0 || nummx > MAX_THREADS || numsm < 0 || numsm > MAX_THREADS || numrd < 0 || numrd > MAX_THREADS || numwr < 0 || numwr > MAX_THREADS ||
@@ -404,12 +452,12 @@ static int __init do_tests(void)return -ERANGE ; } -if ((nummx | numsm | numrd | numwr | numdg) ==0 ) { +if ((numsp | nummx | numsm | numrd | numwr | numdg) ==0 ) {int num =num_online_cpus ();if (num >MAX_THREADS ) num =MAX_THREADS ; - nummx = numsm = numrd = numwr = numdg = num; + numsp = nummx = numsm = numrd = numwr = numdg = num; load =1 ; interval =1 ;
@@ -420,6 +468,7 @@ static int __init do_tests(void)if (verbose)printk ("\nStarting synchronisation primitive tests...\n" ); +spin_lock_init (&spinlock);mutex_init (&mutex);sema_init (&sem, seminit);init_rwsem (&rwsem);
@@ -427,6 +476,12 @@ static int __init do_tests(void)/* kick off all the children */ for (loop =0 ; loop <MAX_THREADS ; loop++) { +if (loop < numsp) { +init_completion (&sp_comp[loop]); +kthread_run (spinlocker, (void *) loop, +"Spinlock%lu" , loop); + } +if (loop < nummx) {init_completion (&mx_comp[loop]);kthread_run (mutexer, (void *) loop,"Mutex%lu" , loop);
@@ -460,6 +515,9 @@ static int __init do_tests(void)add_timer (&timer);/* now wait until it's all done */ +for (loop =0 ; loop < numsp; loop++) +wait_for_completion (&sp_comp[loop]); +for (loop =0 ; loop < nummx; loop++)wait_for_completion (&mx_comp[loop]);
@@ -478,10 +536,14 @@ static int __init do_tests(void)atomic_set (&do_stuff,0 );del_timer (&timer); +if (spin_is_locked (&spinlock)) +printk (KERN_ERR "Spinlock is still locked!\n" ); +if (mutex_is_locked (&mutex))printk (KERN_ERR "Mutex is still locked!\n" );/* count up */ + spinlock_total =total ("SP " , spinlocks_taken, numsp); mutex_total =total ("MTX" , mutexes_taken, nummx); sem_total =total ("SEM" , semaphores_taken, numsm); rd_total =total ("RD " , reads_taken, numrd);
@@ -490,6 +552,7 @@ static int __init do_tests(void)/* print the results */ if (verbose) { +printk ("spinlocks taken: %u\n" , spinlock_total);printk ("mutexes taken: %u\n" , mutex_total);printk ("semaphores taken: %u\n" , sem_total);printk ("reads taken: %u\n" , rd_total);
@@ -501,10 +564,11 @@ static int __init do_tests(void)sprintf (buf,"%d/%d" , interval, load); - printk("%3d %3d %3d %3d %3d %c %5s %9u %9u %9u %9u %9u\n", - nummx, numsm, numrd, numwr, numdg, + printk("%3d %3d %3d %3d %3d %3d %c %5s %9u %9u %9u %9u %9u %9u\n", + numsp, nummx, numsm, numrd, numwr, numdg, do_sched ? 's' : '-', buf, + spinlock_total, mutex_total, sem_total, rd_total,
Signature · 1 line
1.7.7.3
PATCH 2/2 Document default load and interval values in synchro-test module
In reply to Michel LespinasseThe synchro-test module default parameters are to keep the lock for 2uS and wait 2uS between release and the next attempted acquisition. Having the documentation wrong on this point was quite confusing !
--- Documentation/synchro-test.txt | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)
Documentation/synchro-test.txt +2 -2
@@ -42,12 +42,12 @@ The available arguments are: (*) load=N - Each thread delays for N uS whilst holding the lock. The default is 0. + Each thread delays for N uS whilst holding the lock. The default is 2. (*) interval=N Each thread delays for N uS whilst not holding the lock. The default - is 0. + is 2. (*) do_sched=1
Signature · 1 line
1.7.7.3
Andrew Morton
maintainerQuoted 4 lines · On Sun, 30 Dec 2012 18:47:12 -0800 Michel Lespinasse <[email protected]> wrote:
I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?
iirc, we decided that synchro-test wasn't valuable/important enough to put into mainline. So it has been living in -mm for, umm, seven years.
Let's revisit that decision. What do you think is the case for inclusion?
Here it is. It will need a full re-review.
From: David Howells <[email protected]>
Subject: mutex subsystem, synchro-test module
The attached patch adds a module for testing and benchmarking mutexes, semaphores and R/W semaphores.
Using it is simple:
insmod synchro-test.ko <args>
It will exit with error ENOANO after running the tests and printing the results to the kernel console log.
The available arguments are:
(*) mx=N
Start up to N mutex thrashing threads, where N is at most 20. All will try and thrash the same mutex.
(*) sm=N
Start up to N counting semaphore thrashing threads, where N is at most
20. All will try and thrash the same semaphore.
(*) ism=M
Initialise the counting semaphore with M, where M is any positive integer greater than zero. The default is 4.
(*) rd=N
(*) wr=O
(*) dg=P
Start up to N reader thrashing threads, O writer thrashing threads and P downgrader thrashing threads, where N, O and P are at most 20 apiece. All will try and thrash the same read/write semaphore.
(*) elapse=N
Run the tests for N seconds. The default is 5.
(*) load=N
Each thread delays for N uS whilst holding the lock. The dfault is 0.
(*) interval=N
Each thread delays for N uS whilst not holding the lock. The default is 0.
(*) do_sched=1
Each thread will call schedule if required after each iteration.
(*) v=1
Print more verbose information, including a thread iteration distribution list.
The module should be enabled by turning on CONFIG_DEBUG_SYNCHRO_TEST to "m".
---
Documentation/synchro-test.txt | 59 +++ kernel/Makefile | 1 kernel/synchro-test.c | 586 +++++++++++++++++++++++++++++++ lib/Kconfig.debug | 14 4 files changed, 660 insertions(+)
Documentation/synchro-test.txt new file +59 -0
@@ -0,0 +1,59 @@ +The synchro-test.ko module can be used for testing and benchmarking mutexes, +semaphores and R/W semaphores. + +The module is compiled by setting CONFIG_DEBUG_SYNCHRO_TEST to "m" when +configuring the kernel. + +Using it is simple: + + insmod synchro-test.ko <args> + +It will exit with error ENOANO after running the tests and printing the +results to the kernel console log. + +The available arguments are: + + (*) mx=N + + Start up to N mutex thrashing threads, where N is at most 20. All will + try and thrash the same mutex. + + (*) sm=N + + Start up to N counting semaphore thrashing threads, where N is at most + 20. All will try and thrash the same semaphore. + + (*) ism=M + + Initialise the counting semaphore with M, where M is any positive + integer greater than zero. The default is 4. + + (*) rd=N + (*) wr=O + (*) dg=P + + Start up to N reader thrashing threads, O writer thrashing threads and + P downgrader thrashing threads, where N, O and P are at most 20 + apiece. All will try and thrash the same read/write semaphore. + + (*) elapse=N + + Run the tests for N seconds. The default is 5. + + (*) load=N + + Each thread delays for N uS whilst holding the lock. The default is 2. + + (*) interval=N + + Each thread delays for N uS whilst not holding the lock. The default + is 2. + + (*) do_sched=1 + + Each thread will call schedule if required after each iteration. + + (*) v=1 + + Print more verbose information, including a thread iteration + distribution list.
kernel/Makefile renamed from kernel/Makefile~mutex-subsystem-synchro-test-module +1 -0
@@ -66,6 +66,7 @@ obj-$(CONFIG_CPUSETS) += cpuset.o obj-$(CONFIG_UTS_NS) += utsname.o obj-$(CONFIG_USER_NS) += user_namespace.o obj-$(CONFIG_PID_NS) += pid_namespace.o +obj-$(CONFIG_DEBUG_SYNCHRO_TEST) += synchro-test.o obj-$(CONFIG_IKCONFIG) += configs.o obj-$(CONFIG_RESOURCE_COUNTERS) += res_counter.o obj-$(CONFIG_SMP) += stop_machine.o
kernel/synchro-test.c new file +586 -0
586 lines not on this page · raw message
lib/Kconfig.debug renamed from lib/Kconfig.debug~mutex-subsystem-synchro-test-module +14 -0
@@ -931,6 +931,20 @@ config FRAME_POINTER larger and slower, but it gives very useful debugging information in case of kernel bugs. (precise oopses/stacktraces/warnings) +config DEBUG_SYNCHRO_TEST + tristate "Synchronisation primitive testing module" + depends on DEBUG_KERNEL + default n + help + This option provides a kernel module that can thrash the sleepable + synchronisation primitives (mutexes and semaphores). + + You should say N or M here. Whilst the module can be built in, it's + not recommended as it requires module parameters supplying to get it + to do anything. + + See Documentation/synchro-test.txt. + config BOOT_PRINTK_DELAY bool "Delay each boot printk message by N milliseconds" depends on DEBUG_KERNEL && PRINTK && GENERIC_CALIBRATE_DELAY
_
Quoted 8 lines · Andrew Morton
Quoted 4 lines · On Sun, 30 Dec 2012 18:47:12 -0800 Michel Lespinasse <[email protected]> wrote:
I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?
Quoted 4 lines
iirc, we decided that synchro-test wasn't valuable/important enough to put into mainline. So it has been living in -mm for, umm, seven years.
Let's revisit that decision. What do you think is the case for inclusion?
Sorry for the late reply. I agree that this isn't useful on an everyday basis, however such a test could be useful whenever some locking discussions come up, such as with the spinlock performance discussions when I started this thread, or the rwsem write lock stealing right now. Also, I think we've accepted a lot more non-core-kernel things over the last 7 years, so the bar may be lower now.
Quoted 1 line
Here it is. It will need a full re-review.
I think the first thing we may want to do is ask David if he has an updated version - if he's been making changes, it would be rude to him to check in a 7 year old version of his code.
Quoted 13 lines
From: David Howells <[email protected]>
Subject: mutex subsystem, synchro-test moduleThe attached patch adds a module for testing and benchmarking mutexes, semaphores and R/W semaphores.
Using it is simple:
insmod synchro-test.ko <args>
It will exit with error ENOANO after running the tests and printing the results to the kernel console log.
Quoted 6 lines
The available arguments are:
(*) mx=N
Start up to N mutex thrashing threads, where N is at most 20. All will try and thrash the same mutex.
Actually N is at most 64
Quoted 4 lines
(*) sm=N
Start up to N counting semaphore thrashing threads, where N is at most
20. All will try and thrash the same semaphore.
N is at most 64
Quoted 5 lines
(*) ism=M
Initialise the counting semaphore with M, where M is any positive integer greater than zero. The default is 4.
Quoted 7 lines
(*) rd=N
(*) wr=O
(*) dg=PStart up to N reader thrashing threads, O writer thrashing threads and P downgrader thrashing threads, where N, O and P are at most 20 apiece. All will try and thrash the same read/write semaphore.
N, O and P are at most 64
Quoted 7 lines
(*) elapse=N
Run the tests for N seconds. The default is 5.
(*) load=N
Each thread delays for N uS whilst holding the lock. The dfault is 0.
default is 2 (I had actually sent a change to correct the doc here)
Quoted 4 lines
(*) interval=N
Each thread delays for N uS whilst not holding the lock. The default is 0.
default is 2 (I had actually sent a change to correct the doc here)
Quoted 23 lines
(*) do_sched=1
Each thread will call schedule if required after each iteration.
(*) v=1
Print more verbose information, including a thread iteration distribution list.
The module should be enabled by turning on
CONFIG_DEBUG_SYNCHRO_TESTto "m".[[email protected]: fix build errors, add <sched.h> header file][[email protected]: remove smp_lock.h inclusion][[email protected]: kill daemonize() calls][[email protected]: fix printk format warrnings][[email protected]: add spinlock test][[email protected]: document default load and interval values]Signed-off-by David Howells <[email protected]>Signed-off-by Ingo Molnar <[email protected]>Signed-off-by Adrian Bunk <[email protected]>Signed-off-by Randy Dunlap <[email protected]>Signed-off-by Michel Lespinasse <[email protected]>Signed-off-by Andrew Morton <[email protected]>
Quoted 7 lines
---
Documentation/synchro-test.txt | 59 +++ kernel/Makefile | 1 kernel/synchro-test.c | 586 +++++++++++++++++++++++++++++++ lib/Kconfig.debug | 14 4 files changed, 660 insertions(+)
It's unclear if synchro-test.c should be in kernel/ - I think similar tests nowadays tend to be in lib/ instead, though not always :)
Quoted 13 lines
diff -puN
/dev/nullDocumentation/synchro-test.txtDocumentation/synchro-test.txt new file +9
-0@@ -0,0 +1,59 @@ +The synchro-test.ko module can be used for testing and benchmarking mutexes, +semaphores and R/W semaphores. + +The module is compiled by setting CONFIG_DEBUG_SYNCHRO_TEST to "m" when +configuring the kernel. + +Using it is simple: + + insmod synchro-test.ko <args>
Documentation/synchro-test.txt line 19
Quoted 10 lines
Documentation/synchro-test.txt new file +10
-0+ +It will exit with error ENOANO after running the tests and printing the +results to the kernel console log. + +The available arguments are: + + (*) mx=N + + Start up to N mutex thrashing threads, where N is at most 20. All will + try and thrash the same mutex.
max 64
Documentation/synchro-test.txt line 24
Quoted 4 lines
Documentation/synchro-test.txt new file +4
-0+ (*) sm=N + + Start up to N counting semaphore thrashing threads, where N is at most + 20. All will try and thrash the same semaphore.
max 64
Documentation/synchro-test.txt line 37
Quoted 12 lines
Documentation/synchro-test.txt new file +12
-0+ (*) ism=M + + Initialise the counting semaphore with M, where M is any positive + integer greater than zero. The default is 4. + + (*) rd=N + (*) wr=O + (*) dg=P + + Start up to N reader thrashing threads, O writer thrashing threads and + P downgrader thrashing threads, where N, O and P are at most 20 + apiece. All will try and thrash the same read/write semaphore.
max 64
Quoted 11 lines
Documentation/synchro-test.txt new file +11
-0+ (*) elapse=N + + Run the tests for N seconds. The default is 5. + + (*) load=N + + Each thread delays for N uS whilst holding the lock. The default is 2. + + (*) interval=N + + Each thread delays for N uS whilst not holding the lock. The default
Documentation/synchro-test.txt line 59
Quoted 10 lines
Documentation/synchro-test.txt new file +10
-0+ is 2. + + (*) do_sched=1 + + Each thread will call schedule if required after each iteration. + + (*) v=1 + + Print more verbose information, including a thread iteration + distribution list.
Other than that I'm happy enough with the rest of this code :)
Signature · 2 lines
Michel "Walken" Lespinasse A program is never fully debugged until the last user dies.