Skip to contentSkip to conversation

extend synchro-test module to test spinlocks too

PATCH In review Michel Lespinasse 2 participants 5 messages Dec 31, 2012 – Feb 3, 2013

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover extend synchro-test module to test spinlocks too

I wanted a synthetic test to help me understand the performance of Rik's proposed spinlock proportional backoff patches, and the synchro-test in andrew's mmotm tree looked like an obvious candidate to be extended, so I ended up with the following couple patches.

I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?

Michel Lespinasse (2):
add spinlock test to synchro-test module
Document default load and interval values in synchro-test module

 Documentation/synchro-test.txt |    4 +-
 kernel/synchro-test.c          |   76 ++++++++++++++++++++++++++++++++++++---
 2 files changed, 72 insertions(+), 8 deletions(-)
Signature · 1 line
1.7.7.3
#1 lore raw

PATCH 1/2 add spinlock test to synchro-test module

---
 kernel/synchro-test.c |   76 +++++++++++++++++++++++++++++++++++++++++++++----
 1 files changed, 70 insertions(+), 6 deletions(-)
kernel/synchro-test.c +70 -6
index 76e3ad39505f..9abfa955d69e 100644
@@ -41,6 +41,7 @@
 # define VALIDATE_OPERATORS
 #endif
 
+static int numsp;
 static int nummx;
 static int numsm, seminit = 4;
 static int numrd, numwr, numdg;
@@ -54,6 +55,9 @@ MODULE_LICENSE("GPL");
 module_param_named(v, verbose, int, 0);
 MODULE_PARM_DESC(verbose, "Verbosity");
 
+module_param_named(sp, numsp, int, 0);
+MODULE_PARM_DESC(numsp, "Number of spinlock threads");
+
 module_param_named(mx, nummx, int, 0);
 MODULE_PARM_DESC(nummx, "Number of mutex threads");
 
@@ -85,6 +89,7 @@ module_param(do_sched, int, 0);
 MODULE_PARM_DESC(do_sched, "True if each thread should schedule regularly");
 
 /* the semaphores under test */
+static spinlock_t ____cacheline_aligned spinlock;
 static struct mutex ____cacheline_aligned mutex;
 static struct semaphore ____cacheline_aligned sem;
 static struct rw_semaphore ____cacheline_aligned rwsem;
@@ -92,18 +97,21 @@ static struct rw_semaphore ____cacheline_aligned rwsem;
 static atomic_t ____cacheline_aligned do_stuff		= ATOMIC_INIT(0);
 
 #ifdef VALIDATE_OPERATORS
+static atomic_t ____cacheline_aligned spinlocks		= ATOMIC_INIT(0);
 static atomic_t ____cacheline_aligned mutexes		= ATOMIC_INIT(0);
 static atomic_t ____cacheline_aligned semaphores	= ATOMIC_INIT(0);
 static atomic_t ____cacheline_aligned readers		= ATOMIC_INIT(0);
 static atomic_t ____cacheline_aligned writers		= ATOMIC_INIT(0);
 #endif
 
+static unsigned int ____cacheline_aligned spinlocks_taken	[MAX_THREADS];
 static unsigned int ____cacheline_aligned mutexes_taken		[MAX_THREADS];
 static unsigned int ____cacheline_aligned semaphores_taken	[MAX_THREADS];
 static unsigned int ____cacheline_aligned reads_taken		[MAX_THREADS];
 static unsigned int ____cacheline_aligned writes_taken		[MAX_THREADS];
 static unsigned int ____cacheline_aligned downgrades_taken	[MAX_THREADS];
 
+static struct completion ____cacheline_aligned sp_comp[MAX_THREADS];
 static struct completion ____cacheline_aligned mx_comp[MAX_THREADS];
 static struct completion ____cacheline_aligned sm_comp[MAX_THREADS];
 static struct completion ____cacheline_aligned rd_comp[MAX_THREADS];
@@ -130,6 +138,23 @@ do {									\
 #define CHECK(var, cond, val)	do {} while(0)
 #endif
 
+static inline void do_spin_lock(unsigned int N)
+{
+	spin_lock(&spinlock);
+
+	ACCOUNT(spinlocks, N);
+	TRACK(spinlocks, inc);
+	CHECK(spinlocks, ==, 1);
+}
+
+static inline void do_spin_unlock(unsigned int N)
+{
+	CHECK(spinlocks, ==, 1);
+	TRACK(spinlocks, dec);
+
+	spin_unlock(&spinlock);
+}
+
 static inline void do_mutex_lock(unsigned int N)
 {
 	mutex_lock(&mutex);
@@ -221,6 +246,27 @@ static inline void sched(void)
 		schedule();
 }
 
+static int spinlocker(void *arg)
+{
+	unsigned int N = (unsigned long) arg;
+
+	set_user_nice(current, 19);
+
+	while (atomic_read(&do_stuff)) {
+		do_spin_lock(N);
+		if (load)
+			udelay(load);
+		do_spin_unlock(N);
+		sched();
+		if (interval)
+			udelay(interval);
+	}
+
+	if (verbose >= 2)
+		printk("%s: done\n", current->comm);
+	complete_and_exit(&sp_comp[N], 0);
+}
+
 static int mutexer(void *arg)
 {
 	unsigned int N = (unsigned long) arg;
@@ -388,9 +434,11 @@ static unsigned int total(const char *what, unsigned int counts[], int num)
 static int __init do_tests(void)
 {
 	unsigned long loop;
-	unsigned int mutex_total, sem_total, rd_total, wr_total, dg_total;
+	unsigned int spinlock_total, mutex_total, sem_total;
+	unsigned int rd_total, wr_total, dg_total;
 
-	if (nummx < 0 || nummx > MAX_THREADS ||
+	if (numsp < 0 || numsp > MAX_THREADS ||
+	    nummx < 0 || nummx > MAX_THREADS ||
 	    numsm < 0 || numsm > MAX_THREADS ||
 	    numrd < 0 || numrd > MAX_THREADS ||
 	    numwr < 0 || numwr > MAX_THREADS ||
@@ -404,12 +452,12 @@ static int __init do_tests(void)
 		return -ERANGE;
 	}
 
-	if ((nummx | numsm | numrd | numwr | numdg) == 0) {
+	if ((numsp | nummx | numsm | numrd | numwr | numdg) == 0) {
 		int num = num_online_cpus();
 
 		if (num > MAX_THREADS)
 			num = MAX_THREADS;
-		nummx = numsm = numrd = numwr = numdg = num;
+		numsp = nummx = numsm = numrd = numwr = numdg = num;
 
 		load = 1;
 		interval = 1;
@@ -420,6 +468,7 @@ static int __init do_tests(void)
 	if (verbose)
 		printk("\nStarting synchronisation primitive tests...\n");
 
+	spin_lock_init(&spinlock);
 	mutex_init(&mutex);
 	sema_init(&sem, seminit);
 	init_rwsem(&rwsem);
@@ -427,6 +476,12 @@ static int __init do_tests(void)
 
 	/* kick off all the children */
 	for (loop = 0; loop < MAX_THREADS; loop++) {
+		if (loop < numsp) {
+			init_completion(&sp_comp[loop]);
+			kthread_run(spinlocker, (void *) loop,
+				    "Spinlock%lu", loop);
+		}
+
 		if (loop < nummx) {
 			init_completion(&mx_comp[loop]);
 			kthread_run(mutexer, (void *) loop, "Mutex%lu", loop);
@@ -460,6 +515,9 @@ static int __init do_tests(void)
 	add_timer(&timer);
 
 	/* now wait until it's all done */
+	for (loop = 0; loop < numsp; loop++)
+		wait_for_completion(&sp_comp[loop]);
+
 	for (loop = 0; loop < nummx; loop++)
 		wait_for_completion(&mx_comp[loop]);
 
@@ -478,10 +536,14 @@ static int __init do_tests(void)
 	atomic_set(&do_stuff, 0);
 	del_timer(&timer);
 
+	if (spin_is_locked(&spinlock))
+		printk(KERN_ERR "Spinlock is still locked!\n");
+
 	if (mutex_is_locked(&mutex))
 		printk(KERN_ERR "Mutex is still locked!\n");
 
 	/* count up */
+	spinlock_total	= total("SP ", spinlocks_taken, numsp);
 	mutex_total	= total("MTX", mutexes_taken, nummx);
 	sem_total	= total("SEM", semaphores_taken, numsm);
 	rd_total	= total("RD ", reads_taken, numrd);
@@ -490,6 +552,7 @@ static int __init do_tests(void)
 
 	/* print the results */
 	if (verbose) {
+		printk("spinlocks taken: %u\n", spinlock_total);
 		printk("mutexes taken: %u\n", mutex_total);
 		printk("semaphores taken: %u\n", sem_total);
 		printk("reads taken: %u\n", rd_total);
@@ -501,10 +564,11 @@ static int __init do_tests(void)
 
 		sprintf(buf, "%d/%d", interval, load);
 
-		printk("%3d %3d %3d %3d %3d %c %5s %9u %9u %9u %9u %9u\n",
-		       nummx, numsm, numrd, numwr, numdg,
+		printk("%3d %3d %3d %3d %3d %3d %c %5s %9u %9u %9u %9u %9u %9u\n",
+		       numsp, nummx, numsm, numrd, numwr, numdg,
 		       do_sched ? 's' : '-',
 		       buf,
+		       spinlock_total,
 		       mutex_total,
 		       sem_total,
 		       rd_total,
Signature · 1 line
1.7.7.3
#2 lore raw

PATCH 2/2 Document default load and interval values in synchro-test module

In reply to Michel Lespinasse

The synchro-test module default parameters are to keep the lock for 2uS and wait 2uS between release and the next attempted acquisition. Having the documentation wrong on this point was quite confusing !

---
 Documentation/synchro-test.txt |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)
Documentation/synchro-test.txt +2 -2
index 3d6cdda7bb79..554709478e1f 100644
@@ -42,12 +42,12 @@ The available arguments are:
 
  (*) load=N
 
-	Each thread delays for N uS whilst holding the lock. The default is 0.
+	Each thread delays for N uS whilst holding the lock. The default is 2.
 
  (*) interval=N
 
 	Each thread delays for N uS whilst not holding the lock. The default
-	is 0.
+	is 2.
 
  (*) do_sched=1
 
Signature · 1 line
1.7.7.3
maintainer
#3 lore raw
In reply to Michel Lespinasse
Quoted 4 lines · On Sun, 30 Dec 2012 18:47:12 -0800 Michel Lespinasse <[email protected]> wrote:

I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?

iirc, we decided that synchro-test wasn't valuable/important enough to put into mainline. So it has been living in -mm for, umm, seven years.

Let's revisit that decision. What do you think is the case for inclusion?

Here it is. It will need a full re-review.

From: David Howells <[email protected]>
Subject: mutex subsystem, synchro-test module

The attached patch adds a module for testing and benchmarking mutexes, semaphores and R/W semaphores.

Using it is simple:

insmod synchro-test.ko <args>

It will exit with error ENOANO after running the tests and printing the results to the kernel console log.

The available arguments are:

(*) mx=N

Start up to N mutex thrashing threads, where N is at most 20. All will try and thrash the same mutex.

(*) sm=N

Start up to N counting semaphore thrashing threads, where N is at most
20. All will try and thrash the same semaphore.

(*) ism=M

Initialise the counting semaphore with M, where M is any positive integer greater than zero. The default is 4.

(*) rd=N
(*) wr=O
(*) dg=P

Start up to N reader thrashing threads, O writer thrashing threads and P downgrader thrashing threads, where N, O and P are at most 20 apiece. All will try and thrash the same read/write semaphore.

(*) elapse=N

Run the tests for N seconds. The default is 5.

(*) load=N

Each thread delays for N uS whilst holding the lock. The dfault is 0.

(*) interval=N

	Each thread delays for N uS whilst not holding the lock. The default
	is 0.

(*) do_sched=1

Each thread will call schedule if required after each iteration.

(*) v=1

Print more verbose information, including a thread iteration distribution list.

The module should be enabled by turning on CONFIG_DEBUG_SYNCHRO_TEST to "m".

[[email protected]: fix build errors, add <sched.h> header file]
[[email protected]: remove smp_lock.h inclusion]
[[email protected]: kill daemonize() calls]
[[email protected]: fix printk format warrnings]
[[email protected]: add spinlock test]
[[email protected]: document default load and interval values]

---

 Documentation/synchro-test.txt |   59 +++
 kernel/Makefile                |    1 
 kernel/synchro-test.c          |  586 +++++++++++++++++++++++++++++++
 lib/Kconfig.debug              |   14 
 4 files changed, 660 insertions(+)
Documentation/synchro-test.txt new file +59 -0
diff -puN /dev/null Documentation/synchro-test.txt
@@ -0,0 +1,59 @@
+The synchro-test.ko module can be used for testing and benchmarking mutexes,
+semaphores and R/W semaphores.
+
+The module is compiled by setting CONFIG_DEBUG_SYNCHRO_TEST to "m" when
+configuring the kernel.
+
+Using it is simple:
+
+	insmod synchro-test.ko <args>
+
+It will exit with error ENOANO after running the tests and printing the
+results to the kernel console log.
+
+The available arguments are:
+
+ (*) mx=N
+
+	Start up to N mutex thrashing threads, where N is at most 20. All will
+	try and thrash the same mutex.
+
+ (*) sm=N
+
+	Start up to N counting semaphore thrashing threads, where N is at most
+	20. All will try and thrash the same semaphore.
+
+ (*) ism=M
+
+	Initialise the counting semaphore with M, where M is any positive
+	integer greater than zero. The default is 4.
+
+ (*) rd=N
+ (*) wr=O
+ (*) dg=P
+
+	Start up to N reader thrashing threads, O writer thrashing threads and
+	P downgrader thrashing threads, where N, O and P are at most 20
+	apiece. All will try and thrash the same read/write semaphore.
+
+ (*) elapse=N
+
+	Run the tests for N seconds. The default is 5.
+
+ (*) load=N
+
+	Each thread delays for N uS whilst holding the lock. The default is 2.
+
+ (*) interval=N
+
+	Each thread delays for N uS whilst not holding the lock. The default
+	is 2.
+
+ (*) do_sched=1
+
+	Each thread will call schedule if required after each iteration.
+
+ (*) v=1
+
+	Print more verbose information, including a thread iteration
+	distribution list.
kernel/Makefile renamed from kernel/Makefile~mutex-subsystem-synchro-test-module +1 -0
diff -puN kernel/Makefile~mutex-subsystem-synchro-test-module kernel/Makefile
@@ -66,6 +66,7 @@ obj-$(CONFIG_CPUSETS) += cpuset.o
 obj-$(CONFIG_UTS_NS) += utsname.o
 obj-$(CONFIG_USER_NS) += user_namespace.o
 obj-$(CONFIG_PID_NS) += pid_namespace.o
+obj-$(CONFIG_DEBUG_SYNCHRO_TEST) += synchro-test.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_RESOURCE_COUNTERS) += res_counter.o
 obj-$(CONFIG_SMP) += stop_machine.o
kernel/synchro-test.c new file +586 -0

586 lines not on this page · raw message

lib/Kconfig.debug renamed from lib/Kconfig.debug~mutex-subsystem-synchro-test-module +14 -0
diff -puN lib/Kconfig.debug~mutex-subsystem-synchro-test-module lib/Kconfig.debug
@@ -931,6 +931,20 @@ config FRAME_POINTER
 	  larger and slower, but it gives very useful debugging information
 	  in case of kernel bugs. (precise oopses/stacktraces/warnings)
 
+config DEBUG_SYNCHRO_TEST
+	tristate "Synchronisation primitive testing module"
+	depends on DEBUG_KERNEL
+	default n
+	help
+	  This option provides a kernel module that can thrash the sleepable
+	  synchronisation primitives (mutexes and semaphores).
+
+	  You should say N or M here. Whilst the module can be built in, it's
+	  not recommended as it requires module parameters supplying to get it
+	  to do anything.
+
+	  See Documentation/synchro-test.txt.
+
 config BOOT_PRINTK_DELAY
 	bool "Delay each boot printk message by N milliseconds"
 	depends on DEBUG_KERNEL && PRINTK && GENERIC_CALIBRATE_DELAY

_

#4 lore raw
Quoted 8 lines · Andrew Morton
Quoted 4 lines · On Sun, 30 Dec 2012 18:47:12 -0800 Michel Lespinasse <[email protected]> wrote:

I'm not sure whats' the back story with synchro-test though - they seem to have been stuck in andrew's tree for a very long time now. Is there any reason delaying their inclusion or is it just that nobody's been pushing for them ?

Quoted 4 lines

iirc, we decided that synchro-test wasn't valuable/important enough to put into mainline. So it has been living in -mm for, umm, seven years.

Let's revisit that decision. What do you think is the case for inclusion?

Sorry for the late reply. I agree that this isn't useful on an everyday basis, however such a test could be useful whenever some locking discussions come up, such as with the spinlock performance discussions when I started this thread, or the rwsem write lock stealing right now. Also, I think we've accepted a lot more non-core-kernel things over the last 7 years, so the bar may be lower now.

Quoted 1 line

Here it is. It will need a full re-review.

I think the first thing we may want to do is ask David if he has an updated version - if he's been making changes, it would be rude to him to check in a 7 year old version of his code.

Quoted 13 lines

From: David Howells <[email protected]>
Subject: mutex subsystem, synchro-test module

The attached patch adds a module for testing and benchmarking mutexes, semaphores and R/W semaphores.

Using it is simple:

insmod synchro-test.ko <args>

It will exit with error ENOANO after running the tests and printing the results to the kernel console log.

Quoted 6 lines

The available arguments are:

(*) mx=N

Start up to N mutex thrashing threads, where N is at most 20. All will try and thrash the same mutex.

Actually N is at most 64

Quoted 4 lines

(*) sm=N

Start up to N counting semaphore thrashing threads, where N is at most
20. All will try and thrash the same semaphore.

N is at most 64

Quoted 5 lines

(*) ism=M

Initialise the counting semaphore with M, where M is any positive integer greater than zero. The default is 4.

Quoted 7 lines

(*) rd=N
(*) wr=O
(*) dg=P

Start up to N reader thrashing threads, O writer thrashing threads and P downgrader thrashing threads, where N, O and P are at most 20 apiece. All will try and thrash the same read/write semaphore.

N, O and P are at most 64

Quoted 7 lines

(*) elapse=N

Run the tests for N seconds. The default is 5.

(*) load=N

Each thread delays for N uS whilst holding the lock. The dfault is 0.

default is 2 (I had actually sent a change to correct the doc here)

Quoted 4 lines

(*) interval=N

Each thread delays for N uS whilst not holding the lock. The default is 0.

default is 2 (I had actually sent a change to correct the doc here)

Quoted 23 lines

(*) do_sched=1

Each thread will call schedule if required after each iteration.

(*) v=1

Print more verbose information, including a thread iteration distribution list.

The module should be enabled by turning on CONFIG_DEBUG_SYNCHRO_TEST to "m".

[[email protected]: fix build errors, add <sched.h> header file]
[[email protected]: remove smp_lock.h inclusion]
[[email protected]: kill daemonize() calls]
[[email protected]: fix printk format warrnings]
[[email protected]: add spinlock test]
[[email protected]: document default load and interval values]
Quoted 7 lines

---

 Documentation/synchro-test.txt |   59 +++
 kernel/Makefile                |    1
 kernel/synchro-test.c          |  586 +++++++++++++++++++++++++++++++
 lib/Kconfig.debug              |   14
 4 files changed, 660 insertions(+)

It's unclear if synchro-test.c should be in kernel/ - I think similar tests nowadays tend to be in lib/ instead, though not always :)

Quoted 13 lines

diff -puN /dev/null Documentation/synchro-test.txt

Documentation/synchro-test.txt new file +9 -0
@@ -0,0 +1,59 @@
+The synchro-test.ko module can be used for testing and benchmarking mutexes,
+semaphores and R/W semaphores.
+
+The module is compiled by setting CONFIG_DEBUG_SYNCHRO_TEST to "m" when
+configuring the kernel.
+
+Using it is simple:
+
+       insmod synchro-test.ko <args>

Documentation/synchro-test.txt line 19

Quoted 10 lines
Documentation/synchro-test.txt new file +10 -0
+
+It will exit with error ENOANO after running the tests and printing the
+results to the kernel console log.
+
+The available arguments are:
+
+ (*) mx=N
+
+       Start up to N mutex thrashing threads, where N is at most 20. All will
+       try and thrash the same mutex.

max 64

Documentation/synchro-test.txt line 24

Quoted 4 lines
Documentation/synchro-test.txt new file +4 -0
+ (*) sm=N
+
+       Start up to N counting semaphore thrashing threads, where N is at most
+       20. All will try and thrash the same semaphore.

max 64

Documentation/synchro-test.txt line 37

Quoted 12 lines
Documentation/synchro-test.txt new file +12 -0
+ (*) ism=M
+
+       Initialise the counting semaphore with M, where M is any positive
+       integer greater than zero. The default is 4.
+
+ (*) rd=N
+ (*) wr=O
+ (*) dg=P
+
+       Start up to N reader thrashing threads, O writer thrashing threads and
+       P downgrader thrashing threads, where N, O and P are at most 20
+       apiece. All will try and thrash the same read/write semaphore.

max 64

Quoted 11 lines
Documentation/synchro-test.txt new file +11 -0
+ (*) elapse=N
+
+       Run the tests for N seconds. The default is 5.
+
+ (*) load=N
+
+       Each thread delays for N uS whilst holding the lock. The default is 2.
+
+ (*) interval=N
+
+       Each thread delays for N uS whilst not holding the lock. The default

Documentation/synchro-test.txt line 59

Quoted 10 lines
Documentation/synchro-test.txt new file +10 -0
+       is 2.
+
+ (*) do_sched=1
+
+       Each thread will call schedule if required after each iteration.
+
+ (*) v=1
+
+       Print more verbose information, including a thread iteration
+       distribution list.

Other than that I'm happy enough with the rest of this code :)

Signature · 2 lines
Michel "Walken" Lespinasse
A program is never fully debugged until the last user dies.